PHPLIB-351: Cluster and DB-level change streams and startAtOperationTime

src/Client.php

@@ -29,6 +29,7 @@ use MongoDB\Exception\UnsupportedException;
 use MongoDB\Model\DatabaseInfoIterator;
 use MongoDB\Operation\DropDatabase;
 use MongoDB\Operation\ListDatabases;
+use MongoDB\Operation\Watch;
 
 class Client
 {
@@ -37,9 +38,12 @@ class Client
         'document' => 'MongoDB\Model\BSONDocument',
         'root' => 'MongoDB\Model\BSONDocument',
     ];
+    private static $wireVersionForReadConcern = 4;
     private static $wireVersionForWritableCommandWriteConcern = 5;
 
     private $manager;
+    private $readConcern;
+    private $readPreference;
     private $uri;
     private $typeMap;
     private $writeConcern;
@@ -81,6 +85,8 @@ class Client
         unset($driverOptions['typeMap']);
 
         $this->manager = new Manager($uri, $uriOptions, $driverOptions);
+        $this->readConcern = $this->manager->getReadConcern();
+        $this->readPreference = $this->manager->getReadPreference();
         $this->writeConcern = $this->manager->getWriteConcern();
     }
 
@@ -173,7 +179,7 @@ class Client
      */
     public function getReadConcern()
     {
-        return $this->manager->getReadConcern();
+        return $this->readConcern;
     }
 
     /**
@@ -183,7 +189,7 @@ class Client
      */
     public function getReadPreference()
     {
-        return $this->manager->getReadPreference();
+        return $this->readPreference;
     }
 
     /**
@@ -268,4 +274,34 @@ class Client
     {
         return $this->manager->startSession($options);
     }
+
+    /**
+     * Create a change stream for watching changes to the cluster.
+     *
+     * @see Watch::__construct() for supported options
+     * @param array $pipeline List of pipeline operations
+     * @param array $options  Command options
+     * @return ChangeStream
+     * @throws InvalidArgumentException for parameter/option parsing errors
+     */
+    public function watch(array $pipeline = [], array $options = [])
+    {
+        if ( ! isset($options['readPreference'])) {
+            $options['readPreference'] = $this->readPreference;
+        }
+
+        $server = $this->manager->selectServer($options['readPreference']);
+
+        if ( ! isset($options['readConcern']) && \MongoDB\server_supports_feature($server, self::$wireVersionForReadConcern)) {
+            $options['readConcern'] = $this->readConcern;
+        }
+
+        if ( ! isset($options['typeMap'])) {
+            $options['typeMap'] = $this->typeMap;
+        }
+
+        $operation = new Watch($this->manager, null, null, $pipeline, $options);
+
+        return $operation->execute($server);
+    }
 }

src/Database.php

@@ -34,6 +34,7 @@ use MongoDB\Operation\DropCollection;
 use MongoDB\Operation\DropDatabase;
 use MongoDB\Operation\ListCollections;
 use MongoDB\Operation\ModifyCollection;
+use MongoDB\Operation\Watch;
 
 class Database
 {
@@ -42,6 +43,7 @@ class Database
         'document' => 'MongoDB\Model\BSONDocument',
         'root' => 'MongoDB\Model\BSONDocument',
     ];
+    private static $wireVersionForReadConcern = 4;
     private static $wireVersionForWritableCommandWriteConcern = 5;
 
     private $databaseName;
@@ -409,6 +411,36 @@ class Database
         return new Bucket($this->manager, $this->databaseName, $options);
     }
 
+    /**
+     * Create a change stream for watching changes to the database.
+     *
+     * @see Watch::__construct() for supported options
+     * @param array $pipeline List of pipeline operations
+     * @param array $options  Command options
+     * @return ChangeStream
+     * @throws InvalidArgumentException for parameter/option parsing errors
+     */
+    public function watch(array $pipeline = [], array $options = [])
+    {
+        if ( ! isset($options['readPreference'])) {
+            $options['readPreference'] = $this->readPreference;
+        }
+
+        $server = $this->manager->selectServer($options['readPreference']);
+
+        if ( ! isset($options['readConcern']) && \MongoDB\server_supports_feature($server, self::$wireVersionForReadConcern)) {
+            $options['readConcern'] = $this->readConcern;
+        }
+
+        if ( ! isset($options['typeMap'])) {
+            $options['typeMap'] = $this->typeMap;
+        }
+
+        $operation = new Watch($this->manager, $this->databaseName, null, $pipeline, $options);
+
+        return $operation->execute($server);
+    }
+
     /**
      * Get a clone of this database with different options.
      *

src/Operation/Watch.php

@@ -18,13 +18,19 @@
 namespace MongoDB\Operation;
 
 use MongoDB\ChangeStream;
+use MongoDB\BSON\TimestampInterface;
 use MongoDB\Driver\Command;
+use MongoDB\Driver\Cursor;
 use MongoDB\Driver\Manager;
 use MongoDB\Driver\ReadConcern;
 use MongoDB\Driver\ReadPreference;
 use MongoDB\Driver\Server;
 use MongoDB\Driver\Session;
 use MongoDB\Driver\Exception\RuntimeException;
+use MongoDB\Driver\Monitoring\CommandFailedEvent;
+use MongoDB\Driver\Monitoring\CommandSubscriber;
+use MongoDB\Driver\Monitoring\CommandStartedEvent;
+use MongoDB\Driver\Monitoring\CommandSucceededEvent;
 use MongoDB\Exception\InvalidArgumentException;
 use MongoDB\Exception\UnexpectedValueException;
 use MongoDB\Exception\UnsupportedException;
@@ -32,20 +38,27 @@ use MongoDB\Exception\UnsupportedException;
 /**
  * Operation for creating a change stream with the aggregate command.
  *
+ * Note: the implementation of CommandSubscriber is an internal implementation
+ * detail and should not be considered part of the public API.
+ *
  * @api
  * @see \MongoDB\Collection::watch()
  * @see https://docs.mongodb.com/manual/changeStreams/
  */
-class Watch implements Executable
+class Watch implements Executable, /* @internal */ CommandSubscriber
 {
+    private static $wireVersionForOperationTime = 7;
+
     const FULL_DOCUMENT_DEFAULT = 'default';
     const FULL_DOCUMENT_UPDATE_LOOKUP = 'updateLookup';
 
     private $aggregate;
-    private $databaseName;
+    private $aggregateOptions;
+    private $changeStreamOptions;
     private $collectionName;
+    private $databaseName;
+    private $operationTime;
     private $pipeline;
-    private $options;
     private $resumeCallable;
 
     /**
@@ -79,22 +92,44 @@ class Watch implements Executable
      *  * resumeAfter (document): Specifies the logical starting point for the
      *    new change stream.
      *
+     *    Using this option in conjunction with "startAtOperationTime" will
+     *    result in a server error. The options are mutually exclusive.
+     *
      *  * session (MongoDB\Driver\Session): Client session.
      *
      *    Sessions are not supported for server versions < 3.6.
      *
+     *  * startAtOperationTime (MongoDB\BSON\TimestampInterface): If specified,
+     *    the change stream will only provide changes that occurred at or after
+     *    the specified timestamp. Any command run against the server will
+     *    return an operation time that can be used here. Alternatively, an
+     *    operation time may be obtained from MongoDB\Driver\Server::getInfo().
+     *
+     *    Using this option in conjunction with "resumeAfter" will result in a
+     *    server error. The options are mutually exclusive.
+     *
+     *    This option is not supported for server versions < 4.0.
+     *
      *  * typeMap (array): Type map for BSON deserialization. This will be
      *    applied to the returned Cursor (it is not sent to the server).
      *
-     * @param string         $databaseName   Database name
-     * @param string         $collectionName Collection name
+     * Note: A database-level change stream may be created by specifying null
+     * for the collection name. A cluster-level change stream may be created by
+     * specifying null for both the database and collection name.
+     *
+     * @param Manager        $manager        Manager instance from the driver
+     * @param string|null    $databaseName   Database name
+     * @param string|null    $collectionName Collection name
      * @param array          $pipeline       List of pipeline operations
      * @param array          $options        Command options
-     * @param Manager        $manager        Manager instance from the driver
      * @throws InvalidArgumentException for parameter/option parsing errors
      */
     public function __construct(Manager $manager, $databaseName, $collectionName, array $pipeline, array $options = [])
     {
+        if (isset($collectionName) && ! isset($databaseName)) {
+            throw new InvalidArgumentException('$collectionName should also be null if $databaseName is null');
+        }
+
         $options += [
             'fullDocument' => self::FULL_DOCUMENT_DEFAULT,
             'readPreference' => new ReadPreference(ReadPreference::RP_PRIMARY),
@@ -104,10 +139,12 @@ class Watch implements Executable
             throw InvalidArgumentException::invalidType('"fullDocument" option', $options['fullDocument'], 'string');
         }
 
-        if (isset($options['resumeAfter'])) {
-            if ( ! is_array($options['resumeAfter']) && ! is_object($options['resumeAfter'])) {
-                throw InvalidArgumentException::invalidType('"resumeAfter" option', $options['resumeAfter'], 'array or object');
-            }
+        if (isset($options['resumeAfter']) && ! is_array($options['resumeAfter']) && ! is_object($options['resumeAfter'])) {
+            throw InvalidArgumentException::invalidType('"resumeAfter" option', $options['resumeAfter'], 'array or object');
+        }
+
+        if (isset($options['startAtOperationTime']) && ! $options['startAtOperationTime'] instanceof TimestampInterface) {
+            throw InvalidArgumentException::invalidType('"startAtOperationTime" option', $options['startAtOperationTime'], TimestampInterface::class);
         }
 
         /* In the absence of an explicit session, create one to ensure that the
@@ -122,15 +159,47 @@ class Watch implements Executable
             }
         }
 
+        $this->aggregateOptions = array_intersect_key($options, ['batchSize' => 1, 'collation' => 1, 'maxAwaitTimeMS' => 1, 'readConcern' => 1, 'readPreference' => 1, 'session' => 1, 'typeMap' => 1]);
+        $this->changeStreamOptions = array_intersect_key($options, ['fullDocument' => 1, 'resumeAfter' => 1, 'startAtOperationTime' => 1]);
+
+        // Null database name implies a cluster-wide change stream
+        if ($databaseName === null) {
+            $databaseName = 'admin';
+            $this->changeStreamOptions['allChangesForCluster'] = true;
+        }
+
         $this->databaseName = (string) $databaseName;
-        $this->collectionName = (string) $collectionName;
+        $this->collectionName = isset($collectionName) ? (string) $collectionName : null;
         $this->pipeline = $pipeline;
-        $this->options = $options;
 
         $this->aggregate = $this->createAggregate();
         $this->resumeCallable = $this->createResumeCallable($manager);
     }
 
+    /** @internal */
+    final public function commandFailed(CommandFailedEvent $event)
+    {
+    }
+
+    /** @internal */
+    final public function commandStarted(CommandStartedEvent $event)
+    {
+    }
+
+    /** @internal */
+    final public function commandSucceeded(CommandSucceededEvent $event)
+    {
+        if ($event->getCommandName() !== 'aggregate') {
+            return;
+        }
+
+        $reply = $event->getReply();
+
+        if (isset($reply->operationTime) && $reply->operationTime instanceof TimestampInterface) {
+            $this->operationTime = $reply->operationTime;
+        }
+    }
+
     /**
      * Execute the operation.
      *
@@ -142,47 +211,74 @@ class Watch implements Executable
      */
     public function execute(Server $server)
     {
-        $cursor = $this->aggregate->execute($server);
-
-        return new ChangeStream($cursor, $this->resumeCallable);
+        return new ChangeStream($this->executeAggregate($server), $this->resumeCallable);
     }
 
     /**
      * Create the aggregate command for creating a change stream.
      *
-     * This method is also used to recreate the aggregate command if a new
-     * resume token is provided while resuming.
+     * This method is also used to recreate the aggregate command when resuming.
      *
      * @return Aggregate
      */
     private function createAggregate()
     {
-        $changeStreamOptions = array_intersect_key($this->options, ['fullDocument' => 1, 'resumeAfter' => 1]);
-        $changeStream = ['$changeStream' => (object) $changeStreamOptions];
-
         $pipeline = $this->pipeline;
-        array_unshift($pipeline, $changeStream);
+        array_unshift($pipeline, ['$changeStream' => (object) $this->changeStreamOptions]);
 
-        $aggregateOptions = array_intersect_key($this->options, ['batchSize' => 1, 'collation' => 1, 'maxAwaitTimeMS' => 1, 'readConcern' => 1, 'readPreference' => 1, 'session' => 1, 'typeMap' => 1]);
-
-        return new Aggregate($this->databaseName, $this->collectionName, $pipeline, $aggregateOptions);
+        return new Aggregate($this->databaseName, $this->collectionName, $pipeline, $this->aggregateOptions);
     }
 
     private function createResumeCallable(Manager $manager)
     {
         return function($resumeToken = null) use ($manager) {
-            /* If a resume token was provided, recreate the Aggregate operation
-             * using the new resume token. */
+            /* If a resume token was provided, update the "resumeAfter" option
+             * and ensure that "startAtOperationTime" is no longer set. */
             if ($resumeToken !== null) {
-                $this->options['resumeAfter'] = $resumeToken;
-                $this->aggregate = $this->createAggregate();
+                $this->changeStreamOptions['resumeAfter'] = $resumeToken;
+                unset($this->changeStreamOptions['startAtOperationTime']);
+            }
+
+            /* If we captured an operation time from the first aggregate command
+             * and there is no "resumeAfter" option, set "startAtOperationTime"
+             * so that we can resume from the original aggregate's time. */
+            if ($this->operationTime !== null && ! isset($this->changeStreamOptions['resumeAfter'])) {
+                $this->changeStreamOptions['startAtOperationTime'] = $this->operationTime;
             }
 
+            $this->aggregate = $this->createAggregate();
+
             /* Select a new server using the read preference, execute this
              * operation on it, and return the new ChangeStream. */
-            $server = $manager->selectServer($this->options['readPreference']);
+            $server = $manager->selectServer($this->aggregateOptions['readPreference']);
 
             return $this->execute($server);
         };
     }
+
+    /**
+     * Execute the aggregate command and optionally capture its operation time.
+     *
+     * @param Server $server
+     * @return Cursor
+     */
+    private function executeAggregate(Server $server)
+    {
+        /* If we've already captured an operation time or the server does not
+         * support returning an operation time (e.g. MongoDB 3.6), execute the
+         * aggregation directly and return its cursor. */
+        if ($this->operationTime !== null || ! \MongoDB\server_supports_feature($server, self::$wireVersionForOperationTime)) {
+            return $this->aggregate->execute($server);
+        }
+
+        /* Otherwise, execute the aggregation using command monitoring so that
+         * we can capture its operation time with commandSucceeded(). */
+        \MongoDB\Driver\Monitoring\addSubscriber($this);
+
+        try {
+            return $this->aggregate->execute($server);
+        } finally {
+            \MongoDB\Driver\Monitoring\removeSubscriber($this);
+        }
+    }
 }

tests/Operation/WatchFunctionalTest.php

@@ -3,6 +3,7 @@
 namespace MongoDB\Tests\Operation;
 
 use MongoDB\ChangeStream;
+use MongoDB\BSON\TimestampInterface;
 use MongoDB\Driver\Manager;
 use MongoDB\Driver\ReadPreference;
 use MongoDB\Driver\Server;
@@ -130,6 +131,100 @@ class WatchFunctionalTest extends FunctionalTestCase
         $this->assertSame($expectedCommands, $commands);
     }
 
+    public function testResumeBeforeReceivingAnyResultsIncludesStartAtOperationTime()
+    {
+        $operation = new Watch($this->manager, $this->getDatabaseName(), $this->getCollectionName(), [], $this->defaultOptions);
+
+        $operationTime = null;
+        $events = [];
+
+        (new CommandObserver)->observe(
+            function() use ($operation, &$changeStream) {
+                $changeStream = $operation->execute($this->getPrimaryServer());
+            },
+            function (array $event) use (&$events) {
+                $events[] = $event;
+            }
+        );
+
+        $this->assertCount(1, $events);
+        $this->assertSame('aggregate', $events[0]['started']->getCommandName());
+        $operationTime = $events[0]['succeeded']->getReply()->operationTime;
+        $this->assertInstanceOf(TimestampInterface::class, $operationTime);
+
+        $this->assertNull($changeStream->current());
+        $this->killChangeStreamCursor($changeStream);
+
+        $events = [];
+
+        (new CommandObserver)->observe(
+            function() use ($changeStream) {
+                $changeStream->rewind();
+            },
+            function (array $event) use (&$events) {
+                $events[] = $event;
+            }
+        );
+
+        $this->assertCount(4, $events);
+
+        $this->assertSame('getMore', $events[0]['started']->getCommandName());
+        $this->arrayHasKey('failed', $events[0]);
+
+        $this->assertSame('aggregate', $events[1]['started']->getCommandName());
+        $this->assertStartAtOperationTime($operationTime, $events[1]['started']->getCommand());
+        $this->arrayHasKey('succeeded', $events[1]);
+
+        // Original cursor is freed immediately after the change stream resumes
+        $this->assertSame('killCursors', $events[2]['started']->getCommandName());
+        $this->arrayHasKey('succeeded', $events[2]);
+
+        $this->assertSame('getMore', $events[3]['started']->getCommandName());
+        $this->arrayHasKey('succeeded', $events[3]);
+
+        $this->assertNull($changeStream->current());
+        $this->killChangeStreamCursor($changeStream);
+
+        $events = [];
+
+        (new CommandObserver)->observe(
+            function() use ($changeStream) {
+                $changeStream->next();
+            },
+            function (array $event) use (&$events) {
+                $events[] = $event;
+            }
+        );
+
+        $this->assertCount(4, $events);
+
+        $this->assertSame('getMore', $events[0]['started']->getCommandName());
+        $this->arrayHasKey('failed', $events[0]);
+
+        $this->assertSame('aggregate', $events[1]['started']->getCommandName());
+        $this->assertStartAtOperationTime($operationTime, $events[1]['started']->getCommand());
+        $this->arrayHasKey('succeeded', $events[1]);
+
+        // Original cursor is freed immediately after the change stream resumes
+        $this->assertSame('killCursors', $events[2]['started']->getCommandName());
+        $this->arrayHasKey('succeeded', $events[2]);
+
+        $this->assertSame('getMore', $events[3]['started']->getCommandName());
+        $this->arrayHasKey('succeeded', $events[3]);
+
+        $this->assertNull($changeStream->current());
+    }
+
+    private function assertStartAtOperationTime(TimestampInterface $expectedOperationTime, stdClass $command)
+    {
+        $this->assertObjectHasAttribute('pipeline', $command);
+        $this->assertInternalType('array', $command->pipeline);
+        $this->assertArrayHasKey(0, $command->pipeline);
+        $this->assertObjectHasAttribute('$changeStream', $command->pipeline[0]);
+        $this->assertObjectHasAttribute('startAtOperationTime', $command->pipeline[0]->{'$changeStream'});
+        $this->assertEquals($expectedOperationTime, $command->pipeline[0]->{'$changeStream'}->startAtOperationTime);
+    }
+
     public function testRewindResumesAfterConnectionException()
     {
         /* In order to trigger a dropped connection, we'll use a new client with

tests/Operation/WatchTest.php

@@ -4,6 +4,7 @@ namespace MongoDB\Tests\Operation;
 
 use MongoDB\Exception\InvalidArgumentException;
 use MongoDB\Operation\Watch;
+use stdClass;
 
 /**
  * Although these are unit tests, we extend FunctionalTestCase because Watch is
@@ -11,6 +12,14 @@ use MongoDB\Operation\Watch;
  */
 class WatchTest extends FunctionalTestCase
 {
+    public function testConstructorCollectionNameShouldBeNullIfDatabaseNameIsNull()
+    {
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('$collectionName should also be null if $databaseName is null');
+
+        new Watch($this->manager, null, 'foo', []);
+    }
+
     public function testConstructorPipelineArgumentMustBeAList()
     {
         $this->expectException(InvalidArgumentException::class);
@@ -67,10 +76,19 @@ class WatchTest extends FunctionalTestCase
             $options[][] = ['session' => $value];
         }
 
+        foreach ($this->getInvalidTimestampValues() as $value) {
+            $options[][] = ['startAtOperationTime' => $value];
+        }
+
         foreach ($this->getInvalidArrayValues() as $value) {
             $options[][] = ['typeMap' => $value];
         }
 
         return $options;
     }
+
+    private function getInvalidTimestampValues()
+    {
+        return [123, 3.14, 'foo', true, [], new stdClass];
+    }
 }