Skip to content

M
mongo-php-library
Commit 9b1f1646 authored by Will Banfield's avatar Will Banfield Committed by Jeremy Mikola
Browse files
Options

restructure some aspects to be more inline with SPEC and add additional tests

parent ba3efc05
Hide whitespace changes
Inline Side-by-side
Showing with 341 additions and 67 deletions
src/GridFS/Bucket.php
View file @ 9b1f1646
......@@ -43,10 +43,9 @@ class Bucket
*/
public function __construct(Manager $manager, $databaseName, array $options = [])
{
$collectionOptions = [];
$options += [
'bucketName' => 'fs',
'chunkSizeBytes' => 261120,
'bucketName' => 'fs'
];
$this->databaseName = (string) $databaseName;
$this->options = $options;
......@@ -80,7 +79,7 @@ class Bucket
*/
public function uploadFromStream($filename, $source, array $options = [])
{
$options['chunkSizeBytes'] = $this->options['chunkSizeBytes'];
$options+= ['chunkSizeBytes' => $this->options['chunkSizeBytes']];
$gridFsStream = new GridFsUpload($this->collectionsWrapper, $filename, $options);
return $gridFsStream->uploadFromStream($source);
}
......@@ -92,11 +91,11 @@ class Bucket
*/
public function openDownloadStream(\MongoDB\BSON\ObjectId $id)
{
$options = [
'collectionsWrapper' => $this->collectionsWrapper
];
$context = stream_context_create(['gridfs' => $options]);
return fopen(sprintf('gridfs://%s/%s', $this->databaseName, $id), 'r', false, $context);
$file = $this->collectionsWrapper->getFilesCollection()->findOne(['_id' => $id]);
if (is_null($file)) {
throw new \MongoDB\Exception\GridFSFileNotFoundException($id, $this->collectionsWrapper->getFilesCollection()->getNameSpace());
}
return $this->openDownloadStreamByFile($file);
}
/**
* Downloads the contents of the stored file specified by id and writes
......@@ -106,7 +105,11 @@ class Bucket
*/
public function downloadToStream(\MongoDB\BSON\ObjectId $id, $destination)
{
$gridFsStream = new GridFsDownload($this->collectionsWrapper, $id);
$file = $this->collectionsWrapper->getFilesCollection()->findOne(['_id' => $id]);
if (is_null($file)) {
throw new \MongoDB\Exception\GridFSFileNotFoundException($id, $this->collectionsWrapper->getFilesCollection()->getNameSpace());
}
$gridFsStream = new GridFsDownload($this->collectionsWrapper, $file);
$gridFsStream->downloadToStream($destination);
}
/**
......@@ -122,7 +125,6 @@ class Bucket
if (is_null($file)) {
throw new \MongoDB\Exception\GridFSFileNotFoundException($id, $this->collectionsWrapper->getFilesCollection()->getNameSpace());
}
$this->collectionsWrapper->getFilesCollection()->deleteOne(['_id' => $id]);
}
/**
......@@ -133,12 +135,8 @@ class Bucket
*/
public function openDownloadStreamByName($filename, $revision = -1)
{
$file = $this->bucket->findFileRevision($filename, $revision);
$options = ['bucket' => $this->bucket,
'file' => $file
];
$context = stream_context_create(['gridfs' => $options]);
return fopen(sprintf('gridfs://%s/%s', $this->bucket->getDatabaseName(), $filename), 'r', false, $context);
$file = $this->findFileRevision($filename, $revision);
return $this->openDownloadStreamByFile($file);
}
/**
* Download a file from the GridFS bucket by name. Searches for the file by the specified name then loads data into the stream
......@@ -150,7 +148,7 @@ class Bucket
public function downloadToStreamByName($filename, $destination, $revision=-1)
{
$file = $this->findFileRevision($filename, $revision);
$gridFsStream = new GridFsDownload($this->collectionsWrapper, null, $file);
$gridFsStream = new GridFsDownload($this->collectionsWrapper, $file);
$gridFsStream->downloadToStream($destination);
}
/**
......@@ -163,8 +161,25 @@ class Bucket
{
return $this->collectionsWrapper->getFilesCollection()->find($filter, $options);
}
public function getCollectionsWrapper()
{
return $this->collectionsWrapper;
}
public function getDatabaseName(){
return $this->databaseName;
}
private function openDownloadStreamByFile($file)
{
$options = ['collectionsWrapper' => $this->collectionsWrapper,
'file' => $file
];
$context = stream_context_create(['gridfs' => $options]);
//db/prefix/(filter criteria as BSON}
// find criteria being MongoDB\BSON\fromPHP(['_id' => $file['_id']])
// stream wrapper can explode('/', 3), which returns array of db, prefix, and BSON blob
// MongoDB\BSON\toPHP(bson blob) yields find() criteria
return fopen(sprintf('gridfs://%s/%s', $this->databaseName, $file->filename), 'r', false, $context);
}
private function findFileRevision($filename, $revision)
{
if ($revision < 0) {
......@@ -181,8 +196,4 @@ class Bucket
}
return $file;
}
public function getCollectionsWrapper()
{
return $this->collectionsWrapper;
}
}
src/GridFS/GridFSCollectionsWrapper.php
View file @ 9b1f1646
......@@ -14,8 +14,9 @@ use MongoDB\Exception\UnexpectedValueException;
/**
* Bucket abstracts the GridFS files and chunks collections.
*
* @api
*/
//rename to context options
class GridFSCollectionsWrapper
{
private $filesCollection;
......@@ -44,9 +45,13 @@ class GridFSCollectionsWrapper
public function __construct(Manager $manager, $databaseName, $options)
{
$collectionOptions = [];
$options += [
'bucketName' => 'fs',
];
if (isset($options['bucketName']) && ! is_string($options['bucketName'])) {
throw new InvalidArgumentTypeException('"bucketName" option', $options['bucketName'], 'string');
}
if (isset($options['chunkSizeBytes']) && ! is_integer($options['chunkSizeBytes'])) {
throw new InvalidArgumentTypeException('"chunkSizeBytes" option', $options['chunkSizeBytes'], 'integer');
}
if (isset($options['readPreference'])) {
if (! $options['readPreference'] instanceof ReadPreference) {
throw new InvalidArgumentTypeException('"readPreference" option', $options['readPreference'], 'MongoDB\Driver\ReadPreference');
......
src/GridFS/GridFsDownload.php
View file @ 9b1f1646
......@@ -5,9 +5,8 @@ use MongoDB\Collection;
use MongoDB\Exception\RuntimeException;
use MongoDB\BSON\ObjectId;
/**
* GridFsupload abstracts the processes of inserting into a GridFSBucket
* GridFSDownload abstracts the processes of downloading from a GridFSBucket
*
* @api
*/
class GridFsDownload
{
......@@ -33,24 +32,17 @@ class GridFsDownload
*/
public function __construct(
GridFSCollectionsWrapper $collectionsWrapper,
$objectId,
$file = null
$file
)
{
$this->collectionsWrapper = $collectionsWrapper;
if(!is_null($file)) {
$this->file = $file;
} else {
$this->file = $collectionsWrapper->getFilesCollection()->findOne(['_id' => $objectId]);
if (is_null($this->file)) {
throw new \MongoDB\Exception\GridFSFileNotFoundException($objectId, $this->collectionsWrapper->getFilesCollection()->getNameSpace());
}
}
$this->file = $file;
$cursor = $this->collectionsWrapper->getChunksCollection()->find(['files_id' => $this->file->_id], ['sort' => ['n' => 1]]);
$this->chunksIterator = new \IteratorIterator($cursor);
if ($this->file->length >= 0) {
$cursor = $this->collectionsWrapper->getChunksCollection()->find(['files_id' => $this->file->_id], ['sort' => ['n' => 1]]);
$this->chunksIterator = new \IteratorIterator($cursor);
$this->numChunks = ceil($this->file->length / $this->file->chunkSize);
} else {
$this->numChunks = 0;
}
$this->buffer = fopen('php://temp', 'w+');
}
......@@ -88,15 +80,22 @@ class GridFsDownload
while(strlen($output) < $numToRead && $this->advanceChunks()) {
$bytesLeft = $numToRead - strlen($output);
$output .= substr($this->chunksIterator->current()->data, 0, $bytesLeft);
$output .= substr($this->chunksIterator->current()->data->getData(), 0, $bytesLeft);
}
if ($bytesLeft < strlen($this->chunksIterator->current()->data)) {
fwrite($this->buffer, substr($this->chunksIterator->current()->data, $bytesLeft));
if ($this->file->length > 0 && $bytesLeft < strlen($this->chunksIterator->current()->data->getData())) {
fwrite($this->buffer, substr($this->chunksIterator->current()->data->getData(), $bytesLeft));
$this->bufferEmpty=false;
}
return $output;
}
public function getSize()
{
return $this->file->length;
}
public function getId()
{
return $this->file->_id;
}
private function advanceChunks()
{
if($this->chunkOffset >= $this->numChunks) {
......
src/GridFS/GridFsUpload.php
View file @ 9b1f1646
......@@ -9,7 +9,6 @@ use MongoDB\BSON;
/**
* GridFsupload abstracts the processes of inserting into a GridFSBucket
*
* @api
*/
class GridFsUpload
{
......@@ -56,8 +55,8 @@ class GridFsUpload
$this->collectionsWrapper = $collectionsWrapper;
$this->buffer = fopen('php://temp', 'w+');
$this->chunkSize = $options['chunkSizeBytes'];
$uploadDate = time();
$time = $this->millitime();
$uploadDate = new \MongoDB\BSON\UTCDateTime($time);
$objectId = new \MongoDB\BSON\ObjectId();
$main_file = [
"chunkSize" => $this->chunkSize,
......@@ -151,6 +150,14 @@ class GridFsUpload
fclose($this->buffer);
$this->fileCollectionInsert();
}
public function getSize()
{
return $this->length;
}
public function getId()
{
return $this->file["_id"];
}
private function insertChunk($data)
{
$toUpload = ["files_id" => $this->file['_id'], "n" => $this->chunkOffset, "data" => new \MongoDB\BSON\Binary($data, \MongoDB\BSON\Binary::TYPE_GENERIC)];
......@@ -167,4 +174,10 @@ class GridFsUpload
$this->collectionsWrapper->fileInsert($this->file);
return $this->file['_id'];
}
//from: http://stackoverflow.com/questions/3656713/how-to-get-current-time-in-milliseconds-in-php
private function millitime() {
$microtime = microtime();
$comps = explode(' ', $microtime);
return sprintf('%d%03d', $comps[1], $comps[0] * 1000);
}
}
src/GridFS/StreamWrapper.php
View file @ 9b1f1646
......@@ -33,26 +33,29 @@ class StreamWrapper
}
stream_wrapper_register('gridfs', get_called_class(), STREAM_IS_URL);
}
private function initProtocol($path)
{
$parsed_path = parse_url($path);
$this->databaseName = $parsed_path["host"];
$this->identifier = substr($parsed_path["path"], 1);
}
public function stream_write($data)
{
$this->gridFsStream->insertChunks($data);
return strlen($data);
}
public function stream_read($count) {
public function stream_read($count)
{
return $this->gridFsStream->downloadNumBytes($count);
}
public function stream_eof() {
public function stream_eof()
{
return $this->gridFsStream->isEOF();
}
public function stream_close() {
public function stream_close()
{
$this->gridFsStream->close();
}
public function stream_stat()
{
$stat = $this->getStatTemplate();
$stat[7] = $stat['size'] = $this->gridFsStream->getSize();
$stat[2] = $stat['mode'] = $this->mode;
return $stat;
}
public function stream_open($path, $mode, $options, &$openedPath)
{
......@@ -75,12 +78,37 @@ class StreamWrapper
public function openReadStream() {
$context = stream_context_get_options($this->context);
if(isset($context['gridfs']['file'])){
$this->gridFsStream = new GridFsDownload($this->collectionsWrapper, null, $context['gridfs']['file']);
} else {
$objectId = new \MongoDB\BSON\ObjectId($this->identifier);
$this->gridFsStream = new GridFsDownload($this->collectionsWrapper, $objectId);
}
$this->gridFsStream = new GridFsDownload($this->collectionsWrapper, $context['gridfs']['file']);
return true;
}
/**
* Gets a URL stat template with default values
* from https://github.com/aws/aws-sdk-php/blob/master/src/S3/StreamWrapper.php
* @return array
*/
private function getStatTemplate()
{
return [
0 => 0, 'dev' => 0,
1 => 0, 'ino' => 0,
2 => 0, 'mode' => 0,
3 => 0, 'nlink' => 0,
4 => 0, 'uid' => 0,
5 => 0, 'gid' => 0,
6 => -1, 'rdev' => -1,
7 => 0, 'size' => 0,
8 => 0, 'atime' => 0,
9 => 0, 'mtime' => 0,
10 => 0, 'ctime' => 0,
11 => -1, 'blksize' => -1,
12 => -1, 'blocks' => -1,
];
}
private function initProtocol($path)
{
$parsed_path = parse_url($path);
$this->databaseName = $parsed_path["host"];
$this->identifier = substr($parsed_path["path"], 1);
}
}
tests/GridFS/BucketFunctionalTest.php 0 → 100644
View file @ 9b1f1646
<?php
namespace MongoDB\Tests\GridFS;
use MongoDB\GridFS;
/**
* Functional tests for the Bucket class.
*/
class BucketFunctionalTest extends FunctionalTestCase
{
/**
* @expectedException MongoDB\Exception\InvalidArgumentTypeException
* @dataProvider provideInvalidConstructorOptions
*/
public function testConstructorOptionTypeChecks(array $options)
{
new \MongoDB\GridFS\Bucket($this->manager, $this->getDatabaseName(), $options);
}
public function provideInvalidConstructorOptions()
{
$options = [];
$invalidBucketNames = [123, 3.14, true, [], new \stdClass];
$invalidChunkSizes = ['foo', 3.14, true, [], new \stdClass];
foreach ($this->getInvalidReadPreferenceValues() as $value) {
$options[][] = ['readPreference' => $value];
}
foreach ($this->getInvalidWriteConcernValues() as $value) {
$options[][] = ['writeConcern' => $value];
}
foreach ($invalidBucketNames as $value) {
$options[][] = ['bucketName' => $value];
}
foreach ($invalidChunkSizes as $value) {
$options[][] = ['chunkSizeBytes' => $value];
}
return $options;
}
public function testGetDatabaseName()
{
$this->assertEquals($this->getDatabaseName(), $this->bucket->getDatabaseName());
}
public function testBasicOperations()
{
$id = $this->bucket->uploadFromStream("test_filename", $this->generateStream("hello world"));
$contents = stream_get_contents($this->bucket->openDownloadStream($id));
$this->assertEquals("hello world", $contents);
$this->assertEquals(1, $this->bucket->getCollectionsWrapper()->getFilesCollection()->count());
$this->assertEquals(1, $this->bucket->getCollectionsWrapper()->getChunksCollection()->count());
$this->bucket->delete($id);
$error=null;
try{
$this->bucket->openDownloadStream($id);
} catch(\MongoDB\Exception\Exception $e) {
$error = $e;
}
$fileNotFound = '\MongoDB\Exception\GridFSFileNotFoundException';
$this->assertTrue($error instanceof $fileNotFound);
$this->assertEquals(0, $this->bucket->getCollectionsWrapper()->getFilesCollection()->count());
$this->assertEquals(0, $this->bucket->getCollectionsWrapper()->getChunksCollection()->count());
}
public function testMultiChunkDelete()
{
$id = $this->bucket->uploadFromStream("test_filename", $this->generateStream("hello"), ['chunkSizeBytes'=>1]);
$this->assertEquals(1, $this->bucket->getCollectionsWrapper()->getFilesCollection()->count());
$this->assertEquals(5, $this->bucket->getCollectionsWrapper()->getChunksCollection()->count());
$this->bucket->delete($id);
$this->assertEquals(0, $this->bucket->getCollectionsWrapper()->getFilesCollection()->count());
$this->assertEquals(0, $this->bucket->getCollectionsWrapper()->getChunksCollection()->count());
}
public function testEmptyFile()
{
$id = $this->bucket->uploadFromStream("test_filename",$this->generateStream(""));
$contents = stream_get_contents($this->bucket->openDownloadStream($id));
$this->assertEquals("", $contents);
$this->assertEquals(1, $this->bucket->getCollectionsWrapper()->getFilesCollection()->count());
$this->assertEquals(0, $this->bucket->getCollectionsWrapper()->getChunksCollection()->count());
$raw = $this->bucket->getCollectionsWrapper()->getFilesCollection()->findOne();
$this->assertEquals(0, $raw->length);
$this->assertEquals($id, $raw->_id);
$this->assertTrue($raw->uploadDate instanceof \MongoDB\BSON\UTCDateTime);
$this->assertEquals(255 * 1024, $raw->chunkSize);
$this->assertTrue(is_string($raw->md5));
}
public function testUploadEnsureIndexes()
{
$chunks = $this->bucket->getCollectionsWrapper()->getChunksCollection();
$files = $this->bucket->getCollectionsWrapper()->getFilesCollection();
$this->bucket->uploadFromStream("filename", $this->generateStream("junk"));
$chunksIndexed = false;
foreach($chunks->listIndexes() as $index) {
$chunksIndexed = $chunksIndexed || ($index->isUnique() && $index->getKey() === ['files_id' => 1, 'n' => 1]);
}
$this->assertTrue($chunksIndexed);
$filesIndexed = false;
foreach($files->listIndexes() as $index) {
$filesIndexed = $filesIndexed || ($index->getKey() === ['filename' => 1, 'uploadDate' => 1]);
}
$this->assertTrue($filesIndexed);
}
public function testGetLastVersion()
{
$idOne = $this->bucket->uploadFromStream("test",$this->generateStream("foo"));
//$streamTwo = $this->bucket->openUploadStream("test");
//fwrite($streamTwo, "bar");
//echo "Calling FSTAT\n";
//$stat = fstat($streamTwo);
//$idTwo = $stat['uid'];
//var_dump
//var_dump($idTwo);
//fclose($streamTwo);
$idThree = $this->bucket->uploadFromStream("test",$this->generateStream("baz"));
$this->assertEquals("baz", stream_get_contents($this->bucket->openDownloadStreamByName("test")));
$this->bucket->delete($idThree);
//$this->assertEquals("bar", stream_get_contents($this->bucket->openDownloadStreamByName("test")));
//$this->bucket->delete($idTwo);
$this->assertEquals("foo", stream_get_contents($this->bucket->openDownloadStreamByName("test")));
$this->bucket->delete($idOne);
$error = null;
try{
$this->bucket->openDownloadStreamByName("test");
} catch(\MongoDB\Exception\Exception $e) {
$error = $e;
}
$fileNotFound = '\MongoDB\Exception\GridFSFileNotFoundException';
$this->assertTrue($error instanceof $fileNotFound);
}
public function testGetVersion()
{
$this->bucket->uploadFromStream("test",$this->generateStream("foo"));
$this->bucket->uploadFromStream("test",$this->generateStream("bar"));
$this->bucket->uploadFromStream("test",$this->generateStream("baz"));
$this->assertEquals("foo", stream_get_contents($this->bucket->openDownloadStreamByName("test", 0)));
$this->assertEquals("bar", stream_get_contents($this->bucket->openDownloadStreamByName("test", 1)));
$this->assertEquals("baz", stream_get_contents($this->bucket->openDownloadStreamByName("test", 2)));
$this->assertEquals("baz", stream_get_contents($this->bucket->openDownloadStreamByName("test", -1)));
$this->assertEquals("bar", stream_get_contents($this->bucket->openDownloadStreamByName("test", -2)));
$this->assertEquals("foo", stream_get_contents($this->bucket->openDownloadStreamByName("test", -3)));
$fileNotFound = '\MongoDB\Exception\GridFSFileNotFoundException';
$error = null;
try{
$this->bucket->openDownloadStreamByName("test", 3);
} catch(\MongoDB\Exception\Exception $e) {
$error = $e;
}
$this->assertTrue($error instanceof $fileNotFound);
$error = null;
try{
$this->bucket->openDownloadStreamByName("test", -4);
} catch(\MongoDB\Exception\Exception $e) {
$error = $e;
}
$this->assertTrue($error instanceof $fileNotFound);
}
public function testGridfsFind()
{
$this->bucket->uploadFromStream("two",$this->generateStream("test2"));
usleep(5000);
$this->bucket->uploadFromStream("two",$this->generateStream("test2+"));
usleep(5000);
$this->bucket->uploadFromStream("one",$this->generateStream("test1"));
usleep(5000);
$this->bucket->uploadFromStream("two",$this->generateStream("test2++"));
$cursor = $this->bucket->find(["filename" => "two"]);
$count = count($cursor->toArray());
$this->assertEquals(3, $count);
$cursor = $this->bucket->find([]);
$count = count($cursor->toArray());
$this->assertEquals(4, $count);
$cursor = $this->bucket->find([], ["noCursorTimeout"=>false, "sort"=>["uploadDate"=> -1], "skip"=>1, "limit"=>2]);
$outputs = ["test1", "test2+"];
$i=0;
foreach($cursor as $file){
$contents = stream_get_contents($this->bucket->openDownloadStream($file->_id));
$this->assertEquals($outputs[$i], $contents);
$i++;
}
}
public function testGridInNonIntChunksize()
{
$id = $this->bucket->uploadFromStream("f",$this->generateStream("data"));
$this->bucket->getCollectionsWrapper()->getFilesCollection()->updateOne(["filename"=>"f"],
['$set'=> ['chunkSize' => 100.00]]);
$this->assertEquals("data", stream_get_contents($this->bucket->openDownloadStream($id)));
}
private function generateStream($input)
{
$stream = fopen('php://temp', 'w+');
fwrite($stream, $input);
rewind($stream);
return $stream;
}
}
tests/GridFS/FunctionalTestCase.php
View file @ 9b1f1646
......@@ -3,6 +3,7 @@
namespace MongoDB\Tests\GridFS;
use MongoDB\GridFS;
use MongoDB\Collection;
use MongoDB\Tests\FunctionalTestCase as BaseFunctionalTestCase;
/**
......@@ -16,6 +17,10 @@ abstract class FunctionalTestCase extends BaseFunctionalTestCase
public function setUp()
{
parent::setUp();
foreach(['fs.files', 'fs.chunks'] as $collection){
$col = new Collection($this->manager, sprintf("%s.%s",$this->getDatabaseName(), $collection));
$col->drop();
}
$streamWrapper = new \MongoDB\GridFS\StreamWrapper();
$streamWrapper->register($this->manager);
$this->bucket = new \MongoDB\GridFS\Bucket($this->manager, $this->getDatabaseName());
......@@ -27,7 +32,9 @@ abstract class FunctionalTestCase extends BaseFunctionalTestCase
if ($this->hasFailed()) {
return;
}
//$this->database = new \MongoDB\Database($this->manager, $this->getDatabaseName());
// $this->database->drop();
foreach(['fs.files', 'fs.chunks'] as $collection){
$col = new Collection($this->manager, sprintf("%s.%s",$this->getDatabaseName(), $collection));
$col->drop();
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment