Merge pull request #17649 from cakephp/5.1-unbuffered-queries

5.1 unbuffered queries

src/Database/Driver.php

@@ -35,6 +35,7 @@
 use Cake\Database\Schema\TableSchema;
 use Cake\Database\Schema\TableSchemaInterface;
 use Cake\Database\Statement\Statement;
+use Closure;
 use InvalidArgumentException;
 use PDO;
 use PDOException;
@@ -391,13 +392,29 @@ public function prepare(Query|string $query): StatementInterface
             );
         }
 
-        $typeMap = null;
-        if ($query instanceof SelectQuery && $query->isResultsCastingEnabled()) {
-            $typeMap = $query->getSelectTypeMap();
+        /** @var \Cake\Database\StatementInterface */
+        return new (static::STATEMENT_CLASS)($statement, $this, $this->getResultSetDecorators($query));
+    }
+
+    /**
+     * Returns the decorators to be applied to the result set incase of a SelectQuery.
+     *
+     * @param \Cake\Database\Query|string $query The query to be decorated.
+     * @return array<\Closure>
+     */
+    protected function getResultSetDecorators(Query|string $query): array
+    {
+        if ($query instanceof SelectQuery) {
+            $decorators = $query->getResultDecorators();
+            if ($query->isResultsCastingEnabled()) {
+                $typeConverter = new FieldTypeConverter($query->getSelectTypeMap(), $this);
+                array_unshift($decorators, Closure::fromCallable($typeConverter));
+            }
+
+            return $decorators;
         }
 
-        /** @var \Cake\Database\StatementInterface */
-        return new (static::STATEMENT_CLASS)($statement, $this, $typeMap);
+        return [];
     }
 
     /**

src/Database/Driver/Mysql.php

@@ -18,8 +18,11 @@
 
 use Cake\Database\Driver;
 use Cake\Database\DriverFeatureEnum;
+use Cake\Database\Query;
+use Cake\Database\Query\SelectQuery;
 use Cake\Database\Schema\MysqlSchemaDialect;
 use Cake\Database\Schema\SchemaDialect;
+use Cake\Database\StatementInterface;
 use PDO;
 
 /**
@@ -157,6 +160,28 @@ public function connect(): void
         }
     }
 
+    /**
+     * @inheritDoc
+     */
+    public function run(Query $query): StatementInterface
+    {
+        $statement = $this->prepare($query);
+        $query->getValueBinder()->attachTo($statement);
+
+        if ($query instanceof SelectQuery) {
+            try {
+                $this->getPdo()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, $query->isBufferedResultsEnabled());
+                $this->executeStatement($statement);
+            } finally {
+                $this->getPdo()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, true);
+            }
+        } else {
+            $this->executeStatement($statement);
+        }
+
+        return $statement;
+    }
+
     /**
      * Returns whether php is able to use this driver for connecting to database
      *

src/Database/Driver/Sqlserver.php

@@ -214,13 +214,8 @@ public function prepare(Query|string $query): StatementInterface
             ]
         );
 
-        $typeMap = null;
-        if ($query instanceof SelectQuery && $query->isResultsCastingEnabled()) {
-            $typeMap = $query->getSelectTypeMap();
-        }
-
         /** @var \Cake\Database\StatementInterface */
-        return new (static::STATEMENT_CLASS)($statement, $this, $typeMap);
+        return new (static::STATEMENT_CLASS)($statement, $this, $this->getResultSetDecorators($query));
     }
 
     /**

src/Database/Query/SelectQuery.php

@@ -74,7 +74,7 @@ class SelectQuery extends Query implements IteratorAggregate
      * statement upon retrieval. Each one of the callback function will receive
      * the row array as first argument.
      *
-     * @var array<\Closure>
+     * @var list<\Closure>
      */
     protected array $_resultDecorators = [];
 
@@ -85,6 +85,14 @@ class SelectQuery extends Query implements IteratorAggregate
      */
     protected ?iterable $_results = null;
 
+    /**
+     * Boolean for tracking whether buffered results
+     * are enabled.
+     *
+     * @var bool
+     */
+    protected bool $bufferedResults = true;
+
     /**
      * The Type map for fields in the select clause
      *
@@ -111,13 +119,6 @@ public function all(): iterable
     {
         if ($this->_results === null || $this->_dirty) {
             $this->_results = $this->execute()->fetchAll(StatementInterface::FETCH_TYPE_ASSOC);
-            if ($this->_resultDecorators) {
-                foreach ($this->_results as &$row) {
-                    foreach ($this->_resultDecorators as $decorator) {
-                        $row = $decorator($row);
-                    }
-                }
-            }
         }
 
         return $this->_results;
@@ -765,13 +766,17 @@ public function unionAll(Query|string $query, bool $overwrite = false)
      */
     public function getIterator(): Traversable
     {
-        /** @var \Traversable|array $results */
-        $results = $this->all();
-        if (is_array($results)) {
-            return new ArrayIterator($results);
+        if ($this->bufferedResults) {
+            /** @var \Traversable|array $results */
+            $results = $this->all();
+            if (is_array($results)) {
+                return new ArrayIterator($results);
+            }
+
+            return $results;
         }
 
-        return $results;
+        return $this->execute();
     }
 
     /**
@@ -816,6 +821,69 @@ public function decorateResults(?Closure $callback, bool $overwrite = false)
         return $this;
     }
 
+    /**
+     * Get result decorators.
+     *
+     * @return array
+     */
+    public function getResultDecorators(): array
+    {
+        return $this->_resultDecorators;
+    }
+
+    /**
+     * Enables buffered results.
+     *
+     * When enabled the results returned by this query will be
+     * buffered. This enables you to iterate a result set multiple times, or
+     * both cache and iterate it.
+     *
+     * When disabled it will consume less memory as fetched results are not
+     * remembered for future iterations.
+     *
+     * @return $this
+     */
+    public function enableBufferedResults()
+    {
+        $this->_dirty();
+        $this->bufferedResults = true;
+
+        return $this;
+    }
+
+    /**
+     * Disables buffered results.
+     *
+     * Disabling buffering will consume less memory as fetched results are not
+     * remembered for future iterations.
+     *
+     * @return $this
+     */
+    public function disableBufferedResults()
+    {
+        $this->_dirty();
+        $this->bufferedResults = false;
+
+        return $this;
+    }
+
+    /**
+     * Returns whether buffered results are enabled/disabled.
+     *
+     * When enabled the results returned by this query will be
+     * buffered. This enables you to iterate a result set multiple times, or
+     * both cache and iterate it.
+     *
+     * When disabled it will consume less memory as fetched results are not
+     * remembered for future iterations.
+     *
+     * @return bool
+     */
+    public function isBufferedResultsEnabled(): bool
+    {
+        return $this->bufferedResults;
+    }
+
     /**
      * Sets the TypeMap class where the types for each of the fields in the
      * select clause are stored.

src/Database/Statement/Statement.php

@@ -17,11 +17,10 @@
 namespace Cake\Database\Statement;
 
 use Cake\Database\Driver;
-use Cake\Database\FieldTypeConverter;
 use Cake\Database\StatementInterface;
 use Cake\Database\TypeFactory;
 use Cake\Database\TypeInterface;
-use Cake\Database\TypeMap;
+use Generator;
 use InvalidArgumentException;
 use PDO;
 use PDOStatement;
@@ -42,16 +41,6 @@ class Statement implements StatementInterface
      */
     protected Driver $_driver;
 
-    /**
-     * @var \PDOStatement
-     */
-    protected PDOStatement $statement;
-
-    /**
-     * @var \Cake\Database\FieldTypeConverter|null
-     */
-    protected ?FieldTypeConverter $typeConverter;
-
     /**
      * Cached bound parameters used for logging
      *
@@ -62,16 +51,14 @@ class Statement implements StatementInterface
     /**
      * @param \PDOStatement $statement PDO statement
      * @param \Cake\Database\Driver $driver Database driver
-     * @param \Cake\Database\TypeMap|null $typeMap Results type map
+     * @param list<\Closure> $resultDecorators Results decorators
      */
     public function __construct(
-        PDOStatement $statement,
+        protected PDOStatement $statement,
         Driver $driver,
-        ?TypeMap $typeMap = null,
+        protected array $resultDecorators = [],
     ) {
         $this->_driver = $driver;
-        $this->statement = $statement;
-        $this->typeConverter = $typeMap !== null ? new FieldTypeConverter($typeMap, $driver) : null;
     }
 
     /**
@@ -170,8 +157,8 @@ public function fetch(string|int $mode = PDO::FETCH_NUM): mixed
             return false;
         }
 
-        if ($this->typeConverter !== null) {
-            return ($this->typeConverter)($row);
+        foreach ($this->resultDecorators as $decorator) {
+            $row = $decorator($row);
         }
 
         return $row;
@@ -206,8 +193,8 @@ public function fetchAll(string|int $mode = PDO::FETCH_NUM): array
         $mode = $this->convertMode($mode);
         $rows = $this->statement->fetchAll($mode);
 
-        if ($this->typeConverter !== null) {
-            return array_map($this->typeConverter, $rows);
+        foreach ($this->resultDecorators as $decorator) {
+            $rows = array_map($decorator, $rows);
         }
 
         return $rows;
@@ -297,4 +284,24 @@ public function queryString(): string
     {
         return $this->statement->queryString;
     }
+
+    /**
+     * Get the inner iterator
+     *
+     * @return \Generator
+     */
+    public function getIterator(): Generator
+    {
+        $this->statement->setFetchMode(PDO::FETCH_ASSOC);
+
+        foreach ($this->statement as $row) {
+            foreach ($this->resultDecorators as $decorator) {
+                $row = $decorator($row);
+            }
+
+            yield $row;
+        }
+
+        $this->closeCursor();
+    }
 }

src/Database/StatementInterface.php

@@ -16,9 +16,13 @@
  */
 namespace Cake\Database;
 
+use IteratorAggregate;
 use PDO;
 
-interface StatementInterface
+/**
+ * @template-extends \IteratorAggregate<array>
+ */
+interface StatementInterface extends IteratorAggregate
 {
     /**
      * Maps to PDO::FETCH_NUM.

src/ORM/EagerLoader.php

@@ -608,22 +608,28 @@ protected function _resolveJoins(array $associations, array $matching = []): arr
      *
      * @param \Cake\ORM\Query\SelectQuery $query The query for which to eager load external.
      * associations.
-     * @param array $results Results array.
-     * @return array
+     * @param iterable $results Results.
+     * @return iterable
      * @throws \RuntimeException
      */
-    public function loadExternal(SelectQuery $query, array $results): array
+    public function loadExternal(SelectQuery $query, iterable $results): iterable
     {
         if (!$results) {
             return $results;
         }
 
-        $table = $query->getRepository();
-        $external = $this->externalAssociations($table);
+        $external = $this->externalAssociations($query->getRepository());
         if (!$external) {
             return $results;
         }
 
+        if (!is_array($results)) {
+            $results = iterator_to_array($results);
+        }
+        if (!$results) {
+            return $results;
+        }
+
         $collected = $this->_collectKeys($external, $query, $results);
 
         foreach ($external as $meta) {

src/ORM/Query/SelectQuery.php

@@ -1584,9 +1584,10 @@ protected function _execute(): iterable
             return $this->_results;
         }
 
-        $results = parent::all();
-        if (!is_array($results)) {
-            $results = iterator_to_array($results);
+        if ($this->bufferedResults) {
+            $results = parent::all();
+        } else {
+            $results = $this->execute();
         }
         $results = $this->getEagerLoader()->loadExternal($this, $results);