⚡ perf: track import bunch size incrementally instead of full serialize

app/code/Magento/ImportExport/Model/Import/AbstractEntity.php

@@ -419,6 +419,7 @@ protected function _saveValidatedBunches()
     {
         $source = $this->getSource();
         $bunchRows = [];
+        $bunchRowsSize = 2;
         $startNewBunch = false;
 
         $source->rewind();
@@ -438,6 +439,7 @@ protected function _saveValidatedBunches()
                     $this->_dataSourceModel->saveBunch($this->getEntityTypeCode(), $this->getBehavior(), $bunchRows);
 
                 $bunchRows = [];
+                $bunchRowsSize = 2;
                 $startNewBunch = false;
             }
             if ($source->valid()) {
@@ -464,13 +466,18 @@ protected function _saveValidatedBunches()
                     /* Add entity group that passed validation to bunch */
                     if (isset($entityGroup)) {
                         foreach ($entityGroup as $key => $value) {
+                            if ($bunchRows) {
+                                $bunchRowsSize++;
+                            }
                             $bunchRows[$key] = $value;
+                            $bunchRowsSize += strlen($this->serializer->serialize($value))
+                                + strlen($this->serializer->serialize((string)$key))
+                                + 1;
                         }
-                        $productDataSize = strlen($this->serializer->serialize($bunchRows));
 
                         /* Check if the new bunch should be started */
                         $isBunchSizeExceeded = ($this->_bunchSize > 0 && count($bunchRows) >= $this->_bunchSize);
-                        $startNewBunch = $productDataSize >= $this->_maxDataSize || $isBunchSizeExceeded;
+                        $startNewBunch = $bunchRowsSize >= $this->_maxDataSize || $isBunchSizeExceeded;
                     }
 
                     /* And start a new one */

app/code/Magento/ImportExport/Test/Unit/Model/Import/EntityAbstractTest.php

@@ -23,6 +23,7 @@
 use Magento\ImportExport\Model\Import\AbstractSource;
 use Magento\ImportExport\Model\ImportFactory;
 use Magento\ImportExport\Model\ResourceModel\Helper;
+use Magento\ImportExport\Model\ResourceModel\Import\Data as ImportData;
 use PHPUnit\Framework\MockObject\MockObject;
 
 class EntityAbstractTest extends AbstractImportTestCase
@@ -576,6 +577,92 @@ public function testValidateDataAttributeNames()
         $this->assertEquals(1, $errorAggregator->getErrorsCount());
     }
 
+    /**
+     * Test for method _saveValidatedBunches()
+     *
+     * @covers \Magento\ImportExport\Model\Import\AbstractEntity::_saveValidatedBunches
+     */
+    public function testSaveValidatedBunchesSplitsRowsAtLegacySerializedSizeThreshold()
+    {
+        $rows = [
+            ['sku' => 'sku-1', 'description' => str_repeat('a', 60)],
+            ['sku' => 'sku-2', 'description' => str_repeat('b', 60)],
+            ['sku' => 'sku-3', 'description' => str_repeat('c', 60)],
+            ['sku' => 'sku-4', 'description' => str_repeat('d', 60)],
+        ];
+        $maxDataSize = 200;
+        $savedBunches = [];
+        $dataSourceModel = $this->getMockBuilder(ImportData::class)
+            ->disableOriginalConstructor()
+            ->onlyMethods(['cleanProcessedBunches', 'saveBunch'])
+            ->getMock();
+        $dataSourceModel->expects($this->once())
+            ->method('cleanProcessedBunches');
+        $dataSourceModel->expects($this->exactly(2))
+            ->method('saveBunch')
+            ->willReturnCallback(
+                function (string $entityTypeCode, string $behavior, array $bunchRows) use (&$savedBunches): int {
+                    $savedBunches[] = $bunchRows;
+                    return count($savedBunches);
+                }
+            );
+
+        $model = $this->createImportEntityModel($dataSourceModel, $maxDataSize);
+        $model->setSource($this->createImportSource($rows));
+
+        $saveValidatedBunches = new \ReflectionMethod($model, '_saveValidatedBunches');
+        $saveValidatedBunches->invoke($model);
+
+        $expectedBunches = $this->simulateLegacyBunches($rows, $maxDataSize);
+
+        $this->assertCount(2, $savedBunches);
+        $this->assertSame($expectedBunches, $savedBunches);
+    }
+
+    /**
+     * @covers \Magento\ImportExport\Model\Import\AbstractEntity::_saveValidatedBunches
+     */
+    public function testSaveValidatedBunchesAccountsForNonSequentialKeysAfterFlush()
+    {
+        $rows = [
+            ['sku' => 'sku-1', 'description' => str_repeat('a', 30)],
+            ['sku' => 'sku-2', 'description' => str_repeat('b', 30)],
+            ['sku' => 'sku-3', 'description' => str_repeat('c', 30)],
+            ['sku' => 'sku-4', 'description' => str_repeat('d', 30)],
+            ['sku' => 'sku-5', 'description' => str_repeat('e', 30)],
+            ['sku' => 'sku-6', 'description' => str_repeat('f', 30)],
+            ['sku' => 'sku-7', 'description' => str_repeat('g', 30)],
+            ['sku' => 'sku-8', 'description' => str_repeat('h', 30)],
+        ];
+        $maxDataSize = 128;
+        $savedBunches = [];
+        $dataSourceModel = $this->getMockBuilder(ImportData::class)
+            ->disableOriginalConstructor()
+            ->onlyMethods(['cleanProcessedBunches', 'saveBunch'])
+            ->getMock();
+        $dataSourceModel->expects($this->once())
+            ->method('cleanProcessedBunches');
+        $dataSourceModel->expects($this->exactly(4))
+            ->method('saveBunch')
+            ->willReturnCallback(
+                function (string $entityTypeCode, string $behavior, array $bunchRows) use (&$savedBunches): int {
+                    $savedBunches[] = $bunchRows;
+                    return count($savedBunches);
+                }
+            );
+
+        $model = $this->createImportEntityModel($dataSourceModel, $maxDataSize);
+        $model->setSource($this->createImportSource($rows, 100));
+
+        $saveValidatedBunches = new \ReflectionMethod($model, '_saveValidatedBunches');
+        $saveValidatedBunches->invoke($model);
+
+        $expectedBunches = $this->simulateLegacyBunches($rows, $maxDataSize, 100);
+
+        $this->assertCount(4, $savedBunches);
+        $this->assertSame($expectedBunches, $savedBunches);
+    }
+
     /**
      * Create source adapter mock and set it into model object which tested in this class
      *
@@ -591,4 +678,141 @@ protected function _createSourceAdapterMock(array $columns)
 
         return $source;
     }
+
+    /**
+     * @param ImportData|MockObject $dataSourceModel
+     * @param int $maxDataSize
+     * @return AbstractEntity
+     */
+    private function createImportEntityModel(ImportData|MockObject $dataSourceModel, int $maxDataSize): AbstractEntity
+    {
+        $scopeConfig = $this->createMock(ScopeConfigInterface::class);
+        $importFactory = $this->createMock(ImportFactory::class);
+        $resourceHelper = $this->createMock(Helper::class);
+        $resource = $this->createMock(ResourceConnection::class);
+
+        return new class (
+            new StringUtils(),
+            $scopeConfig,
+            $importFactory,
+            $resourceHelper,
+            $resource,
+            $this->getErrorAggregatorObject(),
+            [
+                'data_source_model' => $dataSourceModel,
+                'connection' => 'not_used',
+                'helpers' => [],
+                'page_size' => 1,
+                'max_data_size' => $maxDataSize,
+                'bunch_size' => 0,
+                'collection_by_pages_iterator' => 'not_used',
+            ],
+            new Json()
+        ) extends AbstractEntity {
+            protected $masterAttributeCode = 'sku';
+
+            protected function _importData()
+            {
+                return true;
+            }
+
+            public function getEntityTypeCode()
+            {
+                return 'test_entity';
+            }
+
+            public function validateRow(array $rowData, $rowNumber)
+            {
+                return true;
+            }
+        };
+    }
+
+    /**
+     * @param array $rows
+     * @return AbstractSource
+     */
+    private function createImportSource(array $rows, int $keyOffset = 0): AbstractSource
+    {
+        return new class ($rows, $keyOffset) extends AbstractSource {
+            /**
+             * @var array
+             */
+            private array $rows;
+
+            /**
+             * @var int
+             */
+            private int $position = 0;
+
+            /**
+             * @var int
+             */
+            private int $keyOffset;
+
+            /**
+             * @param array $rows
+             * @param int $keyOffset
+             */
+            public function __construct(array $rows, int $keyOffset)
+            {
+                $this->rows = array_values($rows);
+                $this->keyOffset = $keyOffset;
+                parent::__construct(array_keys($this->rows[0]));
+            }
+
+            /**
+             * @return array|false
+             */
+            protected function _getNextRow()
+            {
+                if (!isset($this->rows[$this->position])) {
+                    return false;
+                }
+
+                return array_values($this->rows[$this->position++]);
+            }
+
+            /**
+             * @return int
+             */
+            #[\ReturnTypeWillChange]
+            public function key()
+            {
+                return parent::key() + $this->keyOffset;
+            }
+        };
+    }
+
+    /**
+     * @param array $rows
+     * @param int $maxDataSize
+     * @return array
+     */
+    private function simulateLegacyBunches(array $rows, int $maxDataSize, int $keyOffset = 0): array
+    {
+        $serializer = new Json();
+        $bunchRows = [];
+        $bunches = [];
+        $previousRow = null;
+
+        foreach ($rows as $key => $row) {
+            if ($previousRow !== null) {
+                $bunchRows[$keyOffset + $key - 1] = $previousRow;
+                if (strlen($serializer->serialize($bunchRows)) >= $maxDataSize) {
+                    $bunches[] = $bunchRows;
+                    $bunchRows = [];
+                }
+            }
+
+            $previousRow = $row;
+        }
+
+        if ($previousRow !== null) {
+            $bunchRows[$keyOffset + count($rows) - 1] = $previousRow;
+            $bunches[] = $bunchRows;
+        }
+
+        return $bunches;
+    }
 }