diff --git a/app/code/Magento/ImportExport/Model/Import/AbstractEntity.php b/app/code/Magento/ImportExport/Model/Import/AbstractEntity.php index 1eb2ddbbdb924..da4df547dc462 100644 --- a/app/code/Magento/ImportExport/Model/Import/AbstractEntity.php +++ b/app/code/Magento/ImportExport/Model/Import/AbstractEntity.php @@ -419,6 +419,7 @@ protected function _saveValidatedBunches() { $source = $this->getSource(); $bunchRows = []; + $bunchRowsSize = 2; $startNewBunch = false; $source->rewind(); @@ -438,6 +439,7 @@ protected function _saveValidatedBunches() $this->_dataSourceModel->saveBunch($this->getEntityTypeCode(), $this->getBehavior(), $bunchRows); $bunchRows = []; + $bunchRowsSize = 2; $startNewBunch = false; } if ($source->valid()) { @@ -464,13 +466,18 @@ protected function _saveValidatedBunches() /* Add entity group that passed validation to bunch */ if (isset($entityGroup)) { foreach ($entityGroup as $key => $value) { + if ($bunchRows) { + $bunchRowsSize++; + } $bunchRows[$key] = $value; + $bunchRowsSize += strlen($this->serializer->serialize($value)) + + strlen($this->serializer->serialize((string)$key)) + + 1; } - $productDataSize = strlen($this->serializer->serialize($bunchRows)); /* Check if the new bunch should be started */ $isBunchSizeExceeded = ($this->_bunchSize > 0 && count($bunchRows) >= $this->_bunchSize); - $startNewBunch = $productDataSize >= $this->_maxDataSize || $isBunchSizeExceeded; + $startNewBunch = $bunchRowsSize >= $this->_maxDataSize || $isBunchSizeExceeded; } /* And start a new one */ diff --git a/app/code/Magento/ImportExport/Test/Unit/Model/Import/EntityAbstractTest.php b/app/code/Magento/ImportExport/Test/Unit/Model/Import/EntityAbstractTest.php index f4e43e9c8982d..15f898b4fa9a7 100644 --- a/app/code/Magento/ImportExport/Test/Unit/Model/Import/EntityAbstractTest.php +++ b/app/code/Magento/ImportExport/Test/Unit/Model/Import/EntityAbstractTest.php @@ -23,6 +23,7 @@ use Magento\ImportExport\Model\Import\AbstractSource; use Magento\ImportExport\Model\ImportFactory; use Magento\ImportExport\Model\ResourceModel\Helper; +use Magento\ImportExport\Model\ResourceModel\Import\Data as ImportData; use PHPUnit\Framework\MockObject\MockObject; class EntityAbstractTest extends AbstractImportTestCase @@ -576,6 +577,92 @@ public function testValidateDataAttributeNames() $this->assertEquals(1, $errorAggregator->getErrorsCount()); } + /** + * Test for method _saveValidatedBunches() + * + * @covers \Magento\ImportExport\Model\Import\AbstractEntity::_saveValidatedBunches + */ + public function testSaveValidatedBunchesSplitsRowsAtLegacySerializedSizeThreshold() + { + $rows = [ + ['sku' => 'sku-1', 'description' => str_repeat('a', 60)], + ['sku' => 'sku-2', 'description' => str_repeat('b', 60)], + ['sku' => 'sku-3', 'description' => str_repeat('c', 60)], + ['sku' => 'sku-4', 'description' => str_repeat('d', 60)], + ]; + $maxDataSize = 200; + $savedBunches = []; + $dataSourceModel = $this->getMockBuilder(ImportData::class) + ->disableOriginalConstructor() + ->onlyMethods(['cleanProcessedBunches', 'saveBunch']) + ->getMock(); + $dataSourceModel->expects($this->once()) + ->method('cleanProcessedBunches'); + $dataSourceModel->expects($this->exactly(2)) + ->method('saveBunch') + ->willReturnCallback( + function (string $entityTypeCode, string $behavior, array $bunchRows) use (&$savedBunches): int { + $savedBunches[] = $bunchRows; + return count($savedBunches); + } + ); + + $model = $this->createImportEntityModel($dataSourceModel, $maxDataSize); + $model->setSource($this->createImportSource($rows)); + + $saveValidatedBunches = new \ReflectionMethod($model, '_saveValidatedBunches'); + $saveValidatedBunches->invoke($model); + + $expectedBunches = $this->simulateLegacyBunches($rows, $maxDataSize); + + $this->assertCount(2, $savedBunches); + $this->assertSame($expectedBunches, $savedBunches); + } + + /** + * @covers \Magento\ImportExport\Model\Import\AbstractEntity::_saveValidatedBunches + */ + public function testSaveValidatedBunchesAccountsForNonSequentialKeysAfterFlush() + { + $rows = [ + ['sku' => 'sku-1', 'description' => str_repeat('a', 30)], + ['sku' => 'sku-2', 'description' => str_repeat('b', 30)], + ['sku' => 'sku-3', 'description' => str_repeat('c', 30)], + ['sku' => 'sku-4', 'description' => str_repeat('d', 30)], + ['sku' => 'sku-5', 'description' => str_repeat('e', 30)], + ['sku' => 'sku-6', 'description' => str_repeat('f', 30)], + ['sku' => 'sku-7', 'description' => str_repeat('g', 30)], + ['sku' => 'sku-8', 'description' => str_repeat('h', 30)], + ]; + $maxDataSize = 128; + $savedBunches = []; + $dataSourceModel = $this->getMockBuilder(ImportData::class) + ->disableOriginalConstructor() + ->onlyMethods(['cleanProcessedBunches', 'saveBunch']) + ->getMock(); + $dataSourceModel->expects($this->once()) + ->method('cleanProcessedBunches'); + $dataSourceModel->expects($this->exactly(4)) + ->method('saveBunch') + ->willReturnCallback( + function (string $entityTypeCode, string $behavior, array $bunchRows) use (&$savedBunches): int { + $savedBunches[] = $bunchRows; + return count($savedBunches); + } + ); + + $model = $this->createImportEntityModel($dataSourceModel, $maxDataSize); + $model->setSource($this->createImportSource($rows, 100)); + + $saveValidatedBunches = new \ReflectionMethod($model, '_saveValidatedBunches'); + $saveValidatedBunches->invoke($model); + + $expectedBunches = $this->simulateLegacyBunches($rows, $maxDataSize, 100); + + $this->assertCount(4, $savedBunches); + $this->assertSame($expectedBunches, $savedBunches); + } + /** * Create source adapter mock and set it into model object which tested in this class * @@ -591,4 +678,141 @@ protected function _createSourceAdapterMock(array $columns) return $source; } + + /** + * @param ImportData|MockObject $dataSourceModel + * @param int $maxDataSize + * @return AbstractEntity + */ + private function createImportEntityModel(ImportData|MockObject $dataSourceModel, int $maxDataSize): AbstractEntity + { + $scopeConfig = $this->createMock(ScopeConfigInterface::class); + $importFactory = $this->createMock(ImportFactory::class); + $resourceHelper = $this->createMock(Helper::class); + $resource = $this->createMock(ResourceConnection::class); + + return new class ( + new StringUtils(), + $scopeConfig, + $importFactory, + $resourceHelper, + $resource, + $this->getErrorAggregatorObject(), + [ + 'data_source_model' => $dataSourceModel, + 'connection' => 'not_used', + 'helpers' => [], + 'page_size' => 1, + 'max_data_size' => $maxDataSize, + 'bunch_size' => 0, + 'collection_by_pages_iterator' => 'not_used', + ], + new Json() + ) extends AbstractEntity { + protected $masterAttributeCode = 'sku'; + + protected function _importData() + { + return true; + } + + public function getEntityTypeCode() + { + return 'test_entity'; + } + + public function validateRow(array $rowData, $rowNumber) + { + return true; + } + }; + } + + /** + * @param array $rows + * @return AbstractSource + */ + private function createImportSource(array $rows, int $keyOffset = 0): AbstractSource + { + return new class ($rows, $keyOffset) extends AbstractSource { + /** + * @var array + */ + private array $rows; + + /** + * @var int + */ + private int $position = 0; + + /** + * @var int + */ + private int $keyOffset; + + /** + * @param array $rows + * @param int $keyOffset + */ + public function __construct(array $rows, int $keyOffset) + { + $this->rows = array_values($rows); + $this->keyOffset = $keyOffset; + parent::__construct(array_keys($this->rows[0])); + } + + /** + * @return array|false + */ + protected function _getNextRow() + { + if (!isset($this->rows[$this->position])) { + return false; + } + + return array_values($this->rows[$this->position++]); + } + + /** + * @return int + */ + #[\ReturnTypeWillChange] + public function key() + { + return parent::key() + $this->keyOffset; + } + }; + } + + /** + * @param array $rows + * @param int $maxDataSize + * @return array + */ + private function simulateLegacyBunches(array $rows, int $maxDataSize, int $keyOffset = 0): array + { + $serializer = new Json(); + $bunchRows = []; + $bunches = []; + $previousRow = null; + + foreach ($rows as $key => $row) { + if ($previousRow !== null) { + $bunchRows[$keyOffset + $key - 1] = $previousRow; + if (strlen($serializer->serialize($bunchRows)) >= $maxDataSize) { + $bunches[] = $bunchRows; + $bunchRows = []; + } + } + + $previousRow = $row; + } + + if ($previousRow !== null) { + $bunchRows[$keyOffset + count($rows) - 1] = $previousRow; + $bunches[] = $bunchRows; + } + + return $bunches; + } }