Merge branch 'feature/prefetch-data' into release/1.17.0

Helper/Product.php

@@ -23,6 +23,7 @@
 use Magento\Bundle\Model\ResourceModel\Selection as BundleResource;
 use Magmodules\Channable\Api\Log\RepositoryInterface as LogRepository;
 use Magmodules\Channable\Service\Product\InventoryData;
+use Magmodules\Channable\Service\Product\MediaData;
 
 /**
  * Class Product
@@ -84,6 +85,12 @@ class Product extends AbstractHelper
  * @var CatalogPrice
  */
  private $commonPriceModel;
+
+ /**
+ * @var MediaData
+ */
+ private $mediaData;
+
  /**
  * @var LogRepository
  */
@@ -128,6 +135,7 @@ public function __construct(
  ConfigurableResource $catalogProductTypeConfigurable,
  CatalogPrice $commonPriceModel,
  InventoryData $inventoryData,
+ MediaData $mediaData,
  LogRepository $logger
  ) {
  $this->galleryReadHandler = $galleryReadHandler;
@@ -143,6 +151,7 @@ public function __construct(
  $this->catalogProductTypeBundle = $catalogProductTypeBundle;
  $this->commonPriceModel = $commonPriceModel;
  $this->inventoryData = $inventoryData;
+ $this->mediaData = $mediaData;
  $this->logger = $logger;
  parent::__construct($context);
  }
@@ -524,8 +533,13 @@ public function getImageData($attribute, $config, $product)
  }
  }
  }
- $this->galleryReadHandler->execute($product);
+
  $galleryImages = $product->getMediaGallery('images');
+ if (!$galleryImages) {
+ $this->galleryReadHandler->execute($product);
+ $galleryImages = $product->getMediaGallery('images');
+ }
+
  foreach ($galleryImages as $image) {
  if (empty($image['disabled']) || !empty($config['inc_hidden_image'])) {
  $images[] = $this->catalogProductMediaConfig->getMediaUrl($image['file']);
@@ -613,9 +627,15 @@ public function getResizedImage($product, $source, $size)
  */
  public function getAttributeSetName($product)
  {
+ static $attributeSets = [];
+
  try {
- $attributeSetRepository = $this->attributeSet->get($product->getAttributeSetId());
- return $attributeSetRepository->getAttributeSetName();
+ if (!isset($attributeSets[$product->getAttributeSetId()])) {
+ $attributeSetName = $this->attributeSet->get($product->getAttributeSetId())->getAttributeSetName();
+ $attributeSets[$product->getAttributeSetId()] = $attributeSetName;
+ }
+
+ return $attributeSets[$product->getAttributeSetId()];
  } catch (\Exception $e) {
  $this->logger->addErrorLog('getAttributeSetName', $e->getMessage());
  }
@@ -1318,4 +1338,20 @@ public function getParentId($product, $filters)
 
  return array_unique($parentIds);
  }
+
+ /**
+ * @return InventoryData
+ */
+ public function getInventoryData()
+ {
+ return $this->inventoryData;
+ }
+
+ /**
+ * @return MediaData
+ */
+ public function getMediaData()
+ {
+ return $this->mediaData;
+ }
 }

Model/Collection/Products.php

@@ -7,6 +7,7 @@
 namespace Magmodules\Channable\Model\Collection;
 
 use Magento\Catalog\Model\ResourceModel\Product\CollectionFactory as ProductCollectionFactory;
+use Magento\Catalog\Model\ResourceModel\Product\Collection as ProductCollection;
 use Magento\Catalog\Model\ResourceModel\Product\Attribute\CollectionFactory as ProductAttributeCollectionFactory;
 use Magento\Eav\Model\Config as EavConfig;
 use Magento\Catalog\Model\Indexer\Product\Flat\StateFactory;
@@ -27,6 +28,10 @@ class Products
  * @var ProductCollectionFactory
  */
  private $productCollectionFactory;
+ /**
+ * @var ProductCollection
+ */
+ private $productCollection;
  /**
  * @var ProductAttributeCollectionFactory
  */
@@ -70,6 +75,7 @@ class Products
  */
  public function __construct(
  ProductCollectionFactory $productCollectionFactory,
+ ProductCollection $productCollection,
  ProductAttributeCollectionFactory $productAttributeCollectionFactory,
  EavConfig $eavConfig,
  StockHelper $stockHelper,
@@ -79,6 +85,7 @@ public function __construct(
  ResourceConnection $resource
  ) {
  $this->productCollectionFactory = $productCollectionFactory;
+ $this->productCollection = $productCollection;
  $this->productAttributeCollectionFactory = $productAttributeCollectionFactory;
  $this->eavConfig = $eavConfig;
  $this->productFlatState = $productFlatState;
@@ -377,49 +384,52 @@ public function joinPriceIndexLeft($collection, $websiteId)
  * @param $parentRelations
  * @param $config
  *
- * @return \Magento\Catalog\Model\ResourceModel\Product\Collection
+ * @return ProductCollection
  * @throws \Magento\Framework\Exception\LocalizedException
  */
- public function getParents($parentRelations, $config)
+ public function getParents($parentRelations, $config): ProductCollection
  {
- if (!empty($parentRelations)) {
- $filters = $config['filters'];
+ if (empty($parentRelations)) {
+ return $this->productCollection; //return empty product collection
+ }
 
- if (!$config['flat']) {
- $productFlatState = $this->productFlatState->create(['isAvailable' => false]);
- } else {
- $productFlatState = $this->productFlatState->create(['isAvailable' => true]);
- }
+ $filters = $config['filters'];
 
- $entityField = $this->generalHelper->getLinkField();
- $attributes = $this->getAttributes($config['attributes']);
+ if (!$config['flat']) {
+ $productFlatState = $this->productFlatState->create(['isAvailable' => false]);
+ } else {
+ $productFlatState = $this->productFlatState->create(['isAvailable' => true]);
+ }
 
- $collection = $this->productCollectionFactory
- ->create(['catalogProductFlatState' => $productFlatState])
- ->addStoreFilter($config['store_id'])
- ->addAttributeToFilter($entityField, ['in' => array_values($parentRelations)])
- ->addAttributeToSelect($attributes)
- ->addUrlRewrite()
- ->setRowIdFieldName($entityField);
+ $entityField = $this->generalHelper->getLinkField();
+ $attributes = $this->getAttributes($config['attributes']);
 
- if (!empty($filters['category_ids'])) {
- if (!empty($filters['category_type'])) {
- $collection->addCategoriesFilter([$filters['category_type'] => $filters['category_ids']]);
- }
- }
+ $collection = $this->productCollectionFactory
+ ->create(['catalogProductFlatState' => $productFlatState])
+ ->addStoreFilter($config['store_id'])
+ ->addAttributeToFilter($entityField, ['in' => array_values($parentRelations)])
+ ->addAttributeToSelect($attributes)
+ ->addUrlRewrite()
+ ->setRowIdFieldName($entityField);
 
- if (!empty($filters['visibility'])) {
- $collection->addAttributeToFilter('visibility', ['in' => $filters['visibility']]);
+ if (!empty($filters['category_ids'])) {
+ if (!empty($filters['category_type'])) {
+ $collection->addCategoriesFilter([$filters['category_type'] => $filters['category_ids']]);
  }
+ }
 
-  if (!empty($config['inventory']['attributes'])) {
-  $this->joinCatalogInventoryLeft($collection, $config);
-  }
+ if (!empty($filters['visibility'])) {
+ $collection->addAttributeToFilter('visibility', ['in' => $filters['visibility']]);
+ }
 
- $this->addFilters($filters, $collection, 'parent');
- $this->joinPriceIndexLeft($collection, $config['website_id']);
- return $collection->load();
+ if (!empty($config['inventory']['attributes'])) {
+ $this->joinCatalogInventoryLeft($collection, $config);
  }
+
+ $this->addFilters($filters, $collection, 'parent');
+ $this->joinPriceIndexLeft($collection, $config['website_id']);
+ return $collection->load();
+
  }
 
  /**

Model/Generate.php

@@ -7,6 +7,7 @@
 namespace Magmodules\Channable\Model;
 
 use Magento\Catalog\Model\Product;
+use Magento\Catalog\Model\ResourceModel\Product\Collection as ProductCollection;
 use Magento\Framework\Exception\LocalizedException;
 use Magmodules\Channable\Model\Collection\Products as ProductsModel;
 use Magmodules\Channable\Model\Item as ItemModel;
@@ -120,6 +121,8 @@ public function generateByStore(
  $parentRelations = $this->productHelper->getParentsFromCollection($products, $config);
  $parents = $this->productModel->getParents($parentRelations, $config);
 
+ $this->prefetchData($products, $parents, $parentRelations, $config);
+
  foreach ($products as $product) {
  /** @var Product $product */
  $parent = null;
@@ -208,4 +211,24 @@ public function getDataRow($product, $parent, $config): ?array
 
  return null;
  }
+
+ /**
+ * Prefetches data to reduce amount of queries required.
+ * This increases performance by a lot for environments with >1ms latency to database.
+ *
+ * @param ProductCollection $products
+ * @param ProductCollection $parents
+ * @param array $parentRelations
+ * @param array $config
+ * @return void
+ */
+ private function prefetchData(
+ ProductCollection $products,
+ ProductCollection $parents,
+ array $parentRelations,
+ array $config
+ ) {
+ $this->productHelper->getInventoryData()->load($products->getColumnValues('sku'), $config);
+ $this->productHelper->getMediaData()->load($products, $parents);
+ }
 }

Service/Product/InventoryData.php

@@ -19,6 +19,16 @@ class InventoryData
  */
  private $resourceConnection;
 
+ /**
+ * @var array
+ */
+ private $inventory;
+
+ /**
+ * @var array
+ */
+ private $reservation;
+
  /**
  * InventoryData constructor.
  *
@@ -30,77 +40,75 @@ public function __construct(
  $this->resourceConnection = $resourceConnection;
  }
 
- /**
- * Get Salable QTY for a product by StockID
- *
- * @param ProductInterface $product
- * @param int $stockId
- *
- * @return float|int|mixed
- */
- public function getSalableQty(ProductInterface $product, int $stockId): float
- {
- $inventoryData = $this->getInventoryData($product->getSku(), $stockId);
- $reservations = $this->getReservations($product->getSku(), $stockId);
-
- $qty = isset($inventoryData['quantity'])
- ? $inventoryData['quantity'] - $reservations
- : 0;
-
- return !empty($inventoryData['is_salable']) ? $qty : 0;
- }
-
  /**
  * Get Inventory Data by SKU and StockID
  *
- * @param string $sku
+ * @param array $skus
  * @param int $stockId
  *
- * @return mixed
+ * @return void
  */
- private function getInventoryData(string $sku, int $stockId)
+ private function getInventoryData(array $skus, int $stockId): void
  {
  $connection = $this->resourceConnection->getConnection();
  $tableName = $this->resourceConnection->getTableName('inventory_stock_' . $stockId);
 
  if (!$connection->isTableExists($tableName)) {
- return [];
+ return;
  }
 
  $select = $connection->select()
  ->from($tableName)
- ->where('sku = ?', $sku)
- ->limit(1);
+ ->where('sku IN (?)', $skus);
 
- return $connection->fetchRow($select);
+ $inventoryData = $connection->fetchAll($select);
+ foreach ($inventoryData as $data) {
+ $this->inventory[$stockId][$data['sku']] = $data;
+ }
  }
 
  /**
  * Returns number of reservations by SKU & StockId
  *
- * @param string $sku
+ * @param array $skus
  * @param int $stockId
  *
- * @return float
+ * @return void
  */
- private function getReservations(string $sku, int $stockId): float
+ private function getReservations(array $skus, int $stockId): void
  {
  $connection = $this->resourceConnection->getConnection();
  $tableName = $this->resourceConnection->getTableName('inventory_reservation');
 
  if (!$connection->isTableExists($tableName)) {
- return 0;
+ return;
  }
 
  $select = $connection->select()
- ->from($tableName, ['quantity' => 'SUM(quantity)'])
- ->where('sku = ?', $sku)
+ ->from($tableName, ['sku', 'quantity' => 'SUM(quantity)'])
+ ->where('sku IN (?)', $skus)
  ->where('stock_id' . ' = ?', $stockId)
- ->limit(1);
+ ->group('sku');
+
+ $reservations = $connection->fetchAll($select);
+ foreach ($reservations as $reservation) {
+ $this->reservation[$stockId][$reservation['sku']] = $reservation['quantity'];
+ }
+ }
 
- return ($reservationQty = $connection->fetchOne($select))
- ? max(0, ($reservationQty * -1))
- : 0;
+ /**
+ * Loads all stock information into memory
+ *
+ * @param array $skus
+ * @param array $config
+ * @return void
+ */
+ public function load(array $skus, array $config): void
+ {
+ if (isset($config['inventory']['stock_id'])) {
+ $this->getInventoryData($skus, (int)$config['inventory']['stock_id']);
+ $this->getReservations($skus, (int)$config['inventory']['stock_id']);
+ }
  }
 
  /**
@@ -119,8 +127,8 @@ public function addDataToProduct(Product $product, array $config): Product
  return $product;
  }
 
- $inventoryData = $this->getInventoryData($product->getSku(), $config['inventory']['stock_id']);
- $reservations = $this->getReservations($product->getSku(), $config['inventory']['stock_id']);
+ $inventoryData = $this->inventory[$config['inventory']['stock_id']][$product->getSku()] ?? [];
+ $reservations = $this->reservation[$config['inventory']['stock_id']][$product->getSku()] ?? 0;
 
  $qty = isset($inventoryData['quantity']) ? $inventoryData['quantity'] - $reservations : 0;
  $isSalable = $inventoryData['is_salable'] ?? 0;