draft of Seller and Finding persistence

hawk/src/main/java/io/irw/hawk/dto/ebay/EbayHighlightDto.java

@@ -0,0 +1,20 @@
+package io.irw.hawk.dto.ebay;
+
+import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
+import io.irw.hawk.scraper.model.MerchandiseVerdictType;
+import java.io.Serializable;
+import lombok.Builder;
+import lombok.Value;
+
+/**
+ * DTO for {@link io.irw.hawk.entity.EbayHighlight}
+ */
+@Value
+@Builder
+public class EbayHighlightDto implements Serializable {
+
+ Long id;
+ HawkScrapeRunDto run;
+ EbayFindingDto ebayFinding;
+ MerchandiseVerdictType finalVerdict;
+}

hawk/src/main/java/io/irw/hawk/dto/ebay/SearchTermDto.java

@@ -11,7 +11,6 @@
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class SearchTermDto {
 
- long id;
  ItemSummarySearchParameterDto searchParams;
 // String name;
 // String query;

hawk/src/main/java/io/irw/hawk/mapper/EbayFindingMapper.java

@@ -1,10 +1,16 @@
 package io.irw.hawk.mapper;
 
+import com.ebay.buy.browse.model.ItemSummary;
+import com.ebay.buy.browse.model.Seller;
 import io.irw.hawk.dto.ebay.EbayFindingDto;
+import io.irw.hawk.dto.ebay.EbaySellerDto;
 import io.irw.hawk.entity.EbayFinding;
+import io.irw.hawk.entity.EbaySeller;
 import org.mapstruct.BeanMapping;
 import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
 import org.mapstruct.MappingTarget;
+import org.mapstruct.Named;
 import org.mapstruct.NullValuePropertyMappingStrategy;
 
 @Mapper(config = ConfigMapper.class)
@@ -18,4 +24,15 @@ public interface EbayFindingMapper {
  EbayFinding partialUpdate(
  EbayFindingDto ebayFindingDto, @MappingTarget EbayFinding ebayFinding);
 
+
+ @BeanMapping(nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE)
+ @Mapping(target = "id", ignore = true)
+ @Mapping(target = "ebayIdStr", ignore = true)
+ EbayFinding updateFindingEntity(@MappingTarget EbayFinding persistentFindingEntity, EbayFindingDto latestFindingDto);
+
+// @Mapping(target = "id", ignore = true)
+// @Mapping(target = "ebayIdStr", source = "itemId")
+// @Mapping(target = "legacyEbayIdStr", source = "legacyItemId")
+// EbayFindingDto itemSummaryToEbayFindingDto(ItemSummary itemSummary);
+
 }

hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java

@@ -1,26 +1,37 @@
 package io.irw.hawk.mapper;
 
-import io.irw.hawk.entity.EbayFinding;
+import io.irw.hawk.dto.ebay.EbayFindingDto;
+import io.irw.hawk.dto.ebay.EbayHighlightDto;
+import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
 import io.irw.hawk.entity.EbayHighlight;
-import io.irw.hawk.entity.HawkScrapeRun;
 import io.irw.hawk.scraper.model.MerchandiseMetadataDto;
+import org.mapstruct.BeanMapping;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
+import org.mapstruct.MappingTarget;
 import org.mapstruct.Mappings;
+import org.mapstruct.NullValuePropertyMappingStrategy;
 
-@Mapper(config = ConfigMapper.class)
+@Mapper(config = ConfigMapper.class, uses = {HawkScrapeRunMapper.class, EbayFindingMapper.class})
 public interface EbayHighlightMapper {
 
-// @BeanMapping(nullValueCheckStrategy = NullValueCheckStrategy.ALWAYS,
+//  @BeanMapping(nullValueCheckStrategy = NullValueCheckStrategy.ALWAYS,
 // nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE)
  @Mappings({
  @Mapping(target = "id", ignore = true),
- @Mapping(target = "run", source = "hawkScrapeRun"),
- @Mapping(target = "ebayFinding", source = "ebayFinding"),
- @Mapping(target = "finalVerdict", source = "merchandiseMetadataDto.finalVerdict"),
+ @Mapping(target = "run", source = "hawkScrapeRunDto"),
+ @Mapping(target = "ebayFinding", source = "ebayFindingDto"),
+ @Mapping(target = "finalVerdict", source = "merchandiseMetadataDto.finalVerdict")
  })
- EbayHighlight toEntity(MerchandiseMetadataDto merchandiseMetadataDto, HawkScrapeRun hawkScrapeRun,
- EbayFinding ebayFinding);
+ EbayHighlightDto fromRuntime(HawkScrapeRunDto hawkScrapeRunDto,
+ EbayFindingDto ebayFindingDto, MerchandiseMetadataDto merchandiseMetadataDto);
 
 
+ EbayHighlight toEntity(EbayHighlightDto ebayHighlightDto);
+
+ EbayHighlightDto toDto(EbayHighlight ebayHighlight);
+
+ @BeanMapping(nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE)
+ EbayHighlight partialUpdate(
+ EbayHighlightDto ebayHighlightDto, @MappingTarget EbayHighlight ebayHighlight);
 }

hawk/src/main/java/io/irw/hawk/mapper/EbaySellerMapper.java

@@ -0,0 +1,44 @@
+package io.irw.hawk.mapper;
+
+import com.ebay.buy.browse.model.Seller;
+import io.irw.hawk.dto.ebay.EbaySellerDto;
+import io.irw.hawk.entity.EbaySeller;
+import org.mapstruct.BeanMapping;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.mapstruct.MappingTarget;
+import org.mapstruct.Named;
+import org.mapstruct.NullValuePropertyMappingStrategy;
+
+@Mapper(config = ConfigMapper.class)
+public interface EbaySellerMapper {
+
+ EbaySeller toEntity(EbaySellerDto ebaySellerDto);
+
+ EbaySellerDto toDto(EbaySeller ebaySeller);
+
+ @BeanMapping(nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE)
+ EbaySeller partialUpdate(
+ EbaySellerDto ebaySellerDto, @MappingTarget EbaySeller ebaySeller);
+
+ @Mapping(target = "reputationPercentage", source = "feedbackPercentage", qualifiedByName = "stringToFloat")
+ @Mapping(target = "feedbackScore", source = "feedbackScore")
+ @Mapping(target = "id", ignore = true)
+ @Mapping(target = "ebayIdStr", source = "username")
+ @Mapping(target = "registeredOn", ignore = true) // TODO: set from seller info
+ EbaySellerDto sellerToEbaySellerDto(Seller seller);
+
+ @Mapping(target = "id", ignore = true)
+ @Mapping(target = "ebayIdStr", ignore = true)
+ @Mapping(target = "registeredOn", ignore = true)
+ EbaySeller updateSellerEntity(@MappingTarget EbaySeller persistentSellerEntity, EbaySellerDto latestSellerDto);
+
+ @Named("stringToFloat")
+ default float stringToFloat(String value) {
+ try {
+ return Float.parseFloat(value);
+ } catch (NumberFormatException e) {
+ return 0.0f; // or some default value
+ }
+ }
+}

hawk/src/main/java/io/irw/hawk/repository/EbayFindingRepository.java

@@ -2,8 +2,11 @@
 
 import io.irw.hawk.entity.EbayFinding;
 import io.irw.hawk.entity.HawkScrapeRun;
+import java.util.Optional;
 import org.springframework.data.jpa.repository.JpaRepository;
 
 public interface EbayFindingRepository extends JpaRepository<EbayFinding, Long> {
 
+ Optional<EbayFinding> findByEbayIdStr(String ebayIdStr);
+
 }

hawk/src/main/java/io/irw/hawk/repository/EbaySellerRepository.java

@@ -1,8 +1,11 @@
 package io.irw.hawk.repository;
 
 import io.irw.hawk.entity.EbaySeller;
+import java.util.Optional;
 import org.springframework.data.jpa.repository.JpaRepository;
 
 public interface EbaySellerRepository extends JpaRepository<EbaySeller, Long> {
 
+ Optional<EbaySeller> findByEbayIdStr(String ebayIdStr);
+
 }

hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseVerdictType.java

@@ -17,6 +17,11 @@ public enum MerchandiseVerdictType {
  */
  UNPROCESSABLE,
 
+ /**
+ * Item is already present at the DB
+ */
+ ITEM_ALREADY_PERSISTED,
+
  /**
  * Hard reject - wrong match, or issues like reputation
  */

hawk/src/main/java/io/irw/hawk/scraper/model/ProcessingPipelineStep.java

@@ -32,7 +32,7 @@ default int compareTo(@NotNull ProcessingPipelineStep other) {
  .getName());
  }
 
- static List<ProcessingPipelineStep> sortOutDependencies(ProcessingPipelineStep... steps) {
+ static List<ProcessingPipelineStep> linearExecutionGraphByDependencies(ProcessingPipelineStep... steps) {
  DirectedPseudograph<ProcessingPipelineStep, DefaultEdge> dependencyGraph = new DirectedPseudograph<>(
  DefaultEdge.class);
  Map<Class<? extends ProcessingPipelineStep>, ProcessingPipelineStep> classToStep = Arrays.stream(steps)

hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayFindingService.java

@@ -1,12 +1,17 @@
 package io.irw.hawk.scraper.service.domain;
 
+import io.irw.hawk.dto.ebay.EbayFindingDto;
+import io.irw.hawk.dto.ebay.EbaySellerDto;
 import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
 import io.irw.hawk.dto.merchandise.ProductVariantEnum;
+import io.irw.hawk.entity.EbayFinding;
+import io.irw.hawk.entity.EbaySeller;
 import io.irw.hawk.mapper.EbayFindingMapper;
 import io.irw.hawk.mapper.HawkScrapeRunMapper;
 import io.irw.hawk.repository.EbayFindingRepository;
 import io.irw.hawk.repository.HawkScrapeRunRepository;
 import java.time.Instant;
+import java.util.Optional;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -22,19 +27,28 @@ public class EbayFindingService {
 
  EbayFindingMapper ebayFindingMapper;
  EbayFindingRepository ebayFindingRepository;
-// HawkScrapeRunMapper hawkScrapeRunMapper;
-// HawkFlightService hawkFlightService;
-//
-// @Transactional
-// public HawkScrapeRunDto startScrapeRun(ProductVariantEnum targetProductVariant) {
-// HawkScrapeRunDto hawkScrapeRunDto = HawkScrapeRunDto.builder()
-// .hawkFlight(hawkFlightService.getCurrentFlight())
-// .startedAt(Instant.now())
-// .productVariant(targetProductVariant)
-// .build();
-// hawkFlightRepository.save(hawkScrapeRunMapper.toEntity(hawkScrapeRunDto));
-// return hawkScrapeRunDto;
-// }
+
+ @Transactional
+ public EbayFindingDto saveFinding(EbayFindingDto ebayFindingDto) {
+ EbayFinding persisted = ebayFindingRepository.save(ebayFindingMapper.toEntity(ebayFindingDto));
+ return ebayFindingMapper.toDto(persisted);
+ }
+
+ public Optional<EbayFindingDto> findByEbayId(String ebayItemIdStr) {
+ return ebayFindingRepository.findByEbayIdStr(ebayItemIdStr)
+ .map(ebayFindingMapper::toDto);
+ }
+
+ @Transactional
+ public EbayFindingDto upsertFinding(EbayFindingDto ebayFindingDto) {
+ return ebayFindingRepository.findByEbayIdStr(ebayFindingDto.getEbayIdStr())
+ .map(existingFinding -> {
+ EbayFinding updatedFinding = ebayFindingRepository.save(
+ ebayFindingMapper.updateFindingEntity(existingFinding, ebayFindingDto));
+ return ebayFindingMapper.toDto(updatedFinding);
+ })
+ .orElseGet(() -> saveFinding(ebayFindingDto));
+ }
 
 
 }

hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayHighlightService.java

@@ -1,12 +1,15 @@
 package io.irw.hawk.scraper.service.domain;
 
+import io.irw.hawk.dto.ebay.EbayHighlightDto;
+import io.irw.hawk.entity.EbayHighlight;
 import io.irw.hawk.mapper.EbayHighlightMapper;
 import io.irw.hawk.repository.EbayHighlightRepository;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 @RequiredArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
@@ -17,20 +20,12 @@ public class EbayHighlightService {
  EbayHighlightMapper ebayHighlightMapper;
  EbayHighlightRepository ebayHighlightRepository;
 
-// HawkScrapeRunRepository hawkFlightRepository;
-// HawkScrapeRunMapper hawkScrapeRunMapper;
-// HawkFlightService hawkFlightService;
-//
-// @Transactional
-// public HawkScrapeRunDto startScrapeRun(ProductVariantEnum targetProductVariant) {
-// HawkScrapeRunDto hawkScrapeRunDto = HawkScrapeRunDto.builder()
-// .hawkFlight(hawkFlightService.getCurrentFlight())
-// .startedAt(Instant.now())
-// .productVariant(targetProductVariant)
-// .build();
-// hawkFlightRepository.save(hawkScrapeRunMapper.toEntity(hawkScrapeRunDto));
-// return hawkScrapeRunDto;
-// }
+ @Transactional
+ public EbayHighlightDto saveHighlight(EbayHighlightDto ebayHighlightDto) {
+ EbayHighlight persisted = ebayHighlightRepository.save(ebayHighlightMapper.toEntity(ebayHighlightDto));
+ return ebayHighlightMapper.toDto(persisted);
+ }
+
 
 
 }

hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbaySellerService.java

@@ -0,0 +1,40 @@
+package io.irw.hawk.scraper.service.domain;
+
+import io.irw.hawk.dto.ebay.EbaySellerDto;
+import io.irw.hawk.entity.EbaySeller;
+import io.irw.hawk.mapper.EbaySellerMapper;
+import io.irw.hawk.repository.EbaySellerRepository;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+@RequiredArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+@Service
+@Slf4j
+public class EbaySellerService {
+
+ EbaySellerMapper ebaySellerMapper;
+ EbaySellerRepository ebaySellerRepository;
+
+ @Transactional
+ public EbaySellerDto saveSeller(EbaySellerDto ebaySellerDto) {
+ EbaySeller persisted = ebaySellerRepository.save(ebaySellerMapper.toEntity(ebaySellerDto));
+ return ebaySellerMapper.toDto(persisted);
+ }
+
+ @Transactional
+ public EbaySellerDto upsertSeller(EbaySellerDto ebaySellerDto) {
+ return ebaySellerRepository.findByEbayIdStr(ebaySellerDto.getEbayIdStr())
+ .map(existingSeller -> {
+ EbaySeller updatedSeller = ebaySellerRepository.save(
+ ebaySellerMapper.updateSellerEntity(existingSeller,ebaySellerDto));
+ return ebaySellerMapper.toDto(updatedSeller);
+ })
+ .orElseGet(() -> saveSeller(ebaySellerDto));
+ }
+
+}

hawk/src/main/java/io/irw/hawk/scraper/service/matchers/ItemAlreadyPersistedDataMatcher.java

@@ -0,0 +1,36 @@
+package io.irw.hawk.scraper.service.matchers;
+
+import io.irw.hawk.dto.merchandise.ProductVariantEnum;
+import io.irw.hawk.scraper.model.ProcessingPipelineStep;
+import io.irw.hawk.scraper.service.extractors.PieceCountExtractor;
+import io.irw.hawk.scraper.service.extractors.PriceExtractor;
+import io.irw.hawk.scraper.service.extractors.ShippingCostExtractor;
+import java.util.List;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+/**
+ * Virtual matcher that checks that basic fields are populated. This matcher is intended to be used as a common
+ * dependency for more advanced matchers (instead of listing all other dependencies separately)
+ */
+@RequiredArgsConstructor
+@Service
+@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+@Slf4j
+public class ItemAlreadyPersistedDataMatcher implements ItemSummaryMatcher {
+
+
+
+ @Override
+ public List<Class<? extends ProcessingPipelineStep>> dependsOn() {
+ return List.of(BaselineItemDataMatcher.class);
+ }
+
+ @Override
+ public boolean isApplicableTo(ProductVariantEnum productVariant) {
+ return true;
+ }
+}

hawk/src/main/java/io/irw/hawk/scraper/service/processors/ProductScrapeProcessor.java

@@ -1,12 +1,13 @@
 package io.irw.hawk.scraper.service.processors;
 
-import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
+import io.irw.hawk.dto.ebay.SearchTermDto;
 import io.irw.hawk.dto.merchandise.ProductVariantEnum;
+import java.util.List;
 
 public interface ProductScrapeProcessor {
 
  boolean supports(ProductVariantEnum productVariant);
 
- void process(HawkScrapeRunDto hawkScrapeRunDto);
+ List<SearchTermDto> generateSearchTerms(ProductVariantEnum productVariant);
 
 }