add reasoning summary, fix few bugs

hawk/src/main/java/io/irw/hawk/dto/ebay/EbayHighlightDto.java

@@ -22,6 +22,7 @@ public class EbayHighlightDto {
  EbayFindingDto ebayFinding;
  ProcessingPipelineMetadata pipelineMetadata;
  MerchandiseVerdictType finalVerdict;
+ MerchandiseVerdictType humanVerdict;
 
 
 }

hawk/src/main/java/io/irw/hawk/entity/EbayHighlight.java

@@ -52,11 +52,19 @@ public class EbayHighlight {
 
  @Type(JsonBinaryType.class)
  @Column(name= "pipeline_metadata", columnDefinition = "jsonb", nullable = false)
- private ProcessingPipelineMetadata pipelineMetadata;
+ ProcessingPipelineMetadata pipelineMetadata;
+
+ @Column(name= "reasoning_summary", nullable = false)
+ String reasoningSummary;
 
  @Enumerated(EnumType.STRING)
  @Column(name = "final_verdict", columnDefinition = "merchandise_verdict_type", nullable = false)
  @Type(PostgreSQLEnumType.class)
  MerchandiseVerdictType finalVerdict;
 
+ @Enumerated(EnumType.STRING)
+ @Column(name = "human_verdict", columnDefinition = "merchandise_verdict_type")
+ @Type(PostgreSQLEnumType.class)
+ MerchandiseVerdictType humanVerdict;
+
 }

hawk/src/main/java/io/irw/hawk/entity/type/CustomObjectMapperSupplier.java

@@ -7,6 +7,9 @@
 import com.fasterxml.jackson.databind.util.StdDateFormat;
 import io.hypersistence.utils.hibernate.type.util.ObjectMapperSupplier;
 
+/**
+ * @see <a href="https://vladmihalcea.com/hibernate-types-customize-jackson-objectmapper/"/a>
+ */
 public class CustomObjectMapperSupplier
  implements ObjectMapperSupplier {
 

hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java

@@ -4,6 +4,8 @@
 import io.irw.hawk.dto.ebay.EbayHighlightDto;
 import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
 import io.irw.hawk.entity.EbayHighlight;
+import io.irw.hawk.scraper.model.ProcessingPipelineMetadata;
+import java.util.stream.Collectors;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
 
@@ -19,8 +21,18 @@ public interface EbayHighlightMapper {
  EbayHighlightDto fromRuntime(HawkScrapeRunDto hawkScrapeRunDto,
  EbayFindingDto ebayFindingDto, EbayHighlightDto ebayHighlightDto);
 
+ @Mapping(target = "reasoningSummary", expression = "java(getReasoningSummary(ebayHighlightDto))")
  EbayHighlight toEntity(EbayHighlightDto ebayHighlightDto);
 
  EbayHighlightDto toDto(EbayHighlight ebayHighlight);
 
+ default String getReasoningSummary(EbayHighlightDto ebayHighlightDto) {
+ return ebayHighlightDto.getPipelineMetadata()
+ .filterReasoningsFromLog()
+ .stream()
+ .map(merchandiseReasoningLog ->
+ "%s %s".formatted(merchandiseReasoningLog.getVerdict(), merchandiseReasoningLog.getReason())
+ )
+ .collect(Collectors.joining("\n"));
+ }
 }

hawk/src/main/java/io/irw/hawk/scraper/service/extractors/BasicFieldExtractor.java

@@ -1,5 +1,8 @@
 package io.irw.hawk.scraper.service.extractors;
 
+import static io.irw.hawk.dto.ebay.EbayBuyingOptionEnum.AUCTION;
+import static io.irw.hawk.dto.ebay.EbayBuyingOptionEnum.BEST_OFFER;
+import static io.irw.hawk.dto.ebay.EbayBuyingOptionEnum.FIXED_PRICE;
 import static io.irw.hawk.dto.ebay.EbayListingStatusEnum.ACTIVE;
 
 import com.ebay.buy.browse.model.Image;
@@ -34,7 +37,7 @@ public void extractDataFromItemSummary(ItemSummary itemSummary, EbayHighlightDto
  ebayFindingDto.setListingStatus(ACTIVE);
  extractListingType(itemSummary, ebayFindingDto);
  extractAllImages(itemSummary, ebayFindingDto);
- if (ebayFindingDto.getBuyingOptions().contains(EbayBuyingOptionEnum.AUCTION)) {
+ if (ebayFindingDto.getBuyingOptions().contains(AUCTION)) {
  ebayFindingDto.setBidCount(Optional.of(itemSummary.getBidCount()));
  }
  }
@@ -44,22 +47,22 @@ private static void extractAllImages(ItemSummary itemSummary, EbayFindingDto eba
  .orElse(List.of());
  List<String> allImageUrls = Stream.concat(
  Stream.of(itemSummary.getImage()), additionalImages.stream())
- .map(image -> image.getImageUrl())
+ .map(Image::getImageUrl)
  .toList();
  ebayFindingDto.setImageUrls(allImageUrls);
  }
 
  private static void extractListingType(ItemSummary itemSummary, EbayFindingDto ebayFindingDto) {
- if (itemSummary.getBuyingOptions().contains("AUCTION")) {
- ebayFindingDto.getBuyingOptions().add(EbayBuyingOptionEnum.AUCTION);
+ if (itemSummary.getBuyingOptions().contains("AUCTION") && ! ebayFindingDto.getBuyingOptions().contains(AUCTION)) {
+ ebayFindingDto.getBuyingOptions().add(AUCTION);
  }
 
- if (itemSummary.getBuyingOptions().contains("BEST_OFFER")) {
- ebayFindingDto.getBuyingOptions().add(EbayBuyingOptionEnum.BEST_OFFER);
+ if (itemSummary.getBuyingOptions().contains("BEST_OFFER") && ! ebayFindingDto.getBuyingOptions().contains(BEST_OFFER)) {
+ ebayFindingDto.getBuyingOptions().add(BEST_OFFER);
  }
 
- if(itemSummary.getBuyingOptions().contains("FIXED_PRICE")) {
- ebayFindingDto.getBuyingOptions().add(EbayBuyingOptionEnum.FIXED_PRICE);
+ if(itemSummary.getBuyingOptions().contains("FIXED_PRICE") && ! ebayFindingDto.getBuyingOptions().contains(FIXED_PRICE)) {
+ ebayFindingDto.getBuyingOptions().add(FIXED_PRICE);
  }
  }
 

hawk/src/main/java/io/irw/hawk/scraper/service/processors/skates/parts/matchers/LabedaWheelsInterestMatcher.java

@@ -55,7 +55,7 @@ public void match(ItemSummary itemSummary, EbayHighlightDto highlightDto) {
  List<MerchandiseReasoningLog> reasonings = checkPricePerPiece(highlightDto,
  currentAucPricePerPieceWithShippingUsd.get());
  reasonings.forEach(reasoning -> highlightDto.getPipelineMetadata().addReasoning(reasoning));
- } else if(itemSummary.getBuyingOptions().contains(FIXED_PRICE)) {
+ } else if(ebayFindingDto.getBuyingOptions().contains(FIXED_PRICE)) {
  var buyNowPricePerPieceWithShippingUsd = ebayFindingDto.getBuyNowPricePerPieceWithShippingUsd();
  List<MerchandiseReasoningLog> reasonings = checkPricePerPiece(highlightDto, buyNowPricePerPieceWithShippingUsd.get());
  reasonings.forEach(reasoning -> highlightDto.getPipelineMetadata().addReasoning(reasoning));

hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScraperService.java

@@ -174,13 +174,13 @@ private ProductScrapeProcessor findMatchingScrapingProcessor(ProductVariantEnum
  }
 
  private static void updateMerchandiseVerdict(EbayHighlightDto highlightDto) {
- MerchandiseVerdictType verdict = highlightDto.getPipelineMetadata().filterReasoningsFromLog()
+ highlightDto.getPipelineMetadata()
+ .filterReasoningsFromLog()
  .stream()
  .min(Comparator.comparing(merchandiseReasoningDto -> merchandiseReasoningDto.getVerdict()
  .ordinal()))
  .map(MerchandiseReasoningLog::getVerdict)
- .orElse(MerchandiseVerdictType.BUYING_OPPORTUNITY);
- highlightDto.setFinalVerdict(verdict);
+ .ifPresent(highlightDto::setFinalVerdict);
  }
 
 }

hawk/src/main/resources/db/changelog/changes/0005-create_ebay_highlight_table.yaml

@@ -38,26 +38,16 @@ databaseChangeLog:
  type: jsonb
  constraints:
  nullable: false
+ - column:
+ name: reasoning_summary
+ type: VARCHAR(255) # Adjust the data type and length as needed
+ constraints:
+ nullable: false
  - column:
  name: final_verdict
  type: merchandise_verdict_type
  constraints:
  nullable: false
-
- # Adding foreign key constraints
- - addForeignKeyConstraint:
- baseColumnNames: run_id
- baseTableName: ebay_highlight
- baseTableSchemaName: merchandise_db
- constraintName: FK_EBAY_HIGHLIGHT_ON_SCRAPE_RUN
- referencedColumnNames: id
- referencedTableName: hawk_scrape_run
- referencedTableSchemaName: merchandise_db
- - addForeignKeyConstraint:
- baseColumnNames: ebay_finding_id
- baseTableName: ebay_highlight
- baseTableSchemaName: merchandise_db
- constraintName: FK_EBAY_HIGHLIGHT_ON_EBAY_FINDING
- referencedColumnNames: id
- referencedTableName: ebay_finding
- referencedTableSchemaName: merchandise_db
+ - column:
+ name: human_verdict
+ type: merchandise_verdict_type

hawk/src/test/java/io/irw/hawk/scraper/service/processors/skates/parts/extractors/WheelCountExtractorTest.java

@@ -11,6 +11,11 @@ class WheelCountExtractorTest {
  @Test
  public void testExtractNumberOfWheelsFromText() {
  // Test cases with known numbers of wheels
+ assertEquals(Optional.of(1), extractNumberOfWheelsFromText(
+ "Labeda Single Wheel RPG X-Soft RED Inline Indoor Hockey~72MM~76MM~80MM~~~~ 74A",
+ ""
+ ));
+
  assertEquals(Optional.of(4), extractNumberOfWheelsFromText(
  "Labeda Gripper Ref Roller Hockey Inline Wheels 80mm X-SOFT 4 Pack NEW IN PACKAGE",
  ""