persist highlights, refactor metadata

hawk/src/main/java/io/irw/hawk/configuration/PersistenceConfiguration.java

@@ -0,0 +1,5 @@
+package io.irw.hawk.configuration;
+
+public class PersistenceConfiguration {
+
+}

hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingTypeEnum.java → hawk/src/main/java/io/irw/hawk/dto/ebay/EbayBuyingOptionEnum.java

@@ -1,6 +1,6 @@
 package io.irw.hawk.dto.ebay;
 
-public enum EbayListingTypeEnum {
+public enum EbayBuyingOptionEnum {
 
  AUCTION, BEST_OFFER, FIXED_PRICE
 

hawk/src/main/java/io/irw/hawk/dto/ebay/EbayFindingDto.java

@@ -40,7 +40,7 @@ public class EbayFindingDto {
  Instant listingEndsAt; // Not implemented yet
 
  @Default
- List<EbayListingTypeEnum> listingTypes = new ArrayList<>();
+ List<EbayBuyingOptionEnum> buyingOptions = new ArrayList<>();
  EbayListingStatusEnum listingStatus;
  EbaySellerDto seller;
 

hawk/src/main/java/io/irw/hawk/dto/ebay/EbayHighlightDto.java

@@ -1,20 +1,27 @@
 package io.irw.hawk.dto.ebay;
 
 import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
-import io.irw.hawk.scraper.model.MerchandiseVerdictType;
-import java.io.Serializable;
+import io.irw.hawk.dto.merchandise.MerchandiseVerdictType;
+import io.irw.hawk.scraper.model.ProcessingPipelineMetadata;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
 import lombok.Builder;
-import lombok.Value;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
 
-/**
- * DTO for {@link io.irw.hawk.entity.EbayHighlight}
- */
-@Value
+@NoArgsConstructor
+@AllArgsConstructor
+@Data
 @Builder
-public class EbayHighlightDto implements Serializable {
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class EbayHighlightDto {
 
  Long id;
  HawkScrapeRunDto run;
  EbayFindingDto ebayFinding;
+ ProcessingPipelineMetadata pipelineMetadata;
  MerchandiseVerdictType finalVerdict;
-}
+
+
+}

hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseVerdictType.java → hawk/src/main/java/io/irw/hawk/dto/merchandise/MerchandiseVerdictType.java

@@ -1,4 +1,4 @@
-package io.irw.hawk.scraper.model;
+package io.irw.hawk.dto.merchandise;
 
 import java.util.Arrays;
 import java.util.List;

hawk/src/main/java/io/irw/hawk/entity/EbayFinding.java

@@ -5,7 +5,7 @@
 import io.hypersistence.utils.hibernate.type.array.internal.AbstractArrayType;
 import io.hypersistence.utils.hibernate.type.basic.PostgreSQLEnumType;
 import io.irw.hawk.dto.ebay.EbayListingStatusEnum;
-import io.irw.hawk.dto.ebay.EbayListingTypeEnum;
+import io.irw.hawk.dto.ebay.EbayBuyingOptionEnum;
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -69,13 +69,13 @@ public class EbayFinding {
  private Instant listingEndsAt;
 
  @Enumerated(EnumType.STRING)
- @Column(name = "listing_types", nullable = false, columnDefinition = "ebay_listing_type[]")
+ @Column(name = "buying_options", nullable = false, columnDefinition = "ebay_buying_option[]")
  @Type(value=EnumArrayType.class,
  parameters = @Parameter(
  name = AbstractArrayType.SQL_ARRAY_TYPE,
- value = "ebay_listing_type"
+ value = "ebay_buying_option"
  ))
- private EbayListingTypeEnum[] listingTypes;
+ private EbayBuyingOptionEnum[] buyingOptions;
 
  @Enumerated(EnumType.STRING)
  @Column(name = "listing_status", nullable = false, columnDefinition = "ebay_listing_status")

hawk/src/main/java/io/irw/hawk/entity/EbayHighlight.java

@@ -1,6 +1,9 @@
 package io.irw.hawk.entity;
 
-import io.irw.hawk.scraper.model.MerchandiseVerdictType;
+import io.hypersistence.utils.hibernate.type.basic.PostgreSQLEnumType;
+import io.hypersistence.utils.hibernate.type.json.JsonBinaryType;
+import io.irw.hawk.dto.merchandise.MerchandiseVerdictType;
+import io.irw.hawk.scraper.model.ProcessingPipelineMetadata;
 import jakarta.persistence.Column;
 import jakarta.persistence.Entity;
 import jakarta.persistence.EnumType;
@@ -15,6 +18,7 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.Setter;
+import org.hibernate.annotations.Type;
 
 /**
  * Items that were found and not filtered out during the scrape run. There was some reasoning and resolution about
@@ -46,8 +50,13 @@ public class EbayHighlight {
  @EqualsAndHashCode.Include
  EbayFinding ebayFinding;
 
+ @Type(JsonBinaryType.class)
+ @Column(name= "pipeline_metadata", columnDefinition = "jsonb", nullable = false)
+ private ProcessingPipelineMetadata pipelineMetadata;
+
  @Enumerated(EnumType.STRING)
- @Column(name = "final_verdict", columnDefinition = "merchandise_verdict_type")
+ @Column(name = "final_verdict", columnDefinition = "merchandise_verdict_type", nullable = false)
+ @Type(PostgreSQLEnumType.class)
  MerchandiseVerdictType finalVerdict;
 
 }

hawk/src/main/java/io/irw/hawk/entity/type/CustomObjectMapperSupplier.java

@@ -0,0 +1,31 @@
+package io.irw.hawk.entity.type;
+
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.util.StdDateFormat;
+import io.hypersistence.utils.hibernate.type.util.ObjectMapperSupplier;
+
+public class CustomObjectMapperSupplier
+ implements ObjectMapperSupplier {
+
+ @Override
+ public ObjectMapper get() {
+ ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
+
+ // Configure it to use ISO 8601 date format for serialization
+ objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+ objectMapper.setDateFormat(new StdDateFormat());
+
+ SimpleModule customBigDecimalMoneyModule = new SimpleModule(
+ "CustomBigDecimalMoneyModule",
+ new Version(1, 0, 0, null, null, null)
+ );
+ customBigDecimalMoneyModule.addSerializer(new MoneySerializer());
+
+ objectMapper.registerModule(customBigDecimalMoneyModule);
+
+ return objectMapper;
+ }
+}

hawk/src/main/java/io/irw/hawk/entity/type/MoneySerializer.java

@@ -0,0 +1,27 @@
+package io.irw.hawk.entity.type;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import java.io.IOException;
+import java.math.BigDecimal;
+
+public class MoneySerializer
+ extends JsonSerializer<BigDecimal> {
+
+ @Override
+ public void serialize(
+ BigDecimal value,
+ JsonGenerator jsonGenerator,
+ SerializerProvider provider)
+ throws IOException {
+ jsonGenerator.writeString(
+ value.setScale(2, BigDecimal.ROUND_HALF_UP).toString()
+ );
+ }
+
+ @Override
+ public Class<BigDecimal> handledType() {
+ return BigDecimal.class;
+ }
+}

hawk/src/main/java/io/irw/hawk/mapper/EbayFindingMapper.java

@@ -18,7 +18,6 @@ public interface EbayFindingMapper {
 
  @BeanMapping(nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE)
  @Mapping(target = "id", ignore = true)
- @Mapping(target = "ebayIdStr", ignore = true)
  EbayFinding updateFindingEntity(@MappingTarget EbayFinding persistentFindingEntity, EbayFindingDto latestFindingDto);
 
  @BeanMapping(nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE, unmappedTargetPolicy = ReportingPolicy.IGNORE)

hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java

@@ -4,28 +4,20 @@
 import io.irw.hawk.dto.ebay.EbayHighlightDto;
 import io.irw.hawk.dto.merchandise.HawkScrapeRunDto;
 import io.irw.hawk.entity.EbayHighlight;
-import io.irw.hawk.scraper.model.MerchandiseMetadataDto;
-import org.mapstruct.BeanMapping;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
-import org.mapstruct.MappingTarget;
-import org.mapstruct.Mappings;
-import org.mapstruct.NullValuePropertyMappingStrategy;
 
 @Mapper(config = ConfigMapper.class, uses = {HawkScrapeRunMapper.class, EbayFindingMapper.class})
 public interface EbayHighlightMapper {
 
 // @BeanMapping(nullValueCheckStrategy = NullValueCheckStrategy.ALWAYS,
 // nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE)
- @Mappings({
- @Mapping(target = "id", ignore = true),
- @Mapping(target = "run", source = "hawkScrapeRunDto"),
- @Mapping(target = "ebayFinding", source = "ebayFindingDto"),
- @Mapping(target = "finalVerdict", source = "merchandiseMetadataDto.finalVerdict")
- })
+ @Mapping(target = "id", ignore = true)
+ @Mapping(target = "run", source = "hawkScrapeRunDto")
+ @Mapping(target = "ebayFinding", source = "ebayFindingDto")
+ @Mapping(target = "finalVerdict", source = "ebayHighlightDto.finalVerdict")
  EbayHighlightDto fromRuntime(HawkScrapeRunDto hawkScrapeRunDto,
- EbayFindingDto ebayFindingDto, MerchandiseMetadataDto merchandiseMetadataDto);
-
+ EbayFindingDto ebayFindingDto, EbayHighlightDto ebayHighlightDto);
 
  EbayHighlight toEntity(EbayHighlightDto ebayHighlightDto);
 

hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseReasoningLog.java

@@ -0,0 +1,24 @@
+package io.irw.hawk.scraper.model;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.irw.hawk.dto.merchandise.MerchandiseVerdictType;
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+@SuperBuilder
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@JsonTypeName("merchandiseReasoning")
+public class MerchandiseReasoningLog extends PipelineStepLog {
+
+ String reasoningMatcher;
+ MerchandiseVerdictType verdict;
+ String reason;
+
+}

hawk/src/main/java/io/irw/hawk/scraper/model/PipelineStepLog.java

@@ -0,0 +1,25 @@
+package io.irw.hawk.scraper.model;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import java.time.Instant;
+import lombok.Builder.Default;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+@SuperBuilder
+@Data
+@NoArgsConstructor
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
+@JsonSubTypes({
+ @JsonSubTypes.Type(value = MerchandiseReasoningLog.class, name = "merchandiseReasoning"),
+ // other subclasses
+})
+public class PipelineStepLog {
+
+ @Default
+ Instant timestamp = Instant.now();
+ String message;
+
+}

hawk/src/main/java/io/irw/hawk/scraper/model/ProcessingPipelineMetadata.java

@@ -0,0 +1,52 @@
+package io.irw.hawk.scraper.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Builder.Default;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@NoArgsConstructor
+@AllArgsConstructor
+@Data
+@Builder
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class ProcessingPipelineMetadata {
+
+ @Default
+ List<ProcessingPipelineStepMetadata> pipelineStepMetadata = new ArrayList<>();
+
+ public void newStep(Class<? extends ProcessingPipelineStep> processingPipelineStep) {
+ pipelineStepMetadata.add(ProcessingPipelineStepMetadata.builder()
+ .pipelineStepClass(processingPipelineStep)
+ .build());
+ }
+
+ public void addReasoning(MerchandiseReasoningLog reasoningDto) {
+ getLastPipelineStep().getLog().add(reasoningDto);
+ }
+
+
+
+ public void addLog(String message) {
+ getLastPipelineStep().getLog().add(PipelineStepLog.builder()
+ .message(message)
+ .build());
+ }
+
+ public List<MerchandiseReasoningLog> filterReasoningsFromLog() { //retrieve from logs by instanceof
+ return getLastPipelineStep().getLog().stream()
+ .filter(MerchandiseReasoningLog.class::isInstance)
+ .map(MerchandiseReasoningLog.class::cast)
+ .toList();
+ }
+
+ private ProcessingPipelineStepMetadata getLastPipelineStep() {
+ return pipelineStepMetadata.get(pipelineStepMetadata.size() - 1);
+ }
+
+}