From 49e530ffab8c3a49af5bfb7c8ff93b2fc8399824 Mon Sep 17 00:00:00 2001 From: Dmitriusan Date: Mon, 13 Nov 2023 21:45:17 +0200 Subject: [PATCH] add persistance entities for findings, seller, highlights --- .idea/compiler.xml | 10 +++ .idea/jpa-buddy.xml | 2 +- .idea/misc.xml | 2 +- .idea/modules.xml | 2 + .idea/modules/Hawk.iml | 5 ++ .idea/modules/Hawk.main.iml | 17 ++++ hawk/build.gradle.kts | 36 ++++---- .../HawkApplicationReadyEventListener.java | 4 +- .../io/irw/hawk/dto/ebay/EbayFindingDto.java | 31 +++++++ .../hawk/dto/ebay/EbayListingStatusEnum.java | 11 +++ .../hawk/dto/ebay/EbayListingTypeEnum.java | 8 ++ .../io/irw/hawk/dto/ebay/EbaySellerDto.java | 22 +++++ .../io/irw/hawk/dto/ebay/SeenItemDto.java | 15 ---- .../hawk/dto/merchandise/HawkFlightDto.java | 8 +- .../io/irw/hawk/dto/merchandise/ItemDto.java | 25 ------ .../java/io/irw/hawk/entity/EbayFinding.java | 50 +++++++++++ .../io/irw/hawk/entity/EbayHighlight.java | 11 ++- .../java/io/irw/hawk/entity/EbaySeller.java | 50 +++++++++++ .../java/io/irw/hawk/entity/HawkFlight.java | 13 ++- .../io/irw/hawk/mapper/EbayFindingMapper.java | 21 +++++ .../irw/hawk/mapper/EbayHighlightMapper.java | 26 ++++++ .../repository/EbayFindingRepository.java | 9 ++ .../repository/EbayHighlightRepository.java | 9 ++ .../hawk/repository/EbaySellerRepository.java | 8 ++ .../hawk/repository/HawkFlightRepository.java | 9 +- .../ProductVariantEnumConverter.java | 2 +- .../scraper/model/MerchandiseMetadataDto.java | 7 +- .../service/domain/EbayFindingService.java | 40 +++++++++ .../service/domain/EbayHighlightService.java | 36 ++++++++ .../{ => domain}/HawkFlightService.java | 18 ++-- .../{ => domain}/HawkScrapeRunService.java | 3 +- .../service/extractors/PriceExtractor.java | 15 +++- .../extractors/ShippingCostExtractor.java | 6 +- .../matchers/LabedaWheelsInterestMatcher.java | 17 ++-- .../{ => scrape}/ProductCatalogService.java | 2 +- .../ScrapeTargetProviderService.java | 4 +- .../service/{ => scrape}/ScraperService.java | 4 +- .../irw/hawk/scraper/utils/PrettyPrinter.java | 2 +- .../main/java/io/irw/launcher/Launcher.java | 35 ++++---- .../changelog/changes/0001-create_enums.yaml | 17 +++- ...ate_hawk_flight_and_scrape_run_tables.yaml | 21 ++--- .../0003-create_ebay_seller_table.yaml | 33 +++++++ .../0004-create_ebay_finding_table.yaml | 88 +++++++++++++++++++ .../0005-create_ebay_highlight_table.yaml | 56 ++++++++++++ 44 files changed, 670 insertions(+), 140 deletions(-) create mode 100644 hawk/src/main/java/io/irw/hawk/dto/ebay/EbayFindingDto.java create mode 100644 hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingStatusEnum.java create mode 100644 hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingTypeEnum.java create mode 100644 hawk/src/main/java/io/irw/hawk/dto/ebay/EbaySellerDto.java delete mode 100644 hawk/src/main/java/io/irw/hawk/dto/ebay/SeenItemDto.java delete mode 100644 hawk/src/main/java/io/irw/hawk/dto/merchandise/ItemDto.java create mode 100644 hawk/src/main/java/io/irw/hawk/entity/EbaySeller.java create mode 100644 hawk/src/main/java/io/irw/hawk/mapper/EbayFindingMapper.java create mode 100644 hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java create mode 100644 hawk/src/main/java/io/irw/hawk/repository/EbayFindingRepository.java create mode 100644 hawk/src/main/java/io/irw/hawk/repository/EbayHighlightRepository.java create mode 100644 hawk/src/main/java/io/irw/hawk/repository/EbaySellerRepository.java create mode 100644 hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayFindingService.java create mode 100644 hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayHighlightService.java rename hawk/src/main/java/io/irw/hawk/scraper/service/{ => domain}/HawkFlightService.java (86%) rename hawk/src/main/java/io/irw/hawk/scraper/service/{ => domain}/HawkScrapeRunService.java (91%) rename hawk/src/main/java/io/irw/hawk/scraper/service/{ => scrape}/ProductCatalogService.java (93%) rename hawk/src/main/java/io/irw/hawk/scraper/service/{ => scrape}/ScrapeTargetProviderService.java (85%) rename hawk/src/main/java/io/irw/hawk/scraper/service/{ => scrape}/ScraperService.java (94%) create mode 100644 hawk/src/main/resources/db/changelog/changes/0003-create_ebay_seller_table.yaml create mode 100644 hawk/src/main/resources/db/changelog/changes/0004-create_ebay_finding_table.yaml create mode 100644 hawk/src/main/resources/db/changelog/changes/0005-create_ebay_highlight_table.yaml diff --git a/.idea/compiler.xml b/.idea/compiler.xml index a1757ae..8ab7a9e 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -3,6 +3,16 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/jpa-buddy.xml b/.idea/jpa-buddy.xml index 9dad284..376a1f4 100644 --- a/.idea/jpa-buddy.xml +++ b/.idea/jpa-buddy.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/.idea/modules/Hawk.iml b/.idea/modules/Hawk.iml index 26d2978..ded924b 100644 --- a/.idea/modules/Hawk.iml +++ b/.idea/modules/Hawk.iml @@ -1,5 +1,10 @@ + + + + + diff --git a/.idea/modules/Hawk.main.iml b/.idea/modules/Hawk.main.iml index 9c9e65e..1d0aa88 100644 --- a/.idea/modules/Hawk.main.iml +++ b/.idea/modules/Hawk.main.iml @@ -1,5 +1,22 @@ + + + + + + + + + + + + + + + + + diff --git a/hawk/build.gradle.kts b/hawk/build.gradle.kts index d632c30..ab6b757 100644 --- a/hawk/build.gradle.kts +++ b/hawk/build.gradle.kts @@ -2,7 +2,7 @@ import java.net.URL plugins { java - id("org.springframework.boot") version "3.1.4" + id("org.springframework.boot") version "3.1.5" id("io.spring.dependency-management") version "1.1.3" // Depedencies for Ebay hawk id ("org.openapi.generator") version "7.0.1" @@ -30,42 +30,48 @@ repositories { mavenCentral() } -extra["springBootAdminVersion"] = "3.1.5" +extra["jacksonVersion"] = "2.15.3" +extra["swaggerVersion"] = "2.2.16" +extra["springBootAdminVersion"] = "3.1.7" dependencies { implementation("org.springframework.boot:spring-boot-starter-actuator") - implementation("org.springframework.boot:spring-boot-starter-data-jpa") implementation("org.springframework.boot:spring-boot-starter-security") implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-validation") implementation("org.springframework.boot:spring-boot-starter-logging") - annotationProcessor("org.projectlombok:lombok") - testImplementation("org.assertj:assertj-core:3.24.2") - - - - implementation("de.codecentric:spring-boot-admin-starter-server") implementation("io.micrometer:micrometer-tracing-bridge-brave") implementation("io.zipkin.reporter2:zipkin-reporter-brave") - implementation("org.liquibase:liquibase-core") - developmentOnly("org.springframework.boot:spring-boot-devtools") - runtimeOnly("org.postgresql:postgresql") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") + + // DB dependencies + implementation("org.springframework.boot:spring-boot-starter-data-jpa") + implementation("io.hypersistence:hypersistence-utils-hibernate-62:3.6.1") + + developmentOnly("org.springframework.boot:spring-boot-devtools") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("org.springframework.security:spring-security-test") + testImplementation("org.assertj:assertj-core:3.24.2") + + implementation("org.liquibase:liquibase-core") + runtimeOnly("org.postgresql:postgresql") implementation("org.projectlombok:lombok:1.18.30") + annotationProcessor("org.projectlombok:lombok") implementation("org.mapstruct:mapstruct:1.5.5.Final") // Mapstruct Processor (Annotation Processor) annotationProcessor("org.mapstruct:mapstruct-processor:1.5.5.Final") + annotationProcessor("org.projectlombok:lombok-mapstruct-binding:0.2.0") // Ebay hawk dependencies // OpenAPI client dependencies - implementation("com.fasterxml.jackson.core:jackson-databind:2.15.3") + implementation("com.fasterxml.jackson.core:jackson-databind:${property("jacksonVersion")}") + implementation("com.fasterxml.jackson.module:jackson-modules-java8:${property("jacksonVersion")}") + + implementation("io.swagger.core.v3:swagger-annotations:${property("swaggerVersion")}") + implementation("io.swagger.core.v3:swagger-models:${property("swaggerVersion")}") implementation("org.openapitools:jackson-databind-nullable:0.2.6") - implementation("io.swagger.core.v3:swagger-annotations:2.2.16") - implementation("io.swagger.core.v3:swagger-models:2.2.16") implementation("com.ebay.auth:ebay-oauth-java-client:1.1.8") implementation("org.jgrapht:jgrapht-core:1.5.2") } diff --git a/hawk/src/main/java/io/irw/hawk/HawkApplicationReadyEventListener.java b/hawk/src/main/java/io/irw/hawk/HawkApplicationReadyEventListener.java index 9b9d816..88b159d 100644 --- a/hawk/src/main/java/io/irw/hawk/HawkApplicationReadyEventListener.java +++ b/hawk/src/main/java/io/irw/hawk/HawkApplicationReadyEventListener.java @@ -1,7 +1,7 @@ package io.irw.hawk; -import io.irw.hawk.scraper.service.HawkFlightService; -import io.irw.hawk.scraper.service.ScraperService; +import io.irw.hawk.scraper.service.domain.HawkFlightService; +import io.irw.hawk.scraper.service.scrape.ScraperService; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; diff --git a/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayFindingDto.java b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayFindingDto.java new file mode 100644 index 0000000..37c280e --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayFindingDto.java @@ -0,0 +1,31 @@ +package io.irw.hawk.dto.ebay; + +import java.math.BigDecimal; +import java.time.Instant; +import lombok.Builder; +import lombok.Value; + +/** + * DTO for {@link io.irw.hawk.entity.EbayFinding} + */ +@Value +@Builder +public class EbayFindingDto { + + Long id; + String ebayIdStr; + String legacyEbayIdStr; + String itemTitle; + String itemDescription; + BigDecimal currentAuctionPriceUsd; + BigDecimal finalPriceUsd; + BigDecimal minShippingUsd; + BigDecimal buyItNowPriceUsd; + int bidCount; + Instant capturedAt; + Instant endsOn; + EbayListingTypeEnum listingType; + EbayListingStatusEnum listingStatusEnum; + EbaySellerDto seller; + +} \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingStatusEnum.java b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingStatusEnum.java new file mode 100644 index 0000000..f385ed1 --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingStatusEnum.java @@ -0,0 +1,11 @@ +package io.irw.hawk.dto.ebay; + +public enum EbayListingStatusEnum { + + ACTIVE, + BOUGHT_THROUGH_AUCTION, + BOUGHT_THROUGH_BUY_IT_NOW, + CANCELLED, + + +} diff --git a/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingTypeEnum.java b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingTypeEnum.java new file mode 100644 index 0000000..faa10d3 --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbayListingTypeEnum.java @@ -0,0 +1,8 @@ +package io.irw.hawk.dto.ebay; + +public enum EbayListingTypeEnum { + + AUCTION, + BUY_IT_NOW + +} diff --git a/hawk/src/main/java/io/irw/hawk/dto/ebay/EbaySellerDto.java b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbaySellerDto.java new file mode 100644 index 0000000..7278087 --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/dto/ebay/EbaySellerDto.java @@ -0,0 +1,22 @@ +package io.irw.hawk.dto.ebay; + +import java.io.Serializable; +import java.time.Instant; +import java.util.Date; +import lombok.Builder; +import lombok.Value; + +/** + * DTO for {@link io.irw.hawk.entity.EbayFinding} + */ +@Value +@Builder +public class EbaySellerDto implements Serializable { + + Long id; + String ebayIdStr; + Instant registeredOn; + float reputationPercentage; + int feedbackScore; + +} \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/dto/ebay/SeenItemDto.java b/hawk/src/main/java/io/irw/hawk/dto/ebay/SeenItemDto.java deleted file mode 100644 index 406bbe2..0000000 --- a/hawk/src/main/java/io/irw/hawk/dto/ebay/SeenItemDto.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.irw.hawk.dto.ebay; - -import lombok.AccessLevel; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@FieldDefaults(level = AccessLevel.PRIVATE) -public class SeenItemDto { - - - -} diff --git a/hawk/src/main/java/io/irw/hawk/dto/merchandise/HawkFlightDto.java b/hawk/src/main/java/io/irw/hawk/dto/merchandise/HawkFlightDto.java index 43cc5e0..8de69da 100644 --- a/hawk/src/main/java/io/irw/hawk/dto/merchandise/HawkFlightDto.java +++ b/hawk/src/main/java/io/irw/hawk/dto/merchandise/HawkFlightDto.java @@ -15,16 +15,14 @@ public class HawkFlightDto implements Serializable { Long id; - HawkFlightStateEnum state; + HawkFlightStatusEnum status; Instant startedAt; Instant endedAt; @Getter @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) - public enum HawkFlightStateEnum { - IN_PROGRESS(0), FAILED(1), ENDED(2); - - int id; + public enum HawkFlightStatusEnum { + IN_PROGRESS, FAILED, ENDED; } } \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/dto/merchandise/ItemDto.java b/hawk/src/main/java/io/irw/hawk/dto/merchandise/ItemDto.java deleted file mode 100644 index 8dd882b..0000000 --- a/hawk/src/main/java/io/irw/hawk/dto/merchandise/ItemDto.java +++ /dev/null @@ -1,25 +0,0 @@ -package io.irw.hawk.dto.merchandise; - -import lombok.AccessLevel; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -/** - * DTO for {@link io.irw.hawk.entity.Item} - */ -/** - * DTO for {@link io.irw.hawk.entity.Item} - */ -@Data -@Builder -@FieldDefaults(level = AccessLevel.PRIVATE) -public class ItemDto { - - Long id; - String ebayId; - String ebayLegacyId; - String itemName; - float priceUsd; - -} diff --git a/hawk/src/main/java/io/irw/hawk/entity/EbayFinding.java b/hawk/src/main/java/io/irw/hawk/entity/EbayFinding.java index d873fa4..ba197d0 100644 --- a/hawk/src/main/java/io/irw/hawk/entity/EbayFinding.java +++ b/hawk/src/main/java/io/irw/hawk/entity/EbayFinding.java @@ -1,7 +1,11 @@ package io.irw.hawk.entity; +import io.irw.hawk.dto.ebay.EbayListingStatusEnum; +import io.irw.hawk.dto.ebay.EbayListingTypeEnum; import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; @@ -9,6 +13,8 @@ import jakarta.persistence.ManyToOne; import jakarta.persistence.SequenceGenerator; import jakarta.persistence.Table; +import java.math.BigDecimal; +import java.time.Instant; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -33,7 +39,51 @@ public class EbayFinding { generator = "ebay_finding_seq") Long id; + @Column(name = "ebay_id_str", nullable = false, unique = true) + @EqualsAndHashCode.Include + private String ebayIdStr; + @Column(name = "legacy_ebay_id_str") + @EqualsAndHashCode.Include + private String legacyEbayIdStr; + @Column(name = "item_title", nullable = false) + private String itemTitle; + + @Column(name = "item_description") + private String itemDescription; + + @Column(name = "current_auction_price_usd") + private BigDecimal currentAuctionPriceUsd; + + @Column(name = "final_price_usd") + private BigDecimal finalPriceUsd; + + @Column(name = "min_shipping_usd") + private BigDecimal minShippingUsd; + + @Column(name = "buy_it_now_price_usd") + private BigDecimal buyItNowPriceUsd; + + @Column(name = "bid_count") + private int bidCount; + + @Column(name = "captured_at", nullable = false) + private Instant capturedAt; + + @Column(name = "ends_on", nullable = false) + private Instant endsOn; + + @Enumerated(EnumType.STRING) + @Column(name = "listing_type", nullable = false, columnDefinition = "ebay_listing_type") + private EbayListingTypeEnum listingType; + + @Enumerated(EnumType.STRING) + @Column(name = "listing_status", nullable = false, columnDefinition = "ebay_listing_status") + private EbayListingStatusEnum listingStatusEnum; + + @ManyToOne + @JoinColumn(name = "seller_id") + private EbaySeller seller; } diff --git a/hawk/src/main/java/io/irw/hawk/entity/EbayHighlight.java b/hawk/src/main/java/io/irw/hawk/entity/EbayHighlight.java index f656710..bc83869 100644 --- a/hawk/src/main/java/io/irw/hawk/entity/EbayHighlight.java +++ b/hawk/src/main/java/io/irw/hawk/entity/EbayHighlight.java @@ -1,7 +1,10 @@ package io.irw.hawk.entity; +import io.irw.hawk.scraper.model.MerchandiseVerdictType; import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; @@ -36,15 +39,15 @@ public class EbayHighlight { @ManyToOne @JoinColumn(name = "run_id") @EqualsAndHashCode.Include - HawkScrapeRun runId; + HawkScrapeRun run; @ManyToOne @JoinColumn(name = "ebay_finding_id") @EqualsAndHashCode.Include EbayFinding ebayFinding; - - - + @Enumerated(EnumType.STRING) + @Column(name = "final_verdict", columnDefinition = "merchandise_verdict_type") + MerchandiseVerdictType finalVerdict; } diff --git a/hawk/src/main/java/io/irw/hawk/entity/EbaySeller.java b/hawk/src/main/java/io/irw/hawk/entity/EbaySeller.java new file mode 100644 index 0000000..a6b79f8 --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/entity/EbaySeller.java @@ -0,0 +1,50 @@ +package io.irw.hawk.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.SequenceGenerator; +import jakarta.persistence.Table; +import java.time.Instant; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * Some info collected about the seller + */ +@Getter +@Setter +@Entity +@Table(name = "ebay_seller", schema = "merchandise_db") +@EqualsAndHashCode(onlyExplicitlyIncluded = true) +public class EbaySeller { + + @Id + @Column(name = "id", updatable = false) + @SequenceGenerator(name = "ebay_seller_seq", + sequenceName = "ebay_seller_seq", + allocationSize = 1) + @GeneratedValue(strategy = GenerationType.SEQUENCE, + generator = "ebay_seller_seq") + Long id; + + @Column(name = "ebay_id_str") + @EqualsAndHashCode.Include + String ebayIdStr; + + @Column(name = "registered_on") + Instant registeredOn; + + @Column(name = "reputation_percentage") + float reputationPercentage; + + @Column(name = "feedback_score") + int feedbackScore; + +} diff --git a/hawk/src/main/java/io/irw/hawk/entity/HawkFlight.java b/hawk/src/main/java/io/irw/hawk/entity/HawkFlight.java index abce8e7..e5874fa 100644 --- a/hawk/src/main/java/io/irw/hawk/entity/HawkFlight.java +++ b/hawk/src/main/java/io/irw/hawk/entity/HawkFlight.java @@ -1,8 +1,11 @@ package io.irw.hawk.entity; -import io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStateEnum; +import io.hypersistence.utils.hibernate.type.basic.PostgreSQLEnumType; +import io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStatusEnum; import jakarta.persistence.Column; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; @@ -14,6 +17,8 @@ import lombok.Getter; import lombok.Setter; import lombok.experimental.FieldDefaults; +import org.hibernate.annotations.ColumnTransformer; +import org.hibernate.annotations.Type; @Getter @Setter @@ -36,8 +41,10 @@ public class HawkFlight { @EqualsAndHashCode.Include Instant startedAt; - @Column(name = "state", nullable = false) - HawkFlightStateEnum state; + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, columnDefinition = "hawk_flight_status") + @Type(PostgreSQLEnumType.class) + HawkFlightStatusEnum status; @Column(name = "ended_at") Instant endedAt; diff --git a/hawk/src/main/java/io/irw/hawk/mapper/EbayFindingMapper.java b/hawk/src/main/java/io/irw/hawk/mapper/EbayFindingMapper.java new file mode 100644 index 0000000..fa99cf8 --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/mapper/EbayFindingMapper.java @@ -0,0 +1,21 @@ +package io.irw.hawk.mapper; + +import io.irw.hawk.dto.ebay.EbayFindingDto; +import io.irw.hawk.entity.EbayFinding; +import org.mapstruct.BeanMapping; +import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; +import org.mapstruct.NullValuePropertyMappingStrategy; + +@Mapper(config = ConfigMapper.class) +public interface EbayFindingMapper { + + EbayFinding toEntity(EbayFindingDto ebayFindingDto); + + EbayFindingDto toDto(EbayFinding ebayFinding); + + @BeanMapping(nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE) + EbayFinding partialUpdate( + EbayFindingDto ebayFindingDto, @MappingTarget EbayFinding ebayFinding); + +} diff --git a/hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java b/hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java new file mode 100644 index 0000000..aac07b8 --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/mapper/EbayHighlightMapper.java @@ -0,0 +1,26 @@ +package io.irw.hawk.mapper; + +import io.irw.hawk.entity.EbayFinding; +import io.irw.hawk.entity.EbayHighlight; +import io.irw.hawk.entity.HawkScrapeRun; +import io.irw.hawk.scraper.model.MerchandiseMetadataDto; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Mappings; + +@Mapper(config = ConfigMapper.class) +public interface EbayHighlightMapper { + +// @BeanMapping(nullValueCheckStrategy = NullValueCheckStrategy.ALWAYS, +// nullValuePropertyMappingStrategy = NullValuePropertyMappingStrategy.IGNORE) + @Mappings({ + @Mapping(target = "id", ignore = true), + @Mapping(target = "run", source = "hawkScrapeRun"), + @Mapping(target = "ebayFinding", source = "ebayFinding"), + @Mapping(target = "finalVerdict", source = "merchandiseMetadataDto.finalVerdict"), + }) + EbayHighlight toEntity(MerchandiseMetadataDto merchandiseMetadataDto, HawkScrapeRun hawkScrapeRun, + EbayFinding ebayFinding); + + +} diff --git a/hawk/src/main/java/io/irw/hawk/repository/EbayFindingRepository.java b/hawk/src/main/java/io/irw/hawk/repository/EbayFindingRepository.java new file mode 100644 index 0000000..9641d0b --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/repository/EbayFindingRepository.java @@ -0,0 +1,9 @@ +package io.irw.hawk.repository; + +import io.irw.hawk.entity.EbayFinding; +import io.irw.hawk.entity.HawkScrapeRun; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface EbayFindingRepository extends JpaRepository { + +} \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/repository/EbayHighlightRepository.java b/hawk/src/main/java/io/irw/hawk/repository/EbayHighlightRepository.java new file mode 100644 index 0000000..20e22af --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/repository/EbayHighlightRepository.java @@ -0,0 +1,9 @@ +package io.irw.hawk.repository; + +import io.irw.hawk.entity.EbayFinding; +import io.irw.hawk.entity.EbayHighlight; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface EbayHighlightRepository extends JpaRepository { + +} \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/repository/EbaySellerRepository.java b/hawk/src/main/java/io/irw/hawk/repository/EbaySellerRepository.java new file mode 100644 index 0000000..0b1999b --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/repository/EbaySellerRepository.java @@ -0,0 +1,8 @@ +package io.irw.hawk.repository; + +import io.irw.hawk.entity.EbaySeller; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface EbaySellerRepository extends JpaRepository { + +} \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/repository/HawkFlightRepository.java b/hawk/src/main/java/io/irw/hawk/repository/HawkFlightRepository.java index c1eedb5..0360d6d 100644 --- a/hawk/src/main/java/io/irw/hawk/repository/HawkFlightRepository.java +++ b/hawk/src/main/java/io/irw/hawk/repository/HawkFlightRepository.java @@ -1,6 +1,6 @@ package io.irw.hawk.repository; -import io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStateEnum; +import io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStatusEnum; import io.irw.hawk.entity.HawkFlight; import java.util.List; import org.springframework.data.jpa.repository.JpaRepository; @@ -9,13 +9,8 @@ public interface HawkFlightRepository extends JpaRepository, JpaSpecificationExecutor { - @Query("select h from HawkFlight h where h.state = ?1") - HawkFlight findByState(HawkFlightStateEnum state); - - HawkFlight findOneByState(HawkFlightStateEnum state); + HawkFlight findOneByStatus(HawkFlightStatusEnum state); List findAllByEndedAtNull(); - HawkFlight findOneByEndedAtNull(); - } \ No newline at end of file diff --git a/hawk/src/main/java/io/irw/hawk/repository/converters/ProductVariantEnumConverter.java b/hawk/src/main/java/io/irw/hawk/repository/converters/ProductVariantEnumConverter.java index 6779a61..40f2b05 100644 --- a/hawk/src/main/java/io/irw/hawk/repository/converters/ProductVariantEnumConverter.java +++ b/hawk/src/main/java/io/irw/hawk/repository/converters/ProductVariantEnumConverter.java @@ -4,7 +4,7 @@ import jakarta.persistence.AttributeConverter; import jakarta.persistence.Converter; -@Converter +@Converter(autoApply = true) public class ProductVariantEnumConverter implements AttributeConverter { @Override diff --git a/hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseMetadataDto.java b/hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseMetadataDto.java index 0f4a44f..4ca270d 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseMetadataDto.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/model/MerchandiseMetadataDto.java @@ -3,6 +3,7 @@ import static io.irw.hawk.scraper.model.MerchandiseVerdictType.BUYING_OPPORTUNITY; import io.irw.hawk.dto.merchandise.HawkScrapeRunDto; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -24,9 +25,9 @@ public class MerchandiseMetadataDto { HawkScrapeRunDto hawkScrapeRunDto; @Default Optional numberOfPieces = Optional.of(1); - Optional pricePerPieceUsd; - double totalPriceUsd; - Optional minShippingCostUsd; + Optional pricePerPieceUsd; + BigDecimal totalPriceUsd; + Optional minShippingCostUsd; @Default MerchandiseVerdictType finalVerdict = BUYING_OPPORTUNITY; @Default diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayFindingService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayFindingService.java new file mode 100644 index 0000000..abe25cf --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayFindingService.java @@ -0,0 +1,40 @@ +package io.irw.hawk.scraper.service.domain; + +import io.irw.hawk.dto.merchandise.HawkScrapeRunDto; +import io.irw.hawk.dto.merchandise.ProductVariantEnum; +import io.irw.hawk.mapper.EbayFindingMapper; +import io.irw.hawk.mapper.HawkScrapeRunMapper; +import io.irw.hawk.repository.EbayFindingRepository; +import io.irw.hawk.repository.HawkScrapeRunRepository; +import java.time.Instant; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@Service +@Slf4j +public class EbayFindingService { + + EbayFindingMapper ebayFindingMapper; + EbayFindingRepository ebayFindingRepository; +// HawkScrapeRunMapper hawkScrapeRunMapper; +// HawkFlightService hawkFlightService; +// +// @Transactional +// public HawkScrapeRunDto startScrapeRun(ProductVariantEnum targetProductVariant) { +// HawkScrapeRunDto hawkScrapeRunDto = HawkScrapeRunDto.builder() +// .hawkFlight(hawkFlightService.getCurrentFlight()) +// .startedAt(Instant.now()) +// .productVariant(targetProductVariant) +// .build(); +// hawkFlightRepository.save(hawkScrapeRunMapper.toEntity(hawkScrapeRunDto)); +// return hawkScrapeRunDto; +// } + + +} diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayHighlightService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayHighlightService.java new file mode 100644 index 0000000..6bb69dd --- /dev/null +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/EbayHighlightService.java @@ -0,0 +1,36 @@ +package io.irw.hawk.scraper.service.domain; + +import io.irw.hawk.mapper.EbayHighlightMapper; +import io.irw.hawk.repository.EbayHighlightRepository; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +@Service +@Slf4j +public class EbayHighlightService { + + EbayHighlightMapper ebayHighlightMapper; + EbayHighlightRepository ebayHighlightRepository; + +// HawkScrapeRunRepository hawkFlightRepository; +// HawkScrapeRunMapper hawkScrapeRunMapper; +// HawkFlightService hawkFlightService; +// +// @Transactional +// public HawkScrapeRunDto startScrapeRun(ProductVariantEnum targetProductVariant) { +// HawkScrapeRunDto hawkScrapeRunDto = HawkScrapeRunDto.builder() +// .hawkFlight(hawkFlightService.getCurrentFlight()) +// .startedAt(Instant.now()) +// .productVariant(targetProductVariant) +// .build(); +// hawkFlightRepository.save(hawkScrapeRunMapper.toEntity(hawkScrapeRunDto)); +// return hawkScrapeRunDto; +// } + + +} diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/HawkFlightService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/HawkFlightService.java similarity index 86% rename from hawk/src/main/java/io/irw/hawk/scraper/service/HawkFlightService.java rename to hawk/src/main/java/io/irw/hawk/scraper/service/domain/HawkFlightService.java index 848781c..75a8774 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/HawkFlightService.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/HawkFlightService.java @@ -1,10 +1,10 @@ -package io.irw.hawk.scraper.service; +package io.irw.hawk.scraper.service.domain; -import static io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStateEnum.FAILED; -import static io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStateEnum.IN_PROGRESS; +import static io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStatusEnum.FAILED; +import static io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStatusEnum.IN_PROGRESS; import io.irw.hawk.dto.merchandise.HawkFlightDto; -import io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStateEnum; +import io.irw.hawk.dto.merchandise.HawkFlightDto.HawkFlightStatusEnum; import io.irw.hawk.entity.HawkFlight; import io.irw.hawk.mapper.HawkFlightMapper; import io.irw.hawk.repository.HawkFlightRepository; @@ -33,7 +33,7 @@ public void startFlight() { forceClosePreviousUnfinishedFlights(); HawkFlightDto hawkFlightDto = HawkFlightDto.builder() - .state(IN_PROGRESS) + .status(IN_PROGRESS) .startedAt(Instant.now()) .build(); hawkFlightRepository.save(hawkFlightMapper.toEntity(hawkFlightDto)); @@ -47,7 +47,7 @@ private void forceClosePreviousUnfinishedFlights() { } allUnfinishedPreviousFlights.forEach(flight -> { - flight.setState(FAILED); + flight.setStatus(FAILED); flight.setEndedAt(Instant.now()); }); hawkFlightRepository.saveAll(allUnfinishedPreviousFlights); @@ -55,13 +55,13 @@ private void forceClosePreviousUnfinishedFlights() { @Transactional(readOnly = true) public HawkFlightDto getCurrentFlight() { - return hawkFlightMapper.toDto(hawkFlightRepository.findOneByState(IN_PROGRESS)); + return hawkFlightMapper.toDto(hawkFlightRepository.findOneByStatus(IN_PROGRESS)); } @Transactional public void finishFlight() { - HawkFlight inProgressFlight = hawkFlightRepository.findOneByState(IN_PROGRESS); - inProgressFlight.setState(HawkFlightStateEnum.ENDED); + HawkFlight inProgressFlight = hawkFlightRepository.findOneByStatus(IN_PROGRESS); + inProgressFlight.setStatus(HawkFlightStatusEnum.ENDED); inProgressFlight.setEndedAt(Instant.now()); hawkFlightRepository.save(inProgressFlight); log.info("Finishing Hawk flight"); diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/HawkScrapeRunService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/HawkScrapeRunService.java similarity index 91% rename from hawk/src/main/java/io/irw/hawk/scraper/service/HawkScrapeRunService.java rename to hawk/src/main/java/io/irw/hawk/scraper/service/domain/HawkScrapeRunService.java index 3151b89..af7aa2b 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/HawkScrapeRunService.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/domain/HawkScrapeRunService.java @@ -1,9 +1,10 @@ -package io.irw.hawk.scraper.service; +package io.irw.hawk.scraper.service.domain; import io.irw.hawk.dto.merchandise.HawkScrapeRunDto; import io.irw.hawk.dto.merchandise.ProductVariantEnum; import io.irw.hawk.mapper.HawkScrapeRunMapper; import io.irw.hawk.repository.HawkScrapeRunRepository; +import io.irw.hawk.scraper.service.domain.HawkFlightService; import java.time.Instant; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/PriceExtractor.java b/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/PriceExtractor.java index 32d50ed..2e10382 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/PriceExtractor.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/PriceExtractor.java @@ -5,11 +5,14 @@ import io.irw.hawk.scraper.model.MerchandiseMetadataDto; import io.irw.hawk.scraper.model.ProcessingPipelineStep; import io.irw.hawk.scraper.service.matchers.ShippingPossibilitiesMatcher; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.List; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.stereotype.Service; @RequiredArgsConstructor @@ -30,9 +33,17 @@ public boolean isApplicableTo(ProductVariantEnum productVariant) { @Override public void extractDataFromItemSummary(ItemSummary itemSummary, MerchandiseMetadataDto metadata) { - metadata.setTotalPriceUsd(Float.valueOf(itemSummary.getPrice().getValue())); + double priceDoubleValue = Double.parseDouble(itemSummary.getPrice().getValue()); + metadata.setTotalPriceUsd(BigDecimal.valueOf(priceDoubleValue)); metadata.setPricePerPieceUsd(metadata.getNumberOfPieces() .flatMap(pieces -> metadata.getMinShippingCostUsd() - .map(shippingCost -> (metadata.getTotalPriceUsd() + shippingCost) / pieces))); + .map(shippingCost -> calculatePricePerPieceWithShipping(metadata, pieces, shippingCost)))); + } + + @NotNull + private static BigDecimal calculatePricePerPieceWithShipping(MerchandiseMetadataDto metadata, Integer pieces, BigDecimal shippingCost) { + return metadata.getTotalPriceUsd() + .add(shippingCost) + .divide(BigDecimal.valueOf(pieces), RoundingMode.HALF_UP); } } diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/ShippingCostExtractor.java b/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/ShippingCostExtractor.java index f75fb4a..2f063bb 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/ShippingCostExtractor.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/extractors/ShippingCostExtractor.java @@ -3,6 +3,7 @@ import com.ebay.buy.browse.model.ItemSummary; import io.irw.hawk.dto.merchandise.ProductVariantEnum; import io.irw.hawk.scraper.model.MerchandiseMetadataDto; +import java.math.BigDecimal; import java.util.Comparator; import java.util.Optional; import lombok.AccessLevel; @@ -24,12 +25,13 @@ public boolean isApplicableTo(ProductVariantEnum productVariant) { @Override public void extractDataFromItemSummary(ItemSummary itemSummary, MerchandiseMetadataDto metadata) { - Optional minShippingCost = Optional.ofNullable(itemSummary.getShippingOptions()) + Optional minShippingCost = Optional.ofNullable(itemSummary.getShippingOptions()) .flatMap(shippingOptions -> shippingOptions.stream() .filter( so -> so.getShippingCost() != null) // Looks like shipping cost from Canada is CALCULATED/null .map(shippingOptionSummary -> Double.valueOf(shippingOptionSummary.getShippingCost().getValue())) - .min(Comparator.naturalOrder())); + .min(Comparator.naturalOrder())) + .map(BigDecimal::valueOf); metadata.setMinShippingCostUsd(minShippingCost); } diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/processors/skates/parts/matchers/LabedaWheelsInterestMatcher.java b/hawk/src/main/java/io/irw/hawk/scraper/service/processors/skates/parts/matchers/LabedaWheelsInterestMatcher.java index 05c621f..94cb3d3 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/processors/skates/parts/matchers/LabedaWheelsInterestMatcher.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/processors/skates/parts/matchers/LabedaWheelsInterestMatcher.java @@ -10,6 +10,7 @@ import io.irw.hawk.scraper.service.matchers.BaselineItemDataMatcher; import io.irw.hawk.scraper.service.matchers.ItemSummaryMatcher; import io.irw.hawk.scraper.service.processors.skates.parts.extractors.WheelCountExtractor; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import lombok.AccessLevel; @@ -24,9 +25,9 @@ @Slf4j public class LabedaWheelsInterestMatcher implements ItemSummaryMatcher { - static double DESIRED_PRICE_PER_WHEEL = 5; - static double DESIRED_MIN_WHEEL_COUNT = 4; - static double MEEST_SHIPPING_AND_HANDLING_PER_PIECE = 4; + static BigDecimal DESIRED_PRICE_PER_WHEEL = BigDecimal.valueOf(5); + static int DESIRED_MIN_WHEEL_COUNT = 4; + static BigDecimal MEEST_SHIPPING_AND_HANDLING_PER_SHIPPING = BigDecimal.valueOf(4); @Override public List> dependsOn() { @@ -40,14 +41,16 @@ public List match(ItemSummary itemSummary, MerchandiseM return result; } - double pricePerPieceUsd = metadata.getPricePerPieceUsd().get(); + BigDecimal pricePerPieceUsd = metadata.getPricePerPieceUsd().get(); int numberOfPieces = metadata.getNumberOfPieces().get(); - if (pricePerPieceUsd > DESIRED_PRICE_PER_WHEEL) { + if (pricePerPieceUsd.compareTo(DESIRED_PRICE_PER_WHEEL) > 0) { result.add(newReasoningDto(String.format("Too pricey: %s$ per wheel > %s$", pricePerPieceUsd, DESIRED_PRICE_PER_WHEEL), NOT_INTERESTING)); } else if (numberOfPieces < DESIRED_MIN_WHEEL_COUNT) { - double priceBenefit = (numberOfPieces * DESIRED_PRICE_PER_WHEEL) - (numberOfPieces * pricePerPieceUsd); - if (priceBenefit < MEEST_SHIPPING_AND_HANDLING_PER_PIECE) { + BigDecimal priceBenefit = BigDecimal.valueOf(numberOfPieces) + .multiply(DESIRED_PRICE_PER_WHEEL) + .subtract((BigDecimal.valueOf(numberOfPieces).multiply(pricePerPieceUsd))); + if (priceBenefit.compareTo(MEEST_SHIPPING_AND_HANDLING_PER_SHIPPING) < 0) { result.add(newReasoningDto( String.format("Items are cheap, but too small quantity for Meest shipping&handling: %s$ price " + "benefit", priceBenefit), NOT_INTERESTING)); diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/ProductCatalogService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ProductCatalogService.java similarity index 93% rename from hawk/src/main/java/io/irw/hawk/scraper/service/ProductCatalogService.java rename to hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ProductCatalogService.java index 3b5ae91..22e7426 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/ProductCatalogService.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ProductCatalogService.java @@ -1,4 +1,4 @@ -package io.irw.hawk.scraper.service; +package io.irw.hawk.scraper.service.scrape; import io.irw.hawk.dto.merchandise.ProductVariantEnum; import java.util.List; diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/ScrapeTargetProviderService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScrapeTargetProviderService.java similarity index 85% rename from hawk/src/main/java/io/irw/hawk/scraper/service/ScrapeTargetProviderService.java rename to hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScrapeTargetProviderService.java index 7408a16..afc3bfb 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/ScrapeTargetProviderService.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScrapeTargetProviderService.java @@ -1,7 +1,5 @@ -package io.irw.hawk.scraper.service; +package io.irw.hawk.scraper.service.scrape; -import io.irw.hawk.dto.merchandise.HawkFlightDto; -import io.irw.hawk.dto.merchandise.HawkScrapeRunDto; import io.irw.hawk.dto.merchandise.ProductVariantEnum; import java.util.Optional; import java.util.Queue; diff --git a/hawk/src/main/java/io/irw/hawk/scraper/service/ScraperService.java b/hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScraperService.java similarity index 94% rename from hawk/src/main/java/io/irw/hawk/scraper/service/ScraperService.java rename to hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScraperService.java index 393b91e..f91ec09 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/service/ScraperService.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/service/scrape/ScraperService.java @@ -1,9 +1,9 @@ -package io.irw.hawk.scraper.service; +package io.irw.hawk.scraper.service.scrape; -import io.irw.hawk.dto.merchandise.HawkFlightDto; import io.irw.hawk.dto.merchandise.HawkScrapeRunDto; import io.irw.hawk.dto.merchandise.ProductVariantEnum; import io.irw.hawk.scraper.exceptions.ScrapingException; +import io.irw.hawk.scraper.service.domain.HawkScrapeRunService; import io.irw.hawk.scraper.service.processors.ProductScrapeProcessor; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/hawk/src/main/java/io/irw/hawk/scraper/utils/PrettyPrinter.java b/hawk/src/main/java/io/irw/hawk/scraper/utils/PrettyPrinter.java index b84a067..8eb0443 100644 --- a/hawk/src/main/java/io/irw/hawk/scraper/utils/PrettyPrinter.java +++ b/hawk/src/main/java/io/irw/hawk/scraper/utils/PrettyPrinter.java @@ -17,7 +17,7 @@ public void printSplitter() { public void prettyPrint(ItemSummary itemSummary, MerchandiseMetadataDto metadataDto) { String priceWithShippingCost = metadataDto.getMinShippingCostUsd() - .map(amount -> amount + metadataDto.getTotalPriceUsd()) + .map(amount -> amount.add(metadataDto.getTotalPriceUsd())) .map(amount -> String.format("%.2f", amount)) .orElse("NO SHIPPING COST"); String numberOfPieces = metadataDto.getNumberOfPieces() diff --git a/hawk/src/main/java/io/irw/launcher/Launcher.java b/hawk/src/main/java/io/irw/launcher/Launcher.java index 854aec4..d147e3f 100644 --- a/hawk/src/main/java/io/irw/launcher/Launcher.java +++ b/hawk/src/main/java/io/irw/launcher/Launcher.java @@ -1,8 +1,10 @@ package io.irw.launcher; import io.irw.hawk.HawkApp; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -13,24 +15,25 @@ public class Launcher { public static void main(String[] args) { try (ExecutorService executorService = Executors.newCachedThreadPool()) { - - // Start the first Spring Boot application in a separate thread - executorService.submit(() -> { - ConfigurableApplicationContext hawkAppContext = SpringApplication.run(HawkApp.class, args); - hawkAppContext.close(); + // Start the Spring Boot application in a separate thread + Future future = executorService.submit(() -> { + return SpringApplication.run(HawkApp.class, args); }); - - // Shutdown the executor service gracefully when both applications are done - executorService.shutdown(); - try { - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - Thread.currentThread() - .interrupt(); - } + // Get the application context from the future and manage its lifecycle + ConfigurableApplicationContext hawkAppContext = future.get(); + // Perform any necessary operations with hawkAppContext + + // Close the application context gracefully + Runtime.getRuntime() + .addShutdownHook(new Thread(hawkAppContext::close)); + } catch (InterruptedException e) { + Thread.currentThread() + .interrupt(); // Preserve interrupt status + throw new RuntimeException("Thread interrupted", e); + } catch (ExecutionException e) { + throw new RuntimeException("Execution exception in application startup", e); } } - - } + diff --git a/hawk/src/main/resources/db/changelog/changes/0001-create_enums.yaml b/hawk/src/main/resources/db/changelog/changes/0001-create_enums.yaml index 417d374..0c570ee 100644 --- a/hawk/src/main/resources/db/changelog/changes/0001-create_enums.yaml +++ b/hawk/src/main/resources/db/changelog/changes/0001-create_enums.yaml @@ -1,6 +1,6 @@ databaseChangeLog: - changeSet: - id: create_enums + id: 0001-create_enums author: dmitriusan changes: - sql: @@ -20,4 +20,19 @@ databaseChangeLog: 'BUYING_OPPORTUNITY', 'SNIPE' ); + - sql: + sql: | + CREATE TYPE ebay_listing_status AS ENUM ( + 'ACTIVE', + 'BOUGHT_THROUGH_AUCTION', + 'BOUGHT_THROUGH_BUY_IT_NOW', + 'CANCELLED' + ); + - sql: + sql: | + CREATE TYPE ebay_listing_type AS ENUM ( + 'AUCTION', + 'BUY_IT_NOW' + ); + diff --git a/hawk/src/main/resources/db/changelog/changes/0002-create_hawk_flight_and_scrape_run_tables.yaml b/hawk/src/main/resources/db/changelog/changes/0002-create_hawk_flight_and_scrape_run_tables.yaml index 596c9bb..340d669 100644 --- a/hawk/src/main/resources/db/changelog/changes/0002-create_hawk_flight_and_scrape_run_tables.yaml +++ b/hawk/src/main/resources/db/changelog/changes/0002-create_hawk_flight_and_scrape_run_tables.yaml @@ -1,16 +1,12 @@ databaseChangeLog: - changeSet: - id: create_hawk_flight_and_scrape_run_tables + id: 0002-create_hawk_flight_and_scrape_run_tables author: dmitriusan changes: - createSequence: schemaName: merchandise_db sequenceName: hawk_flight_seq startValue: 1 - - createSequence: - schemaName: merchandise_db - sequenceName: hawk_scrape_run_seq - startValue: 1 - createTable: schemaName: merchandise_db tableName: hawk_flight @@ -26,17 +22,16 @@ databaseChangeLog: name: started_at type: TIMESTAMPTZ - column: - name: state - type: SMALLINT + name: status + type: hawk_flight_status - column: name: ended_at type: TIMESTAMPTZ - - addForeignKeyConstraint: - baseTableName: hawk_flight - baseColumnNames: state - referencedTableName: hawk_flight_enum - referencedColumnNames: id - constraintName: fk_hawk_flight_state + + - createSequence: + schemaName: merchandise_db + sequenceName: hawk_scrape_run_seq + startValue: 1 - createTable: schemaName: merchandise_db tableName: hawk_scrape_run diff --git a/hawk/src/main/resources/db/changelog/changes/0003-create_ebay_seller_table.yaml b/hawk/src/main/resources/db/changelog/changes/0003-create_ebay_seller_table.yaml new file mode 100644 index 0000000..986a6be --- /dev/null +++ b/hawk/src/main/resources/db/changelog/changes/0003-create_ebay_seller_table.yaml @@ -0,0 +1,33 @@ +databaseChangeLog: + - changeSet: + id: 0003-create_ebay_seller_table + author: dmitriusan + changes: + # Create ebay_seller table + - createSequence: + schemaName: merchandise_db + sequenceName: ebay_seller_seq + startValue: 1 + - createTable: + schemaName: merchandise_db + tableName: ebay_seller + columns: + - column: + name: id + type: BIGSERIAL + constraints: + nullable: false + primaryKey: true + primaryKeyName: pk_ebay_seller + - column: + name: ebay_id_str + type: VARCHAR + - column: + name: registered_on + type: TIMESTAMPTZ + - column: + name: reputation_percentage + type: FLOAT + - column: + name: feedback_score + type: INTEGER diff --git a/hawk/src/main/resources/db/changelog/changes/0004-create_ebay_finding_table.yaml b/hawk/src/main/resources/db/changelog/changes/0004-create_ebay_finding_table.yaml new file mode 100644 index 0000000..16ab527 --- /dev/null +++ b/hawk/src/main/resources/db/changelog/changes/0004-create_ebay_finding_table.yaml @@ -0,0 +1,88 @@ +databaseChangeLog: + - changeSet: + id: 0004-create_ebay_finding_table + author: dmitriusan + changes: + # Create ebay_finding table + - createSequence: + schemaName: merchandise_db + sequenceName: ebay_finding_seq + startValue: 1 + - createTable: + schemaName: merchandise_db + tableName: ebay_finding + columns: + - column: + name: id + type: BIGSERIAL + constraints: + nullable: false + primaryKey: true + primaryKeyName: pk_ebay_finding + - column: + name: ebay_id_str + type: VARCHAR + constraints: + nullable: false + - column: + name: legacy_ebay_id_str + type: VARCHAR + - column: + name: item_title + type: VARCHAR + constraints: + nullable: false + - column: + name: item_description + type: VARCHAR + - column: + name: current_auction_price_usd + type: NUMERIC(8,2) + - column: + name: final_price_usd + type: NUMERIC(8,2) + - column: + name: min_shipping_usd + type: NUMERIC(8,2) + - column: + name: buy_it_now_price_usd + type: NUMERIC(8,2) + - column: + name: bid_count + type: INTEGER + - column: + name: captured_at + type: TIMESTAMPTZ + constraints: + nullable: false + - column: + name: ends_on + type: TIMESTAMPTZ + constraints: + nullable: false + - column: + name: listing_type + type: ebay_listing_type + constraints: + nullable: false + - column: + name: listing_status + type: ebay_listing_status + constraints: + nullable: false + - column: + name: seller_id + type: BIGINT + constraints: + foreignKeyName: fk_ebay_finding_seller_id + references: ebay_seller(id) + + # Adding foreign key constraints + - addForeignKeyConstraint: + baseColumnNames: seller_id + baseTableName: ebay_finding + baseTableSchemaName: merchandise_db + constraintName: FK_EBAY_FINDING_ON_EBAY_SELLER + referencedColumnNames: id + referencedTableName: ebay_seller + referencedTableSchemaName: merchandise_db \ No newline at end of file diff --git a/hawk/src/main/resources/db/changelog/changes/0005-create_ebay_highlight_table.yaml b/hawk/src/main/resources/db/changelog/changes/0005-create_ebay_highlight_table.yaml new file mode 100644 index 0000000..f126c2b --- /dev/null +++ b/hawk/src/main/resources/db/changelog/changes/0005-create_ebay_highlight_table.yaml @@ -0,0 +1,56 @@ +databaseChangeLog: + - changeSet: + id: 0005-create_ebay_highlight_table + author: dmitriusan + changes: + # Create ebay_highlight table + - createSequence: + schemaName: merchandise_db + sequenceName: ebay_highlight_seq + startValue: 1 + - createTable: + schemaName: merchandise_db + tableName: ebay_highlight + columns: + - column: + name: id + type: BIGSERIAL + constraints: + nullable: false + primaryKey: true + primaryKeyName: pk_ebay_highlight + - column: + name: run_id + type: BIGINT + constraints: + nullable: false + foreignKeyName: fk_ebay_highlight_run_id + references: hawk_scrape_run(id) + - column: + name: ebay_finding_id + type: BIGINT + constraints: + nullable: false + foreignKeyName: fk_ebay_highlight_ebay_finding_id + references: ebay_finding(id) + - column: + name: final_verdict + type: merchandise_verdict_type + + # Adding foreign key constraints + - addForeignKeyConstraint: + baseColumnNames: run_id + baseTableName: ebay_highlight + baseTableSchemaName: merchandise_db + constraintName: FK_EBAY_HIGHLIGHT_ON_SCRAPE_RUN + referencedColumnNames: id + referencedTableName: hawk_scrape_run + referencedTableSchemaName: merchandise_db + - addForeignKeyConstraint: + baseColumnNames: ebay_finding_id + baseTableName: ebay_highlight + baseTableSchemaName: merchandise_db + constraintName: FK_EBAY_HIGHLIGHT_ON_EBAY_FINDING + referencedColumnNames: id + referencedTableName: ebay_finding + referencedTableSchemaName: merchandise_db \ No newline at end of file