merge branches

parsing-service/.run/ParsingService [local].run.xml

@@ -14,4 +14,18 @@
       <option name="Make" enabled="true" />
     </method>
   </configuration>
+  <configuration default="false" name="ParsingService [local]" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
+    <option name="ACTIVE_PROFILES" value="dev" />
+    <envs>
+      <env name="JDBC_PASSWORD" value="postgres" />
+      <env name="JDBC_USERNAME" value="postgres" />
+      <env name="JDBC_URL" value="localhost:5432/parsed_data" />
+      <env name="SERVER_PORT" value="8080" />
+    </envs>
+    <module name="parsing-service.main" />
+    <option name="SPRING_BOOT_MAIN_CLASS" value="ru.pricepulse.parsingservice.ParsingServiceApplication" />
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
 </component>

parsing-service/build.gradle

@@ -48,6 +48,7 @@ dependencies {
 	testImplementation 'org.springframework.kafka:spring-kafka-test'
 
 	testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+	implementation 'org.springframework.boot:spring-boot-starter-webflux'
 }
 
 tasks.named('test') {

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/DateTimeFormatterConfig.java

@@ -1,10 +1,10 @@
 package ru.pricepulse.parsingservice.config;
 
-import java.time.format.DateTimeFormatter;
-
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import java.time.format.DateTimeFormatter;
+
 @Configuration
 public class DateTimeFormatterConfig {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/KafkaConfig.java

@@ -1,7 +1,6 @@
 package ru.pricepulse.parsingservice.config;
 
 import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
-import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/WebDriverConfig.java

@@ -1,8 +1,5 @@
 package ru.pricepulse.parsingservice.config;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import io.github.bonigarcia.wdm.WebDriverManager;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;
@@ -13,6 +10,9 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
 import org.springframework.context.annotation.Scope;
 
+import java.util.HashMap;
+import java.util.Map;
+
 @Configuration
 public class WebDriverConfig {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/properties/OzonConfigProperties.java

@@ -1,11 +1,11 @@
 package ru.pricepulse.parsingservice.config.properties;
 
-import java.util.List;
-
 import lombok.Getter;
 import lombok.Setter;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
+import java.util.List;
+
 @Getter
 @Setter
 @ConfigurationProperties(prefix = "marketplace.ozon")

parsing-service/src/main/java/ru/pricepulse/parsingservice/enumeration/Category.java

@@ -0,0 +1,5 @@
+package ru.pricepulse.parsingservice.enumeration;
+
+public enum Category {
+    LAPTOP
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/enumeration/Marketplace.java

@@ -0,0 +1,8 @@
+package ru.pricepulse.parsingservice.enumeration;
+
+public enum Marketplace {
+    WILDBERRIES,
+    OZON,
+    DNS
+}
+

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/pool/WebDriverPool.java

@@ -1,15 +1,13 @@
-package ru.pricepulse.parsingservice.pool;
+package ru.pricepulse.parsingservice.ozon_parser.pool;
-
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
 
 import lombok.extern.slf4j.Slf4j;
 import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.chrome.ChromeOptions;
 import org.springframework.beans.factory.ObjectFactory;
 import org.springframework.stereotype.Component;
 
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
 @Slf4j
 @Component
 public class WebDriverPool {

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/DataParser.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.service;
+package ru.pricepulse.parsingservice.ozon_parser.service;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -6,8 +6,8 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.service.dto.ParsedData;
+import ru.pricepulse.parsingservice.ozon_parser.service.dto.ParsedData;
-import ru.pricepulse.parsingservice.service.messaging.ParsedDataProducer;
+import ru.pricepulse.parsingservice.ozon_parser.service.messaging.ParsedDataProducer;
 
 @Slf4j
 @Service

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/MarketplaceParsingService.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.service;
+package ru.pricepulse.parsingservice.ozon_parser.service;
 
 public interface MarketplaceParsingService {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/PartitionService.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.service;
+package ru.pricepulse.parsingservice.ozon_parser.service;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.jdbc.core.JdbcTemplate;

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/dto/ParsedData.java

@@ -1,14 +1,14 @@
-package ru.pricepulse.parsingservice.service.dto;
+package ru.pricepulse.parsingservice.ozon_parser.service.dto;
 
 import lombok.Getter;
 import lombok.Setter;
-import ru.pricepulse.parsingservice.persistence.enums.MarketplaceEnum;
+import ru.pricepulse.parsingservice.enumeration.Marketplace;
 
 @Getter
 @Setter
 public class ParsedData {
 
-    private MarketplaceEnum marketplace;
+    private Marketplace marketplace;
 
     private String category;
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/MarketplacePage.java

@@ -0,0 +1,7 @@
+package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon;
+
+public interface MarketplacePage {
+
+    boolean isLoaded();
+
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/page/AccessDeniedPage.java

@@ -1,12 +1,10 @@
-package ru.pricepulse.parsingservice.service.marketplace.ozon.page;
+package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page;
 
 import lombok.extern.slf4j.Slf4j;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
-import org.openqa.selenium.support.ui.ExpectedConditions;
 import org.openqa.selenium.support.ui.WebDriverWait;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.MarketplacePage;
+import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.MarketplacePage;
 
 @Slf4j
 public class AccessDeniedPage implements MarketplacePage {

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/page/CategoryPage.java

@@ -1,24 +1,18 @@
-package ru.pricepulse.parsingservice.service.marketplace.ozon.page;
+package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page;
 
-import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElements;
+import lombok.extern.slf4j.Slf4j;
-import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfElementLocated;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.MarketplacePage;
 
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import lombok.extern.slf4j.Slf4j;
+import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElements;
-import org.openqa.selenium.By;
+import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfElementLocated;
-import org.openqa.selenium.StaleElementReferenceException;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
-import org.openqa.selenium.support.ui.ExpectedConditions;
-import org.openqa.selenium.support.ui.WebDriverWait;
-import org.springframework.retry.annotation.Recover;
-import org.springframework.retry.annotation.Retryable;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.MarketplacePage;
 
 @Slf4j
 public class CategoryPage implements MarketplacePage {

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/parsing/CategoryPageParsingService.java

@@ -1,28 +1,23 @@
-package ru.pricepulse.parsingservice.service.marketplace.ozon.parsing;
+package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing;
+
+import lombok.extern.slf4j.Slf4j;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.slf4j.MDC;
+import org.springframework.retry.annotation.Recover;
+import org.springframework.retry.annotation.Retryable;
+import org.springframework.stereotype.Service;
+import ru.pricepulse.parsingservice.ozon_parser.pool.WebDriverPool;
+import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page.AccessDeniedPage;
+import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page.CategoryPage;
 
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-import lombok.extern.slf4j.Slf4j;
-import org.openqa.selenium.JavascriptExecutor;
-import org.openqa.selenium.StaleElementReferenceException;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.support.ui.WebDriverWait;
-import org.slf4j.MDC;
-import org.springframework.beans.factory.ObjectFactory;
-import org.springframework.retry.annotation.Recover;
-import org.springframework.retry.annotation.Retryable;
-import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.pool.WebDriverPool;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.page.AccessDeniedPage;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.page.CategoryPage;
-
 @Slf4j
 @Service
 public class CategoryPageParsingService {

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/parsing/ParsingService.java

@@ -1,27 +1,14 @@
-package ru.pricepulse.parsingservice.service.marketplace.ozon.parsing;
+package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import ru.pricepulse.parsingservice.ozon_parser.service.MarketplaceParsingService;
 
-import java.time.Duration;
-import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import lombok.extern.slf4j.Slf4j;
-import org.openqa.selenium.JavascriptExecutor;
-import org.openqa.selenium.StaleElementReferenceException;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.support.ui.WebDriverWait;
-import org.slf4j.MDC;
-import org.springframework.beans.factory.ObjectFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.retry.annotation.Retryable;
-import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.service.MarketplaceParsingService;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.page.AccessDeniedPage;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.page.CategoryPage;
-
 @Slf4j
 @Service
 public class ParsingService implements MarketplaceParsingService {

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/messaging/ParsedDataProducer.java

@@ -1,9 +1,9 @@
-package ru.pricepulse.parsingservice.service.messaging;
+package ru.pricepulse.parsingservice.ozon_parser.service.messaging;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.service.dto.ParsedData;
+import ru.pricepulse.parsingservice.ozon_parser.service.dto.ParsedData;
 
 @Service
 @RequiredArgsConstructor

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/request/PageFetcher.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.service.request;
+package ru.pricepulse.parsingservice.ozon_parser.service.request;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/scheduler/OzonProductUpdater.java

@@ -1,10 +1,10 @@
-package ru.pricepulse.parsingservice.service.scheduler;
+package ru.pricepulse.parsingservice.ozon_parser.service.scheduler;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import ru.pricepulse.parsingservice.config.properties.OzonConfigProperties;
-import ru.pricepulse.parsingservice.service.marketplace.ozon.parsing.ParsingService;
+import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing.ParsingService;
 
 @Service
 @RequiredArgsConstructor

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/scheduler/PartitionScheduler.java

@@ -1,14 +1,14 @@
-package ru.pricepulse.parsingservice.service.scheduler;
+package ru.pricepulse.parsingservice.ozon_parser.service.scheduler;
-
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
 
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.service.PartitionService;
+import ru.pricepulse.parsingservice.ozon_parser.service.PartitionService;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
 
 @Slf4j
 @Service

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/PriceHistoryEntity.java

@@ -1,20 +1,22 @@
 package ru.pricepulse.parsingservice.persistence.entity;
 
-import java.math.BigDecimal;
-import java.util.Objects;
-
 import jakarta.persistence.Column;
 import jakarta.persistence.EmbeddedId;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Table;
-import lombok.Getter;
+import lombok.*;
-import lombok.Setter;
 import org.hibernate.proxy.HibernateProxy;
 
+import java.math.BigDecimal;
+import java.util.Objects;
+
 @Getter
 @Setter
 @Entity
 @Table(name = "price_history")
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
 public class PriceHistoryEntity {
 
     @EmbeddedId

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/PriceHistoryId.java

@@ -1,22 +1,22 @@
 package ru.pricepulse.parsingservice.persistence.entity;
 
-import java.io.Serializable;
+import jakarta.persistence.*;
-import java.time.OffsetDateTime;
+import lombok.AllArgsConstructor;
-import java.util.Objects;
-
-import jakarta.persistence.Column;
-import jakarta.persistence.Embeddable;
-import jakarta.persistence.FetchType;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.hibernate.annotations.OnDelete;
 import org.hibernate.annotations.OnDeleteAction;
 import org.hibernate.proxy.HibernateProxy;
 
+import java.io.Serializable;
+import java.time.OffsetDateTime;
+import java.util.Objects;
+
 @Getter
 @Setter
+@AllArgsConstructor
+@NoArgsConstructor
 @Embeddable
 public class PriceHistoryId implements Serializable {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/ProductEntity.java

@@ -1,39 +1,34 @@
 package ru.pricepulse.parsingservice.persistence.entity;
 
+import jakarta.persistence.*;
+import lombok.*;
+import org.hibernate.proxy.HibernateProxy;
+import ru.pricepulse.parsingservice.enumeration.Category;
+import ru.pricepulse.parsingservice.enumeration.Marketplace;
+
 import java.time.LocalDateTime;
 import java.util.Objects;
 
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.EnumType;
-import jakarta.persistence.Enumerated;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.PrePersist;
-import jakarta.persistence.Table;
-import lombok.Getter;
-import lombok.Setter;
-import org.hibernate.proxy.HibernateProxy;
-import ru.pricepulse.parsingservice.persistence.enums.MarketplaceEnum;
-
 @Getter
 @Setter
 @Entity
 @Table(name = "product")
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
 public class ProductEntity {
-
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
     @Column(name = "id", nullable = false)
     private Long id;
 
-    @Enumerated(EnumType.STRING)
     @Column(name = "marketplace", nullable = false, length = Integer.MAX_VALUE)
-    private MarketplaceEnum marketplace;
+    @Enumerated(EnumType.STRING)
+    private Marketplace marketplace;
 
     @Column(name = "category", nullable = false, length = Integer.MAX_VALUE)
-    private String category;
+    @Enumerated(EnumType.STRING)
+    private Category category;
 
     @Column(name = "brand", nullable = false, length = Integer.MAX_VALUE)
     private String brand;

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/enums/MarketplaceEnum.java

@@ -1,7 +0,0 @@
-package ru.pricepulse.parsingservice.persistence.enums;
-
-public enum MarketplaceEnum {
-
-    OZON
-
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/repository/ProductPriceRepository.java

@@ -0,0 +1,7 @@
+package ru.pricepulse.parsingservice.persistence.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryEntity;
+
+public interface ProductPriceRepository extends JpaRepository<PriceHistoryEntity, Long> {
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/repository/ProductRepository.java

@@ -0,0 +1,7 @@
+package ru.pricepulse.parsingservice.persistence.repository;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
+
+public interface ProductRepository extends JpaRepository<ProductEntity, Long> {
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/MarketplacePage.java

@@ -1,7 +0,0 @@
-package ru.pricepulse.parsingservice.service.marketplace.ozon;
-
-public interface MarketplacePage {
-
-    boolean isLoaded();
-
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/web/handler/CommonExceptionHandler.java

@@ -1,7 +1,5 @@
 package ru.pricepulse.parsingservice.web.handler;
 
-import java.net.URI;
-
 import jakarta.persistence.EntityNotFoundException;
 import jakarta.servlet.http.HttpServletRequest;
 import org.springframework.http.HttpStatus;
@@ -10,6 +8,8 @@ import org.springframework.web.bind.annotation.ControllerAdvice;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
+import java.net.URI;
+
 @ControllerAdvice
 public class CommonExceptionHandler {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/web/handler/ErrorResponse.java

@@ -1,9 +1,9 @@
 package ru.pricepulse.parsingservice.web.handler;
 
-import java.net.URI;
-
 import org.springframework.http.HttpStatus;
 
+import java.net.URI;
+
 public record ErrorResponse (
     Integer statusCode,
     HttpStatus status,

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/DebugRunner.java

@@ -0,0 +1,19 @@
+package ru.pricepulse.parsingservice.wildberries_parser;
+
+import lombok.AllArgsConstructor;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.stereotype.Component;
+import ru.pricepulse.parsingservice.wildberries_parser.service.ParsingService;
+
+@Component
+@AllArgsConstructor
+public class DebugRunner implements CommandLineRunner {
+    private final ParsingService parsingService;
+
+    @Override
+    public void run(String... args){
+        System.out.println("Начинаем отладку...");
+        parsingService.parse();
+        System.out.println("Заканчиваем отладку...");
+    }
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/configuration/WbProperties.java

@@ -0,0 +1,21 @@
+package ru.pricepulse.parsingservice.wildberries_parser.configuration;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties(prefix = "marketplace.wildberries")
+@Getter
+@Setter
+public class WbProperties {
+    private String baseUrl;
+    private String catalogUrl;
+    private String userAgent;
+    private String catalogWbUrl;
+    private int retryAttempts;
+    private long retryDelay;
+    private String laptopUrl;
+    private String shard;
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/configuration/WebClientConfig.java

@@ -0,0 +1,18 @@
+package ru.pricepulse.parsingservice.wildberries_parser.configuration;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.reactive.function.client.WebClient;
+
+@Configuration
+public class WebClientConfig {
+    @Bean
+    public WebClient webClient() {
+        return WebClient.builder()
+                .defaultHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)")
+                .codecs(configurer -> configurer
+                        .defaultCodecs()
+                        .maxInMemorySize(10 * 1024 * 1024))
+                .build();
+    }
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/converter/ProductInfoDto2ProductEntity.java

@@ -0,0 +1,25 @@
+package ru.pricepulse.parsingservice.wildberries_parser.converter;
+
+import org.springframework.core.convert.converter.Converter;
+import org.springframework.stereotype.Component;
+import ru.pricepulse.parsingservice.enumeration.Category;
+import ru.pricepulse.parsingservice.enumeration.Marketplace;
+import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
+import ru.pricepulse.parsingservice.wildberries_parser.service.dto.ProductInfoDto;
+
+import java.time.LocalDateTime;
+
+
+@Component
+public class ProductInfoDto2ProductEntity implements Converter<ProductInfoDto, ProductEntity> {
+    @Override
+    public ProductEntity convert(ProductInfoDto source) {
+        return ProductEntity.builder()
+                .marketplace(Marketplace.WILDBERRIES)
+                .category(Category.LAPTOP)
+                .brand(source.getBrand())
+                .productName(source.getName())
+                .createdAt(LocalDateTime.now())
+                .build();
+    }
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/ParsingService.java

@@ -0,0 +1,85 @@
+package ru.pricepulse.parsingservice.wildberries_parser.service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import jakarta.transaction.Transactional;
+import lombok.AllArgsConstructor;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.stereotype.Service;
+import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryEntity;
+import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryId;
+import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
+import ru.pricepulse.parsingservice.persistence.repository.ProductPriceRepository;
+import ru.pricepulse.parsingservice.persistence.repository.ProductRepository;
+import ru.pricepulse.parsingservice.wildberries_parser.configuration.WbProperties;
+import ru.pricepulse.parsingservice.wildberries_parser.service.client.Client;
+import ru.pricepulse.parsingservice.wildberries_parser.service.dto.ProductInfoDto;
+
+import java.math.BigDecimal;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@AllArgsConstructor
+public class ParsingService {
+    private final Client client;
+    private final ObjectMapper objectMapper;
+    private final ConversionService conversionService;
+    private final ProductRepository productRepository;
+    private final WbProperties wbProperties;
+    private final ProductPriceRepository productPriceRepository;
+
+    @Transactional
+    public void parse() {
+        List<ProductEntity> productEntities = new ArrayList<>();
+        List<PriceHistoryEntity> priceHistories = new ArrayList<>();
+
+        final int elementsInPage = 100;
+        int page = 1;
+        Integer totalPages = null;
+
+        do {
+            var pageData = client.scrapPage(page, wbProperties.getShard(), wbProperties.getLaptopUrl());
+
+            if (totalPages == null) {
+                Map<String, Object> dataMap = (Map<String, Object>) pageData.get("data");
+                int totalElements = (int) dataMap.get("total");
+                totalPages = (int) Math.ceil((double) totalElements / elementsInPage);
+            }
+
+            List<ProductInfoDto> productInfoDtoList = convertMapObjectToListProductInfoDto(pageData);
+
+            productInfoDtoList.forEach(dto -> {
+
+                ProductEntity productEntity = conversionService.convert(dto, ProductEntity.class);
+
+                PriceHistoryEntity priceHistory = PriceHistoryEntity.builder()
+                        .id(new PriceHistoryId(productEntity, OffsetDateTime.now()))
+                        .price(BigDecimal.valueOf(dto.getSalePriceU() / 100.0))
+                        .build();
+
+                productEntities.add(productEntity);
+                priceHistories.add(priceHistory);
+            });
+
+            page++;
+        } while (page <= totalPages);
+
+        productRepository.saveAll(productEntities);
+        productPriceRepository.saveAll(priceHistories);
+    }
+
+    private List<ProductInfoDto> convertMapObjectToListProductInfoDto(Map<String, Object> map) {
+        Map<String, ArrayList<Object>> dataMap = (Map<String, ArrayList<Object>>) map.get("data");
+        return getProductInfoDtos(dataMap);
+    }
+
+    private List<ProductInfoDto> getProductInfoDtos(Map<String, ArrayList<Object>> dataMap) {
+        return objectMapper.convertValue(
+                dataMap.get("products"),
+                new TypeReference<>() {}
+        );
+    }
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/client/Client.java

@@ -0,0 +1,7 @@
+package ru.pricepulse.parsingservice.wildberries_parser.service.client;
+
+import java.util.Map;
+
+public interface Client {
+    Map<String, Object> scrapPage(int page, String shard, String query);
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/client/ClientImpl.java

@@ -0,0 +1,33 @@
+package ru.pricepulse.parsingservice.wildberries_parser.service.client;
+
+import lombok.AllArgsConstructor;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.stereotype.Service;
+import org.springframework.web.reactive.function.client.WebClient;
+import ru.pricepulse.parsingservice.wildberries_parser.configuration.WbProperties;
+
+import java.util.Map;
+
+@AllArgsConstructor
+@Service
+public class ClientImpl implements Client {
+
+    private final WebClient webClient;
+    private final WbProperties wbProperties;
+
+    @Override
+    public Map<String, Object> scrapPage(int page, String shard, String query) {
+        String url = wbProperties.getCatalogWbUrl() +
+                shard +
+                query +
+                "?dest=-1257786&page=" + page + "&subject=2290";
+
+        return webClient.get()
+                .uri(url)
+                .retrieve()
+                .bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
+                })
+                .retry(50)
+                .block();
+    }
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/dto/ProductInfoDto.java

@@ -0,0 +1,20 @@
+package ru.pricepulse.parsingservice.wildberries_parser.service.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class ProductInfoDto {
+    private Long id;
+    private String brand;
+    private String name;
+    private String supplier;
+    private Double supplierRating;
+    private Integer salePriceU;
+    private Integer reviewRating;
+}

parsing-service/src/main/resources/application.yml

@@ -21,6 +21,18 @@ marketplace:
   ozon:
     categories-urls:
       - https://www.ozon.ru/category/noutbuki-15692
+  wildberries:
+    base-url: "https://static-basket-01.wbbasket.ru"
+    catalog-url: "/vol0/data/main-menu-ru-ru-v3.json"
+    user-agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)"
+    catalog-wb-url: "https://catalog.wb.ru/catalog/"
+    retry-attempts: 5
+    retry-delay: 1000
+    shard: "electronic15"
+    laptop-url: "/catalog"
+
 logging:
   pattern:
     console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg %X%n"
+
+