parsing-service/.run/ParsingService [local].run.xml

@@ -14,18 +14,4 @@
       <option name="Make" enabled="true" />
     </method>
   </configuration>
-  <configuration default="false" name="ParsingService [local]" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
-    <option name="ACTIVE_PROFILES" value="dev" />
-    <envs>
-      <env name="JDBC_PASSWORD" value="postgres" />
-      <env name="JDBC_USERNAME" value="postgres" />
-      <env name="JDBC_URL" value="localhost:5432/parsed_data" />
-      <env name="SERVER_PORT" value="8080" />
-    </envs>
-    <module name="parsing-service.main" />
-    <option name="SPRING_BOOT_MAIN_CLASS" value="ru.pricepulse.parsingservice.ParsingServiceApplication" />
-    <method v="2">
-      <option name="Make" enabled="true" />
-    </method>
-  </configuration>
 </component>

parsing-service/build.gradle

@@ -48,7 +48,6 @@ dependencies {
 	testImplementation 'org.springframework.kafka:spring-kafka-test'
 
 	testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
-	implementation 'org.springframework.boot:spring-boot-starter-webflux'
 }
 
 tasks.named('test') {

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/DateTimeFormatterConfig.java

@@ -1,10 +1,10 @@
 package ru.pricepulse.parsingservice.config;
 
+import java.time.format.DateTimeFormatter;
+
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import java.time.format.DateTimeFormatter;
-
 @Configuration
 public class DateTimeFormatterConfig {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/KafkaConfig.java

@@ -1,6 +1,7 @@
 package ru.pricepulse.parsingservice.config;
 
 import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
+import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/WebDriverConfig.java

@@ -1,5 +1,8 @@
 package ru.pricepulse.parsingservice.config;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import io.github.bonigarcia.wdm.WebDriverManager;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;
@@ -10,9 +13,6 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
 import org.springframework.context.annotation.Scope;
 
-import java.util.HashMap;
-import java.util.Map;
-
 @Configuration
 public class WebDriverConfig {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/config/properties/OzonConfigProperties.java

@@ -1,11 +1,11 @@
 package ru.pricepulse.parsingservice.config.properties;
 
+import java.util.List;
+
 import lombok.Getter;
 import lombok.Setter;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
-import java.util.List;
-
 @Getter
 @Setter
 @ConfigurationProperties(prefix = "marketplace.ozon")

parsing-service/src/main/java/ru/pricepulse/parsingservice/enumeration/Category.java

@@ -1,5 +0,0 @@
-package ru.pricepulse.parsingservice.enumeration;
-
-public enum Category {
-    LAPTOP
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/enumeration/Marketplace.java

@@ -1,8 +0,0 @@
-package ru.pricepulse.parsingservice.enumeration;
-
-public enum Marketplace {
-    WILDBERRIES,
-    OZON,
-    DNS
-}
-

parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/MarketplacePage.java

@@ -1,7 +0,0 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon;
-
-public interface MarketplacePage {
-
-    boolean isLoaded();
-
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/PriceHistoryEntity.java

@@ -1,22 +1,20 @@
 package ru.pricepulse.parsingservice.persistence.entity;
 
+import java.math.BigDecimal;
+import java.util.Objects;
+
 import jakarta.persistence.Column;
 import jakarta.persistence.EmbeddedId;
 import jakarta.persistence.Entity;
 import jakarta.persistence.Table;
-import lombok.*;
+import lombok.Getter;
+import lombok.Setter;
 import org.hibernate.proxy.HibernateProxy;
 
-import java.math.BigDecimal;
-import java.util.Objects;
-
 @Getter
 @Setter
 @Entity
 @Table(name = "price_history")
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
 public class PriceHistoryEntity {
 
     @EmbeddedId

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/PriceHistoryId.java

@@ -1,22 +1,22 @@
 package ru.pricepulse.parsingservice.persistence.entity;
 
-import jakarta.persistence.*;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import org.hibernate.annotations.OnDelete;
-import org.hibernate.annotations.OnDeleteAction;
-import org.hibernate.proxy.HibernateProxy;
-
 import java.io.Serializable;
 import java.time.OffsetDateTime;
 import java.util.Objects;
 
+import jakarta.persistence.Column;
+import jakarta.persistence.Embeddable;
+import jakarta.persistence.FetchType;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
+import lombok.Getter;
+import lombok.Setter;
+import org.hibernate.annotations.OnDelete;
+import org.hibernate.annotations.OnDeleteAction;
+import org.hibernate.proxy.HibernateProxy;
+
 @Getter
 @Setter
-@AllArgsConstructor
-@NoArgsConstructor
 @Embeddable
 public class PriceHistoryId implements Serializable {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/ProductEntity.java

@@ -1,34 +1,39 @@
 package ru.pricepulse.parsingservice.persistence.entity;
 
-import jakarta.persistence.*;
-import lombok.*;
-import org.hibernate.proxy.HibernateProxy;
-import ru.pricepulse.parsingservice.enumeration.Category;
-import ru.pricepulse.parsingservice.enumeration.Marketplace;
-
 import java.time.LocalDateTime;
 import java.util.Objects;
 
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.EnumType;
+import jakarta.persistence.Enumerated;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.PrePersist;
+import jakarta.persistence.Table;
+import lombok.Getter;
+import lombok.Setter;
+import org.hibernate.proxy.HibernateProxy;
+import ru.pricepulse.parsingservice.persistence.enums.MarketplaceEnum;
+
 @Getter
 @Setter
 @Entity
 @Table(name = "product")
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
 public class ProductEntity {
+
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
     @Column(name = "id", nullable = false)
     private Long id;
 
-    @Column(name = "marketplace", nullable = false, length = Integer.MAX_VALUE)
     @Enumerated(EnumType.STRING)
-    private Marketplace marketplace;
+    @Column(name = "marketplace", nullable = false, length = Integer.MAX_VALUE)
+    private MarketplaceEnum marketplace;
 
     @Column(name = "category", nullable = false, length = Integer.MAX_VALUE)
-    @Enumerated(EnumType.STRING)
+    private String category;
-    private Category category;
 
     @Column(name = "brand", nullable = false, length = Integer.MAX_VALUE)
     private String brand;

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/enums/MarketplaceEnum.java

@@ -0,0 +1,7 @@
+package ru.pricepulse.parsingservice.persistence.enums;
+
+public enum MarketplaceEnum {
+
+    OZON
+
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/repository/ProductPriceRepository.java

@@ -1,7 +0,0 @@
-package ru.pricepulse.parsingservice.persistence.repository;
-
-import org.springframework.data.jpa.repository.JpaRepository;
-import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryEntity;
-
-public interface ProductPriceRepository extends JpaRepository<PriceHistoryEntity, Long> {
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/repository/ProductRepository.java

@@ -1,7 +0,0 @@
-package ru.pricepulse.parsingservice.persistence.repository;
-
-import org.springframework.data.jpa.repository.JpaRepository;
-import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
-
-public interface ProductRepository extends JpaRepository<ProductEntity, Long> {
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/pool/WebDriverPool.java

@@ -1,13 +1,15 @@
-package ru.pricepulse.parsingservice.ozon_parser.pool;
+package ru.pricepulse.parsingservice.pool;
-
-import lombok.extern.slf4j.Slf4j;
-import org.openqa.selenium.WebDriver;
-import org.springframework.beans.factory.ObjectFactory;
-import org.springframework.stereotype.Component;
 
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
+import lombok.extern.slf4j.Slf4j;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.stereotype.Component;
+
 @Slf4j
 @Component
 public class WebDriverPool {

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/DataParser.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.ozon_parser.service;
+package ru.pricepulse.parsingservice.service;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -6,8 +6,8 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.ozon_parser.service.dto.ParsedData;
+import ru.pricepulse.parsingservice.service.dto.ParsedData;
-import ru.pricepulse.parsingservice.ozon_parser.service.messaging.ParsedDataProducer;
+import ru.pricepulse.parsingservice.service.messaging.ParsedDataProducer;
 
 @Slf4j
 @Service

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/MarketplaceParsingService.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.ozon_parser.service;
+package ru.pricepulse.parsingservice.service;
 
 public interface MarketplaceParsingService {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/PartitionService.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.ozon_parser.service;
+package ru.pricepulse.parsingservice.service;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.jdbc.core.JdbcTemplate;

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/dto/ParsedData.java

@@ -1,14 +1,14 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.dto;
+package ru.pricepulse.parsingservice.service.dto;
 
 import lombok.Getter;
 import lombok.Setter;
-import ru.pricepulse.parsingservice.enumeration.Marketplace;
+import ru.pricepulse.parsingservice.persistence.enums.MarketplaceEnum;
 
 @Getter
 @Setter
 public class ParsedData {
 
-    private Marketplace marketplace;
+    private MarketplaceEnum marketplace;
 
     private String category;
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/MarketplacePage.java

@@ -0,0 +1,7 @@
+package ru.pricepulse.parsingservice.service.marketplace.ozon;
+
+public interface MarketplacePage {
+
+    boolean isLoaded();
+
+}

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/page/AccessDeniedPage.java

@@ -1,10 +1,12 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page;
+package ru.pricepulse.parsingservice.service.marketplace.ozon.page;
 
 import lombok.extern.slf4j.Slf4j;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.ui.ExpectedConditions;
 import org.openqa.selenium.support.ui.WebDriverWait;
-import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.MarketplacePage;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.MarketplacePage;
 
 @Slf4j
 public class AccessDeniedPage implements MarketplacePage {

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/page/CategoryPage.java

@@ -1,18 +1,24 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page;
+package ru.pricepulse.parsingservice.service.marketplace.ozon.page;
 
-import lombok.extern.slf4j.Slf4j;
+import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElements;
-import org.openqa.selenium.By;
+import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfElementLocated;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
-import org.openqa.selenium.support.ui.WebDriverWait;
-import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.MarketplacePage;
 
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElements;
+import lombok.extern.slf4j.Slf4j;
-import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfElementLocated;
+import org.openqa.selenium.By;
+import org.openqa.selenium.StaleElementReferenceException;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.springframework.retry.annotation.Recover;
+import org.springframework.retry.annotation.Retryable;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.MarketplacePage;
 
 @Slf4j
 public class CategoryPage implements MarketplacePage {

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/parsing/CategoryPageParsingService.java

@@ -1,23 +1,28 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing;
+package ru.pricepulse.parsingservice.service.marketplace.ozon.parsing;
-
-import lombok.extern.slf4j.Slf4j;
-import org.openqa.selenium.support.ui.WebDriverWait;
-import org.slf4j.MDC;
-import org.springframework.retry.annotation.Recover;
-import org.springframework.retry.annotation.Retryable;
-import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.ozon_parser.pool.WebDriverPool;
-import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page.AccessDeniedPage;
-import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page.CategoryPage;
 
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import lombok.extern.slf4j.Slf4j;
+import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.StaleElementReferenceException;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.slf4j.MDC;
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.retry.annotation.Recover;
+import org.springframework.retry.annotation.Retryable;
+import org.springframework.stereotype.Service;
+import ru.pricepulse.parsingservice.pool.WebDriverPool;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.page.AccessDeniedPage;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.page.CategoryPage;
+
 @Slf4j
 @Service
 public class CategoryPageParsingService {

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/parsing/ParsingService.java

@@ -1,14 +1,27 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing;
+package ru.pricepulse.parsingservice.service.marketplace.ozon.parsing;
-
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.ozon_parser.service.MarketplaceParsingService;
 
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import lombok.extern.slf4j.Slf4j;
+import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.StaleElementReferenceException;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.slf4j.MDC;
+import org.springframework.beans.factory.ObjectFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.retry.annotation.Retryable;
+import org.springframework.stereotype.Service;
+import ru.pricepulse.parsingservice.service.MarketplaceParsingService;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.page.AccessDeniedPage;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.page.CategoryPage;
+
 @Slf4j
 @Service
 public class ParsingService implements MarketplaceParsingService {

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/messaging/ParsedDataProducer.java

@@ -1,9 +1,9 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.messaging;
+package ru.pricepulse.parsingservice.service.messaging;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.ozon_parser.service.dto.ParsedData;
+import ru.pricepulse.parsingservice.service.dto.ParsedData;
 
 @Service
 @RequiredArgsConstructor

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/request/PageFetcher.java

@@ -1,4 +1,4 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.request;
+package ru.pricepulse.parsingservice.service.request;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/scheduler/OzonProductUpdater.java

@@ -1,10 +1,10 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.scheduler;
+package ru.pricepulse.parsingservice.service.scheduler;
 
 import lombok.RequiredArgsConstructor;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import ru.pricepulse.parsingservice.config.properties.OzonConfigProperties;
-import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing.ParsingService;
+import ru.pricepulse.parsingservice.service.marketplace.ozon.parsing.ParsingService;
 
 @Service
 @RequiredArgsConstructor

parsing-service/src/main/java/ru/pricepulse/parsingservice/service/scheduler/PartitionScheduler.java

@@ -1,14 +1,14 @@
-package ru.pricepulse.parsingservice.ozon_parser.service.scheduler;
+package ru.pricepulse.parsingservice.service.scheduler;
+
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
 
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.ozon_parser.service.PartitionService;
+import ru.pricepulse.parsingservice.service.PartitionService;
-
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
 
 @Slf4j
 @Service

parsing-service/src/main/java/ru/pricepulse/parsingservice/web/handler/CommonExceptionHandler.java

@@ -1,5 +1,7 @@
 package ru.pricepulse.parsingservice.web.handler;
 
+import java.net.URI;
+
 import jakarta.persistence.EntityNotFoundException;
 import jakarta.servlet.http.HttpServletRequest;
 import org.springframework.http.HttpStatus;
@@ -8,8 +10,6 @@ import org.springframework.web.bind.annotation.ControllerAdvice;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseStatus;
 
-import java.net.URI;
-
 @ControllerAdvice
 public class CommonExceptionHandler {
 

parsing-service/src/main/java/ru/pricepulse/parsingservice/web/handler/ErrorResponse.java

@@ -1,9 +1,9 @@
 package ru.pricepulse.parsingservice.web.handler;
 
-import org.springframework.http.HttpStatus;
-
 import java.net.URI;
 
+import org.springframework.http.HttpStatus;
+
 public record ErrorResponse (
     Integer statusCode,
     HttpStatus status,

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/DebugRunner.java

@@ -1,19 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser;
-
-import lombok.AllArgsConstructor;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.stereotype.Component;
-import ru.pricepulse.parsingservice.wildberries_parser.service.ParsingService;
-
-@Component
-@AllArgsConstructor
-public class DebugRunner implements CommandLineRunner {
-    private final ParsingService parsingService;
-
-    @Override
-    public void run(String... args){
-        System.out.println("Начинаем отладку...");
-        parsingService.parse();
-        System.out.println("Заканчиваем отладку...");
-    }
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/configuration/WbProperties.java

@@ -1,21 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.configuration;
-
-import lombok.Getter;
-import lombok.Setter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties(prefix = "marketplace.wildberries")
-@Getter
-@Setter
-public class WbProperties {
-    private String baseUrl;
-    private String catalogUrl;
-    private String userAgent;
-    private String catalogWbUrl;
-    private int retryAttempts;
-    private long retryDelay;
-    private String laptopUrl;
-    private String shard;
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/configuration/WebClientConfig.java

@@ -1,18 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.configuration;
-
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.web.reactive.function.client.WebClient;
-
-@Configuration
-public class WebClientConfig {
-    @Bean
-    public WebClient webClient() {
-        return WebClient.builder()
-                .defaultHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)")
-                .codecs(configurer -> configurer
-                        .defaultCodecs()
-                        .maxInMemorySize(10 * 1024 * 1024))
-                .build();
-    }
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/converter/ProductInfoDto2ProductEntity.java

@@ -1,25 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.converter;
-
-import org.springframework.core.convert.converter.Converter;
-import org.springframework.stereotype.Component;
-import ru.pricepulse.parsingservice.enumeration.Category;
-import ru.pricepulse.parsingservice.enumeration.Marketplace;
-import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
-import ru.pricepulse.parsingservice.wildberries_parser.service.dto.ProductInfoDto;
-
-import java.time.LocalDateTime;
-
-
-@Component
-public class ProductInfoDto2ProductEntity implements Converter<ProductInfoDto, ProductEntity> {
-    @Override
-    public ProductEntity convert(ProductInfoDto source) {
-        return ProductEntity.builder()
-                .marketplace(Marketplace.WILDBERRIES)
-                .category(Category.LAPTOP)
-                .brand(source.getBrand())
-                .productName(source.getName())
-                .createdAt(LocalDateTime.now())
-                .build();
-    }
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/ParsingService.java

@@ -1,85 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.service;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.transaction.Transactional;
-import lombok.AllArgsConstructor;
-import org.springframework.core.convert.ConversionService;
-import org.springframework.stereotype.Service;
-import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryEntity;
-import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryId;
-import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
-import ru.pricepulse.parsingservice.persistence.repository.ProductPriceRepository;
-import ru.pricepulse.parsingservice.persistence.repository.ProductRepository;
-import ru.pricepulse.parsingservice.wildberries_parser.configuration.WbProperties;
-import ru.pricepulse.parsingservice.wildberries_parser.service.client.Client;
-import ru.pricepulse.parsingservice.wildberries_parser.service.dto.ProductInfoDto;
-
-import java.math.BigDecimal;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-@Service
-@AllArgsConstructor
-public class ParsingService {
-    private final Client client;
-    private final ObjectMapper objectMapper;
-    private final ConversionService conversionService;
-    private final ProductRepository productRepository;
-    private final WbProperties wbProperties;
-    private final ProductPriceRepository productPriceRepository;
-
-    @Transactional
-    public void parse() {
-        List<ProductEntity> productEntities = new ArrayList<>();
-        List<PriceHistoryEntity> priceHistories = new ArrayList<>();
-
-        final int elementsInPage = 100;
-        int page = 1;
-        Integer totalPages = null;
-
-        do {
-            var pageData = client.scrapPage(page, wbProperties.getShard(), wbProperties.getLaptopUrl());
-
-            if (totalPages == null) {
-                Map<String, Object> dataMap = (Map<String, Object>) pageData.get("data");
-                int totalElements = (int) dataMap.get("total");
-                totalPages = (int) Math.ceil((double) totalElements / elementsInPage);
-            }
-
-            List<ProductInfoDto> productInfoDtoList = convertMapObjectToListProductInfoDto(pageData);
-
-            productInfoDtoList.forEach(dto -> {
-
-                ProductEntity productEntity = conversionService.convert(dto, ProductEntity.class);
-
-                PriceHistoryEntity priceHistory = PriceHistoryEntity.builder()
-                        .id(new PriceHistoryId(productEntity, OffsetDateTime.now()))
-                        .price(BigDecimal.valueOf(dto.getSalePriceU() / 100.0))
-                        .build();
-
-                productEntities.add(productEntity);
-                priceHistories.add(priceHistory);
-            });
-
-            page++;
-        } while (page <= totalPages);
-
-        productRepository.saveAll(productEntities);
-        productPriceRepository.saveAll(priceHistories);
-    }
-
-    private List<ProductInfoDto> convertMapObjectToListProductInfoDto(Map<String, Object> map) {
-        Map<String, ArrayList<Object>> dataMap = (Map<String, ArrayList<Object>>) map.get("data");
-        return getProductInfoDtos(dataMap);
-    }
-
-    private List<ProductInfoDto> getProductInfoDtos(Map<String, ArrayList<Object>> dataMap) {
-        return objectMapper.convertValue(
-                dataMap.get("products"),
-                new TypeReference<>() {}
-        );
-    }
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/client/Client.java

@@ -1,7 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.service.client;
-
-import java.util.Map;
-
-public interface Client {
-    Map<String, Object> scrapPage(int page, String shard, String query);
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/client/ClientImpl.java

@@ -1,33 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.service.client;
-
-import lombok.AllArgsConstructor;
-import org.springframework.core.ParameterizedTypeReference;
-import org.springframework.stereotype.Service;
-import org.springframework.web.reactive.function.client.WebClient;
-import ru.pricepulse.parsingservice.wildberries_parser.configuration.WbProperties;
-
-import java.util.Map;
-
-@AllArgsConstructor
-@Service
-public class ClientImpl implements Client {
-
-    private final WebClient webClient;
-    private final WbProperties wbProperties;
-
-    @Override
-    public Map<String, Object> scrapPage(int page, String shard, String query) {
-        String url = wbProperties.getCatalogWbUrl() +
-                shard +
-                query +
-                "?dest=-1257786&page=" + page + "&subject=2290";
-
-        return webClient.get()
-                .uri(url)
-                .retrieve()
-                .bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
-                })
-                .retry(50)
-                .block();
-    }
-}

parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/dto/ProductInfoDto.java

@@ -1,20 +0,0 @@
-package ru.pricepulse.parsingservice.wildberries_parser.service.dto;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
-public class ProductInfoDto {
-    private Long id;
-    private String brand;
-    private String name;
-    private String supplier;
-    private Double supplierRating;
-    private Integer salePriceU;
-    private Integer reviewRating;
-}

parsing-service/src/main/resources/application.yml

@@ -21,18 +21,6 @@ marketplace:
   ozon:
     categories-urls:
       - https://www.ozon.ru/category/noutbuki-15692
-  wildberries:
-    base-url: "https://static-basket-01.wbbasket.ru"
-    catalog-url: "/vol0/data/main-menu-ru-ru-v3.json"
-    user-agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)"
-    catalog-wb-url: "https://catalog.wb.ru/catalog/"
-    retry-attempts: 5
-    retry-delay: 1000
-    shard: "electronic15"
-    laptop-url: "/catalog"
-
 logging:
   pattern:
-    console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg %X%n"
+    console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg %X%n"
-
-