Emelyanov535/Price_Pulse
2
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Compare commits

base: Emelyanov535:84f344084ca1b8245b1ad92c9b67a34648b174b1
Branches Tags
Emelyanov535:master
Emelyanov535:feature/parsing-service
Emelyanov535:feature/ozon-parser-v0.1
Emelyanov535:module_analysis
Emelyanov535:wb
..
compare: Emelyanov535:ef2240e8ab49ffc3ac40ef9985e278b17ec7cc33
Branches Tags
Emelyanov535:master
Emelyanov535:feature/parsing-service
Emelyanov535:feature/ozon-parser-v0.1
Emelyanov535:module_analysis
Emelyanov535:wb

No commits in common. "84f344084ca1b8245b1ad92c9b67a34648b174b1" and "ef2240e8ab49ffc3ac40ef9985e278b17ec7cc33" have entirely different histories.
84f344084c ... ef2240e8ab

40 changed files with 140 additions and 383 deletions
Show Stats Expand all files Collapse all files

14
parsing-service/.run/ParsingService [local].run.xml
View File

@ -14,18 +14,4 @@
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="false" name="ParsingService [local]" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<option name="ACTIVE_PROFILES" value="dev" />
<envs>
<env name="JDBC_PASSWORD" value="postgres" />
<env name="JDBC_USERNAME" value="postgres" />
<env name="JDBC_URL" value="localhost:5432/parsed_data" />
<env name="SERVER_PORT" value="8080" />
</envs>
<module name="parsing-service.main" />
<option name="SPRING_BOOT_MAIN_CLASS" value="ru.pricepulse.parsingservice.ParsingServiceApplication" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

1
parsing-service/build.gradle
View File

@ -48,7 +48,6 @@ dependencies {
testImplementation 'org.springframework.kafka:spring-kafka-test'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
implementation 'org.springframework.boot:spring-boot-starter-webflux'
}
tasks.named('test') {

4
parsing-service/src/main/java/ru/pricepulse/parsingservice/config/DateTimeFormatterConfig.java
View File

@ -1,10 +1,10 @@
package ru.pricepulse.parsingservice.config;
import java.time.format.DateTimeFormatter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.time.format.DateTimeFormatter;
@Configuration
public class DateTimeFormatterConfig {

1
parsing-service/src/main/java/ru/pricepulse/parsingservice/config/KafkaConfig.java
View File

@ -1,6 +1,7 @@
package ru.pricepulse.parsingservice.config;
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;

6
parsing-service/src/main/java/ru/pricepulse/parsingservice/config/WebDriverConfig.java
View File

@ -1,5 +1,8 @@
package ru.pricepulse.parsingservice.config;
import java.util.HashMap;
import java.util.Map;
import io.github.bonigarcia.wdm.WebDriverManager;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
@ -10,9 +13,6 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.context.annotation.Scope;
import java.util.HashMap;
import java.util.Map;
@Configuration
public class WebDriverConfig {

4
parsing-service/src/main/java/ru/pricepulse/parsingservice/config/properties/OzonConfigProperties.java
View File

@ -1,11 +1,11 @@
package ru.pricepulse.parsingservice.config.properties;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.List;
@Getter
@Setter
@ConfigurationProperties(prefix = "marketplace.ozon")

5
parsing-service/src/main/java/ru/pricepulse/parsingservice/enumeration/Category.java
View File

@ -1,5 +0,0 @@
package ru.pricepulse.parsingservice.enumeration;
public enum Category {
LAPTOP
}

8
parsing-service/src/main/java/ru/pricepulse/parsingservice/enumeration/Marketplace.java
View File

@ -1,8 +0,0 @@
package ru.pricepulse.parsingservice.enumeration;
public enum Marketplace {
WILDBERRIES,
OZON,
DNS
}

7
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/MarketplacePage.java
View File

@ -1,7 +0,0 @@
package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon;
public interface MarketplacePage {
boolean isLoaded();
}

12
parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/PriceHistoryEntity.java
View File

@ -1,22 +1,20 @@
package ru.pricepulse.parsingservice.persistence.entity;
import java.math.BigDecimal;
import java.util.Objects;
import jakarta.persistence.Column;
import jakarta.persistence.EmbeddedId;
import jakarta.persistence.Entity;
import jakarta.persistence.Table;
import lombok.*;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.proxy.HibernateProxy;
import java.math.BigDecimal;
import java.util.Objects;
@Getter
@Setter
@Entity
@Table(name = "price_history")
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PriceHistoryEntity {
@EmbeddedId

22
parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/PriceHistoryId.java
View File

@ -1,22 +1,22 @@
package ru.pricepulse.parsingservice.persistence.entity;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.hibernate.annotations.OnDelete;
import org.hibernate.annotations.OnDeleteAction;
import org.hibernate.proxy.HibernateProxy;
import java.io.Serializable;
import java.time.OffsetDateTime;
import java.util.Objects;
import jakarta.persistence.Column;
import jakarta.persistence.Embeddable;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.OnDelete;
import org.hibernate.annotations.OnDeleteAction;
import org.hibernate.proxy.HibernateProxy;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Embeddable
public class PriceHistoryId implements Serializable {

31
parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/entity/ProductEntity.java
View File

@ -1,34 +1,39 @@
package ru.pricepulse.parsingservice.persistence.entity;
import jakarta.persistence.*;
import lombok.*;
import org.hibernate.proxy.HibernateProxy;
import ru.pricepulse.parsingservice.enumeration.Category;
import ru.pricepulse.parsingservice.enumeration.Marketplace;
import java.time.LocalDateTime;
import java.util.Objects;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.PrePersist;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.proxy.HibernateProxy;
import ru.pricepulse.parsingservice.persistence.enums.MarketplaceEnum;
@Getter
@Setter
@Entity
@Table(name = "product")
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class ProductEntity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "id", nullable = false)
private Long id;
@Column(name = "marketplace", nullable = false, length = Integer.MAX_VALUE)
@Enumerated(EnumType.STRING)
private Marketplace marketplace;
@Column(name = "marketplace", nullable = false, length = Integer.MAX_VALUE)
private MarketplaceEnum marketplace;
@Column(name = "category", nullable = false, length = Integer.MAX_VALUE)
@Enumerated(EnumType.STRING)
private Category category;
private String category;
@Column(name = "brand", nullable = false, length = Integer.MAX_VALUE)
private String brand;

7
parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/enums/MarketplaceEnum.java Normal file
View File

@ -0,0 +1,7 @@
package ru.pricepulse.parsingservice.persistence.enums;
public enum MarketplaceEnum {
OZON
}

7
parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/repository/ProductPriceRepository.java
View File

@ -1,7 +0,0 @@
package ru.pricepulse.parsingservice.persistence.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryEntity;
public interface ProductPriceRepository extends JpaRepository<PriceHistoryEntity, Long> {
}

7
parsing-service/src/main/java/ru/pricepulse/parsingservice/persistence/repository/ProductRepository.java
View File

@ -1,7 +0,0 @@
package ru.pricepulse.parsingservice.persistence.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
public interface ProductRepository extends JpaRepository<ProductEntity, Long> {
}

14
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/pool/WebDriverPool.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/pool/WebDriverPool.java
View File

@ -1,13 +1,15 @@
package ru.pricepulse.parsingservice.ozon_parser.pool;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.WebDriver;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.stereotype.Component;
package ru.pricepulse.parsingservice.pool;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WebDriverPool {

6
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/DataParser.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/DataParser.java
View File

@ -1,4 +1,4 @@
package ru.pricepulse.parsingservice.ozon_parser.service;
package ru.pricepulse.parsingservice.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -6,8 +6,8 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.ozon_parser.service.dto.ParsedData;
import ru.pricepulse.parsingservice.ozon_parser.service.messaging.ParsedDataProducer;
import ru.pricepulse.parsingservice.service.dto.ParsedData;
import ru.pricepulse.parsingservice.service.messaging.ParsedDataProducer;
@Slf4j
@Service

2
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/MarketplaceParsingService.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/MarketplaceParsingService.java
View File

@ -1,4 +1,4 @@
package ru.pricepulse.parsingservice.ozon_parser.service;
package ru.pricepulse.parsingservice.service;
public interface MarketplaceParsingService {

2
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/PartitionService.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/PartitionService.java
View File

@ -1,4 +1,4 @@
package ru.pricepulse.parsingservice.ozon_parser.service;
package ru.pricepulse.parsingservice.service;
import lombok.RequiredArgsConstructor;
import org.springframework.jdbc.core.JdbcTemplate;

6
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/dto/ParsedData.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/dto/ParsedData.java
View File

@ -1,14 +1,14 @@
package ru.pricepulse.parsingservice.ozon_parser.service.dto;
package ru.pricepulse.parsingservice.service.dto;
import lombok.Getter;
import lombok.Setter;
import ru.pricepulse.parsingservice.enumeration.Marketplace;
import ru.pricepulse.parsingservice.persistence.enums.MarketplaceEnum;
@Getter
@Setter
public class ParsedData {
private Marketplace marketplace;
private MarketplaceEnum marketplace;
private String category;

7
parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/MarketplacePage.java Normal file
View File

@ -0,0 +1,7 @@
package ru.pricepulse.parsingservice.service.marketplace.ozon;
public interface MarketplacePage {
boolean isLoaded();
}

6
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/page/AccessDeniedPage.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/page/AccessDeniedPage.java
View File

@ -1,10 +1,12 @@
package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page;
package ru.pricepulse.parsingservice.service.marketplace.ozon.page;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.MarketplacePage;
import ru.pricepulse.parsingservice.service.marketplace.ozon.MarketplacePage;
@Slf4j
public class AccessDeniedPage implements MarketplacePage {

24
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/page/CategoryPage.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/page/CategoryPage.java
View File

@ -1,18 +1,24 @@
package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page;
package ru.pricepulse.parsingservice.service.marketplace.ozon.page;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.WebDriverWait;
import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.MarketplacePage;
import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElements;
import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfElementLocated;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfAllElements;
import static org.openqa.selenium.support.ui.ExpectedConditions.visibilityOfElementLocated;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.By;
import org.openqa.selenium.StaleElementReferenceException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import ru.pricepulse.parsingservice.service.marketplace.ozon.MarketplacePage;
@Slf4j
public class CategoryPage implements MarketplacePage {

27
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/parsing/CategoryPageParsingService.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/parsing/CategoryPageParsingService.java
View File

@ -1,23 +1,28 @@
package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.slf4j.MDC;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.ozon_parser.pool.WebDriverPool;
import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page.AccessDeniedPage;
import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.page.CategoryPage;
package ru.pricepulse.parsingservice.service.marketplace.ozon.parsing;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.StaleElementReferenceException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.slf4j.MDC;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.pool.WebDriverPool;
import ru.pricepulse.parsingservice.service.marketplace.ozon.page.AccessDeniedPage;
import ru.pricepulse.parsingservice.service.marketplace.ozon.page.CategoryPage;
@Slf4j
@Service
public class CategoryPageParsingService {

23
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/marketplace/ozon/parsing/ParsingService.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/marketplace/ozon/parsing/ParsingService.java
View File

@ -1,14 +1,27 @@
package ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.ozon_parser.service.MarketplaceParsingService;
package ru.pricepulse.parsingservice.service.marketplace.ozon.parsing;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.StaleElementReferenceException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.slf4j.MDC;
import org.springframework.beans.factory.ObjectFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.service.MarketplaceParsingService;
import ru.pricepulse.parsingservice.service.marketplace.ozon.page.AccessDeniedPage;
import ru.pricepulse.parsingservice.service.marketplace.ozon.page.CategoryPage;
@Slf4j
@Service
public class ParsingService implements MarketplaceParsingService {

4
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/messaging/ParsedDataProducer.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/messaging/ParsedDataProducer.java
View File

@ -1,9 +1,9 @@
package ru.pricepulse.parsingservice.ozon_parser.service.messaging;
package ru.pricepulse.parsingservice.service.messaging;
import lombok.RequiredArgsConstructor;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.ozon_parser.service.dto.ParsedData;
import ru.pricepulse.parsingservice.service.dto.ParsedData;
@Service
@RequiredArgsConstructor

2
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/request/PageFetcher.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/request/PageFetcher.java
View File

@ -1,4 +1,4 @@
package ru.pricepulse.parsingservice.ozon_parser.service.request;
package ru.pricepulse.parsingservice.service.request;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

4
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/scheduler/OzonProductUpdater.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/scheduler/OzonProductUpdater.java
View File

@ -1,10 +1,10 @@
package ru.pricepulse.parsingservice.ozon_parser.service.scheduler;
package ru.pricepulse.parsingservice.service.scheduler;
import lombok.RequiredArgsConstructor;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.config.properties.OzonConfigProperties;
import ru.pricepulse.parsingservice.ozon_parser.service.marketplace.ozon.parsing.ParsingService;
import ru.pricepulse.parsingservice.service.marketplace.ozon.parsing.ParsingService;
@Service
@RequiredArgsConstructor

10
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/scheduler/PartitionScheduler.java → parsing-service/src/main/java/ru/pricepulse/parsingservice/service/scheduler/PartitionScheduler.java
View File

@ -1,14 +1,14 @@
package ru.pricepulse.parsingservice.ozon_parser.service.scheduler;
package ru.pricepulse.parsingservice.service.scheduler;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.ozon_parser.service.PartitionService;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import ru.pricepulse.parsingservice.service.PartitionService;
@Slf4j
@Service

4
parsing-service/src/main/java/ru/pricepulse/parsingservice/web/handler/CommonExceptionHandler.java
View File

@ -1,5 +1,7 @@
package ru.pricepulse.parsingservice.web.handler;
import java.net.URI;
import jakarta.persistence.EntityNotFoundException;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.http.HttpStatus;
@ -8,8 +10,6 @@ import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseStatus;
import java.net.URI;
@ControllerAdvice
public class CommonExceptionHandler {

4
parsing-service/src/main/java/ru/pricepulse/parsingservice/web/handler/ErrorResponse.java
View File

@ -1,9 +1,9 @@
package ru.pricepulse.parsingservice.web.handler;
import org.springframework.http.HttpStatus;
import java.net.URI;
import org.springframework.http.HttpStatus;
public record ErrorResponse (
Integer statusCode,
HttpStatus status,

19
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/DebugRunner.java
View File

@ -1,19 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser;
import lombok.AllArgsConstructor;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import ru.pricepulse.parsingservice.wildberries_parser.service.ParsingService;
@Component
@AllArgsConstructor
public class DebugRunner implements CommandLineRunner {
private final ParsingService parsingService;
@Override
public void run(String... args){
System.out.println("Начинаем отладку...");
parsingService.parse();
System.out.println("Заканчиваем отладку...");
}
}

21
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/configuration/WbProperties.java
View File

@ -1,21 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.configuration;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties(prefix = "marketplace.wildberries")
@Getter
@Setter
public class WbProperties {
private String baseUrl;
private String catalogUrl;
private String userAgent;
private String catalogWbUrl;
private int retryAttempts;
private long retryDelay;
private String laptopUrl;
private String shard;
}

18
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/configuration/WebClientConfig.java
View File

@ -1,18 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.configuration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.reactive.function.client.WebClient;
@Configuration
public class WebClientConfig {
@Bean
public WebClient webClient() {
return WebClient.builder()
.defaultHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)")
.codecs(configurer -> configurer
.defaultCodecs()
.maxInMemorySize(10 * 1024 * 1024))
.build();
}
}

25
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/converter/ProductInfoDto2ProductEntity.java
View File

@ -1,25 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.converter;
import org.springframework.core.convert.converter.Converter;
import org.springframework.stereotype.Component;
import ru.pricepulse.parsingservice.enumeration.Category;
import ru.pricepulse.parsingservice.enumeration.Marketplace;
import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
import ru.pricepulse.parsingservice.wildberries_parser.service.dto.ProductInfoDto;
import java.time.LocalDateTime;
@Component
public class ProductInfoDto2ProductEntity implements Converter<ProductInfoDto, ProductEntity> {
@Override
public ProductEntity convert(ProductInfoDto source) {
return ProductEntity.builder()
.marketplace(Marketplace.WILDBERRIES)
.category(Category.LAPTOP)
.brand(source.getBrand())
.productName(source.getName())
.createdAt(LocalDateTime.now())
.build();
}
}

85
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/ParsingService.java
View File

@ -1,85 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.transaction.Transactional;
import lombok.AllArgsConstructor;
import org.springframework.core.convert.ConversionService;
import org.springframework.stereotype.Service;
import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryEntity;
import ru.pricepulse.parsingservice.persistence.entity.PriceHistoryId;
import ru.pricepulse.parsingservice.persistence.entity.ProductEntity;
import ru.pricepulse.parsingservice.persistence.repository.ProductPriceRepository;
import ru.pricepulse.parsingservice.persistence.repository.ProductRepository;
import ru.pricepulse.parsingservice.wildberries_parser.configuration.WbProperties;
import ru.pricepulse.parsingservice.wildberries_parser.service.client.Client;
import ru.pricepulse.parsingservice.wildberries_parser.service.dto.ProductInfoDto;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Service
@AllArgsConstructor
public class ParsingService {
private final Client client;
private final ObjectMapper objectMapper;
private final ConversionService conversionService;
private final ProductRepository productRepository;
private final WbProperties wbProperties;
private final ProductPriceRepository productPriceRepository;
@Transactional
public void parse() {
List<ProductEntity> productEntities = new ArrayList<>();
List<PriceHistoryEntity> priceHistories = new ArrayList<>();
final int elementsInPage = 100;
int page = 1;
Integer totalPages = null;
do {
var pageData = client.scrapPage(page, wbProperties.getShard(), wbProperties.getLaptopUrl());
if (totalPages == null) {
Map<String, Object> dataMap = (Map<String, Object>) pageData.get("data");
int totalElements = (int) dataMap.get("total");
totalPages = (int) Math.ceil((double) totalElements / elementsInPage);
}
List<ProductInfoDto> productInfoDtoList = convertMapObjectToListProductInfoDto(pageData);
productInfoDtoList.forEach(dto -> {
ProductEntity productEntity = conversionService.convert(dto, ProductEntity.class);
PriceHistoryEntity priceHistory = PriceHistoryEntity.builder()
.id(new PriceHistoryId(productEntity, OffsetDateTime.now()))
.price(BigDecimal.valueOf(dto.getSalePriceU() / 100.0))
.build();
productEntities.add(productEntity);
priceHistories.add(priceHistory);
});
page++;
} while (page <= totalPages);
productRepository.saveAll(productEntities);
productPriceRepository.saveAll(priceHistories);
}
private List<ProductInfoDto> convertMapObjectToListProductInfoDto(Map<String, Object> map) {
Map<String, ArrayList<Object>> dataMap = (Map<String, ArrayList<Object>>) map.get("data");
return getProductInfoDtos(dataMap);
}
private List<ProductInfoDto> getProductInfoDtos(Map<String, ArrayList<Object>> dataMap) {
return objectMapper.convertValue(
dataMap.get("products"),
new TypeReference<>() {}
);
}
}

7
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/client/Client.java
View File

@ -1,7 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.service.client;
import java.util.Map;
public interface Client {
Map<String, Object> scrapPage(int page, String shard, String query);
}

33
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/client/ClientImpl.java
View File

@ -1,33 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.service.client;
import lombok.AllArgsConstructor;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import ru.pricepulse.parsingservice.wildberries_parser.configuration.WbProperties;
import java.util.Map;
@AllArgsConstructor
@Service
public class ClientImpl implements Client {
private final WebClient webClient;
private final WbProperties wbProperties;
@Override
public Map<String, Object> scrapPage(int page, String shard, String query) {
String url = wbProperties.getCatalogWbUrl() +
shard +
query +
"?dest=-1257786&page=" + page + "&subject=2290";
return webClient.get()
.uri(url)
.retrieve()
.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
})
.retry(50)
.block();
}
}

20
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/dto/ProductInfoDto.java
View File

@ -1,20 +0,0 @@
package ru.pricepulse.parsingservice.wildberries_parser.service.dto;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class ProductInfoDto {
private Long id;
private String brand;
private String name;
private String supplier;
private Double supplierRating;
private Integer salePriceU;
private Integer reviewRating;
}

14
parsing-service/src/main/resources/application.yml
View File

@ -21,18 +21,6 @@ marketplace:
ozon:
categories-urls:
- https://www.ozon.ru/category/noutbuki-15692
wildberries:
base-url: "https://static-basket-01.wbbasket.ru"
catalog-url: "/vol0/data/main-menu-ru-ru-v3.json"
user-agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)"
catalog-wb-url: "https://catalog.wb.ru/catalog/"
retry-attempts: 5
retry-delay: 1000
shard: "electronic15"
laptop-url: "/catalog"
logging:
pattern:
console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg %X%n"
console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg %X%n"