From 171cc650f1c6258186b9fd9e4ec0e8b5de6ff369 Mon Sep 17 00:00:00 2001 From: "danil.markov" Date: Tue, 12 Nov 2024 15:40:00 +0400 Subject: [PATCH] =?UTF-8?q?Feature/ozon-parser-v0.1:=20=D0=92=D0=B5=D1=80?= =?UTF-8?q?=D1=81=D0=B8=D1=8F=20=D1=80=D0=B0=D0=B1=D0=BE=D1=87=D0=B0=D1=8F?= =?UTF-8?q?,=20=D0=BF=D0=B0=D1=80=D1=81=D0=B8=D1=82=20=D0=B1=D1=8B=D1=81?= =?UTF-8?q?=D1=82=D1=80=D0=BE=20+=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.run/ParsingService [local].run.xml | 2 +- parsing-service/build.gradle | 1 + .../config/DateTimeFormatterConfig.java | 4 +- .../config/DynamicProxyInterceptor.java | 6 +- .../parsingservice/config/ProxyProvider.java | 4 +- .../config/UserAgentProvider.java | 4 +- .../config/WebClientConfig.java | 4 +- .../config/WebDriverConfig.java | 6 +- .../properties/OzonConfigProperties.java | 8 +- .../parsingservice/enumeration/Category.java | 3 +- .../ozon_parser/enumeration/OzonCategory.java | 31 +++ .../ozon_parser/pool/WebDriverPool.java | 41 ++-- .../service/MarketplaceParsingService.java | 7 - .../ozon_parser/service/OzonService.java | 17 ++ .../ozon_parser/service/dto/ParsedData.java | 2 + .../service/page/CategoryPage.java | 14 +- .../service/page/NoContentPage.java | 6 +- .../service/page/OzonCategoryPage.java | 228 ++++++++++++++++++ ...rsingService.java => OzonHtmlFetcher.java} | 94 ++++---- .../service/parsing/OzonPageParser.java | 19 ++ .../service/parsing/OzonParsingService.java | 108 ++++++--- .../service/parsing/PageScroller.java | 56 +++++ .../service/scheduler/OzonProductUpdater.java | 7 +- .../service/task/OzonParsingTask.java | 4 + .../entity/PriceHistoryEntity.java | 18 +- .../persistence/entity/PriceHistoryId.java | 14 +- .../persistence/entity/ProductEntity.java | 24 +- .../repository/ProductPriceRepository.java | 8 + .../repository/ProductRepository.java | 22 +- .../service/ProductService.java | 60 +++-- .../service/dto/PriceHistoryDto.java | 20 ++ .../service/dto/ProductDto.java | 28 +++ .../service/dto/ProductsPageDto.java | 22 ++ .../service/mapper/PriceHistoryMapper.java | 19 ++ .../service/mapper/ProductMapper.java | 22 ++ .../service/scheduler/PartitionScheduler.java | 2 +- .../web/handler/CommonExceptionHandler.java | 4 +- .../web/handler/ErrorResponse.java | 4 +- .../parsingservice/web/rest/MainRestApi.java | 50 ++++ .../parsingservice/web/rest/OzonRestApi.java | 23 ++ .../ProductInfoDto2ProductEntity.java | 4 +- .../service/ParsingService.java | 14 +- .../service/ProductService.java | 8 +- .../service/client/ClientImpl.java | 6 +- .../src/main/resources/application.yml | 9 +- .../ParsingServiceApplicationTests.java | 1 - 46 files changed, 850 insertions(+), 208 deletions(-) create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/enumeration/OzonCategory.java delete mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/MarketplaceParsingService.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/OzonService.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/page/OzonCategoryPage.java rename parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/parsing/{OzonCategoryPageParsingService.java => OzonHtmlFetcher.java} (55%) create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/parsing/OzonPageParser.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/parsing/PageScroller.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/task/OzonParsingTask.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/service/dto/PriceHistoryDto.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/service/dto/ProductDto.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/service/dto/ProductsPageDto.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/service/mapper/PriceHistoryMapper.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/service/mapper/ProductMapper.java rename parsing-service/src/main/java/ru/pricepulse/parsingservice/{ozon_parser => }/service/scheduler/PartitionScheduler.java (96%) create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/web/rest/MainRestApi.java create mode 100644 parsing-service/src/main/java/ru/pricepulse/parsingservice/web/rest/OzonRestApi.java diff --git a/parsing-service/.run/ParsingService [local].run.xml b/parsing-service/.run/ParsingService [local].run.xml index 9ee8cab..312c89f 100644 --- a/parsing-service/.run/ParsingService [local].run.xml +++ b/parsing-service/.run/ParsingService [local].run.xml @@ -1,6 +1,6 @@ -