Emelyanov535/Price_Pulse
2
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

features: change parse logic

Browse Source
This commit is contained in:
Emelyanov535 2024-10-15 12:13:46 +04:00
parent 42d947440c
commit 1df7dc94b8
2 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
parsing-service/.run/ParsingService [local].run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager"> <component name="ProjectRunConfigurationManager">
<configuration default="false" name="ParsingService [local]" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot"> <configuration default="false" name="ParsingService [local]" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<option name="ACTIVE_PROFILES" value="dev,headless,ozon,postgres_stat" /> <option name="ACTIVE_PROFILES" value="dev,headless,postgres_stat" />
<option name="SCHEDULED_DEBUGGER" value="true" /> <option name="SCHEDULED_DEBUGGER" value="true" />
<envs> <envs>
<env name="POSTGRES_JDBC_PASSWORD" value="postgres" /> <env name="POSTGRES_JDBC_PASSWORD" value="postgres" />

9
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/ParsingService.java
View File

@ -28,15 +28,12 @@ public class ParsingService {
private final ProductService productService; private final ProductService productService;
public void parse() { public void parse() {
List<ProductEntity> productEntities = new ArrayList<>();
List<PriceHistoryEntity> priceHistories = new ArrayList<>();
final int elementsInPage = 100; final int elementsInPage = 100;
int page = 1; int page = 1;
Integer totalPages = null; Integer totalPages = null;
do { do {
var pageData = client.scrapPage(page, marketplacesConfig.getWildberriesConfigProperties().getShard(), marketplacesConfig.getWildberriesConfigProperties().getLaptopUrl()); var pageData = client.scrapPage(page, marketplacesConfig.getWildberriesConfigProperties().getShard(), marketplacesConfig.getWildberriesConfigProperties().getLaptopUrl());
System.out.println("Получена страница: " + page); System.out.println("Получена страница: " + page);
if (totalPages == null) { if (totalPages == null) {
@ -45,6 +42,8 @@ public class ParsingService {
totalPages = (int) Math.ceil((double) totalElements / elementsInPage); totalPages = (int) Math.ceil((double) totalElements / elementsInPage);
} }
List<ProductEntity> productEntities = new ArrayList<>();
List<PriceHistoryEntity> priceHistories = new ArrayList<>();
List<ProductInfoDto> productInfoDtoList = convertMapObjectToListProductInfoDto(pageData); List<ProductInfoDto> productInfoDtoList = convertMapObjectToListProductInfoDto(pageData);
productInfoDtoList.forEach(dto -> { productInfoDtoList.forEach(dto -> {
@ -62,8 +61,8 @@ public class ParsingService {
}); });
productService.saveData(productEntities, priceHistories); productService.saveData(productEntities, priceHistories);
page++; page++;
} while (page <= totalPages); // } while (page <= totalPages);
// } while (page <= 5); } while (page <= 5);
} }
private List<ProductInfoDto> convertMapObjectToListProductInfoDto(Map<String, Object> map) { private List<ProductInfoDto> convertMapObjectToListProductInfoDto(Map<String, Object> map) {