Emelyanov535/Price_Pulse
2
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

features: change parse logic

Browse Source
This commit is contained in:
Emelyanov535 2024-10-15 12:13:46 +04:00
parent 42d947440c
commit 1df7dc94b8
2 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
parsing-service/.run/ParsingService [local].run.xml
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ParsingService [local]" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<option name="ACTIVE_PROFILES" value="dev,headless,ozon,postgres_stat" />
<option name="ACTIVE_PROFILES" value="dev,headless,postgres_stat" />
<option name="SCHEDULED_DEBUGGER" value="true" />
<envs>
<env name="POSTGRES_JDBC_PASSWORD" value="postgres" />

9
parsing-service/src/main/java/ru/pricepulse/parsingservice/wildberries_parser/service/ParsingService.java
View File

@ -28,15 +28,12 @@ public class ParsingService {
private final ProductService productService;
public void parse() {
List<ProductEntity> productEntities = new ArrayList<>();
List<PriceHistoryEntity> priceHistories = new ArrayList<>();
final int elementsInPage = 100;
int page = 1;
Integer totalPages = null;
do {
var pageData = client.scrapPage(page, marketplacesConfig.getWildberriesConfigProperties().getShard(), marketplacesConfig.getWildberriesConfigProperties().getLaptopUrl());
System.out.println("Получена страница: " + page);
if (totalPages == null) {
@ -45,6 +42,8 @@ public class ParsingService {
totalPages = (int) Math.ceil((double) totalElements / elementsInPage);
}
List<ProductEntity> productEntities = new ArrayList<>();
List<PriceHistoryEntity> priceHistories = new ArrayList<>();
List<ProductInfoDto> productInfoDtoList = convertMapObjectToListProductInfoDto(pageData);
productInfoDtoList.forEach(dto -> {
@ -62,8 +61,8 @@ public class ParsingService {
});
productService.saveData(productEntities, priceHistories);
page++;
} while (page <= totalPages);
// } while (page <= 5);
// } while (page <= totalPages);
} while (page <= 5);
}
private List<ProductInfoDto> convertMapObjectToListProductInfoDto(Map<String, Object> map) {