Emelyanov535/Price_Pulse
2
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Feature/parsing-service: save

Browse Source
This commit is contained in:
danil.markov 2024-11-13 14:20:24 +04:00
parent 4f5dda4dbf
commit 8afc758987
4 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
parsing-service/src/main/java/ru/pricepulse/parsingservice/config/DynamicProxyInterceptor.java
View File

@ -20,12 +20,12 @@ public class DynamicProxyInterceptor implements ClientHttpRequestInterceptor {
@Override
public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException {
// Получаем случайный прокси
InetSocketAddress proxyAddress = proxyProvider.getNextProxy();
log.info("Используемый прокси: {}:{}", proxyAddress.getHostName(), proxyAddress.getPort());
//InetSocketAddress proxyAddress = proxyProvider.getNextProxy();
//log.info("Используемый прокси: {}:{}", proxyAddress.getHostName(), proxyAddress.getPort());
// Устанавливаем прокси
System.setProperty("http.proxyHost", proxyAddress.getHostName());
System.setProperty("http.proxyPort", String.valueOf(proxyAddress.getPort()));
//System.setProperty("http.proxyHost", proxyAddress.getHostName());
//System.setProperty("http.proxyPort", String.valueOf(proxyAddress.getPort()));
//Устанавливаем динамический user-agent
String randomUserAgent = userAgentProvider.getRandomUserAgent();

2
parsing-service/src/main/java/ru/pricepulse/parsingservice/config/ProxyProvider.java
View File

@ -28,7 +28,7 @@ public class ProxyProvider {
this.resourceLoader = resourceLoader;
}
@PostConstruct
//@PostConstruct
public void init() throws IOException {
Resource proxy = resourceLoader.getResource("classpath:proxy.txt");
Resource okProxy = resourceLoader.getResource("classpath:ok-proxy.txt");

2
parsing-service/src/main/java/ru/pricepulse/parsingservice/ozon_parser/service/scheduler/OzonProductUpdater.java
View File

@ -13,7 +13,7 @@ public class OzonProductUpdater {
private final OzonParsingService ozonParsingService;
@Scheduled(cron = "0 0 0,6,12,18 * * *")
@Scheduled(fixedRate = 7200000)
public void updateOzonProducts() {
ozonParsingService.startProcessing();
}

6
parsing-service/src/main/resources/application.yml
View File

@ -25,7 +25,7 @@ marketplace:
max-threads: ${OZON_MAX_PROCESSING_THREADS:5}
max-num-of-pages-on-screen: ${OZON_MAX_NUM_OF_PAGES_ON_SCREEN:100}
wildberries:
status: false
status: true
base-url: "https://static-basket-01.wbbasket.ru"
catalog-url: "/vol0/data/main-menu-ru-ru-v3.json"
user-agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)"
@ -38,8 +38,8 @@ marketplace:
logging:
pattern:
console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg %X%n"
level:
sql: debug
# level:
# sql: debug
# level:
# org:
# springframework: