From 553edd934c17955f9ef55276b9ca74af726ddd79 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Tue, 6 May 2025 09:42:02 +0300 Subject: [PATCH 01/26] feat: refactored stats and released recommendations feature --- core/event-service/pom.xml | 181 +++++++-------- .../controller/PublicEventController.java | 47 +++- .../mapper/event/UtilEventClass.java | 4 +- .../main/java/ru/practicum/model/Event.java | 2 +- .../practicum/repository/EventRepository.java | 16 ++ .../practicum/service/event/EventService.java | 13 +- .../service/event/EventServiceImpl.java | 215 ++++++++---------- .../ru/practicum/dto/event/EventFullDto.java | 2 +- .../dto/event/EventRecommendationDto.java | 17 +- .../ru/practicum/dto/event/EventShortDto.java | 2 +- core/pom.xml | 65 ++++++ core/request-service/pom.xml | 176 +++++++------- .../practicum/service/RequestServiceImpl.java | 4 + core/user-service/pom.xml | 202 ++++++++-------- docker-compose.yml | 186 +++++++++------ .../config/stats.stats-server/application.yml | 20 -- .../config/stats/aggregator/application.yml | 11 + .../config/stats/analyzer/application.yml | 39 ++++ .../config/stats/collector/application.yml | 20 ++ pom.xml | 36 ++- stats/{stats-server => aggregator}/Dockerfile | 0 stats/{stats-server => aggregator}/pom.xml | 31 ++- .../ru/practicum/AggregatorApplication.java | 15 ++ .../java/ru/practicum/config/KafkaConfig.java | 53 +++++ .../ru/practicum/config/KafkaProperties.java | 33 +++ .../practicum/config/UserActionsConsumer.java | 25 ++ .../practicum/service/MinWeightsMatrix.java | 22 ++ .../practicum/service/SimilarityService.java | 110 +++++++++ .../src/main/resources/application.yml | 28 +++ stats/analyzer/Dockerfile | 5 + stats/analyzer/pom.xml | 143 ++++++++++++ .../ru/practicum/AnalyzerApplication.java} | 7 +- .../config/EventsSimilarityConsumer.java | 25 ++ .../java/ru/practicum/config/KafkaConfig.java | 65 ++++++ .../ru/practicum/config/KafkaProperties.java | 22 ++ .../practicum/config/UserActionsConsumer.java | 25 ++ .../controller/RecommendationsController.java | 106 +++++++++ .../ru/practicum/model/EventSimilarity.java | 28 +++ .../ru/practicum/model/RecommendedEvent.java | 7 + .../java/ru/practicum/model/UserAction.java | 35 +++ .../repository/EventSimilarityRepository.java | 11 + .../repository/UserActionRepository.java | 15 ++ .../service/RecommendationService.java | 112 +++++++++ .../service/event/EventSimilarityService.java | 7 + .../event/EventSimilarityServiceImpl.java | 50 ++++ .../service/user/UserActionService.java | 7 + .../service/user/UserActionServiceImpl.java | 58 +++++ .../src/main/resources/application.yml | 29 +-- stats/collector/Dockerfile | 5 + stats/collector/pom.xml | 94 ++++++++ .../ru/practicum/CollectorApplication.java | 15 ++ .../practicum/config/KafkaProducerConfig.java | 38 ++++ .../ru/practicum/config/KafkaProperties.java | 29 +++ .../ru/practicum/mapper/UserActionMapper.java | 29 +++ .../service/KafkaMessageProducer.java | 20 ++ .../ru/practicum/service/MessageProducer.java | 7 + .../service/UserActionController.java | 41 ++++ .../src/main/resources/application.yml | 28 +++ stats/pom.xml | 6 +- stats/serialization/avro-schemas/pom.xml | 81 +++++++ .../main/avro/EventSimilarityProtocol.avdl | 10 + .../src/main/avro/UserActionAvro.avdl | 16 ++ .../java/ru/practicum/AvroDeserializer.java | 51 +++++ .../java/ru/practicum/AvroSerializer.java | 39 ++++ stats/serialization/pom.xml | 27 +++ stats/serialization/proto-schemas/pom.xml | 93 ++++++++ .../recommendations_controller.proto | 17 ++ .../controller/user_action_controller.proto | 10 + .../messages/recommendations_messages.proto | 25 ++ .../main/protobuf/messages/user_action.proto | 21 ++ stats/stats-client/pom.xml | 33 ++- .../java/ru/practicum/AnalyzerClient.java | 78 +++++++ .../java/ru/practicum/CollectorClient.java | 55 +++++ .../main/java/ru/practicum/StatClient.java | 48 ++-- .../java/ru/practicum/StatServiceClient.java | 68 +++--- .../src/main/resources/application.properties | 1 - .../src/main/resources/application.yml | 13 ++ stats/stats-dto/pom.xml | 39 ---- .../java/ru/practicum/dto/EndpointHitDto.java | 48 ---- .../practicum/dto/EndpointHitResponseDto.java | 21 -- .../dto/EndpointHitSaveRequestDto.java | 27 --- .../practicum/dto/SecondaryViewStatsDto.java | 26 --- .../main/java/ru/practicum/ErrorResponse.java | 32 --- .../practicum/controller/StatController.java | 67 ------ .../practicum/mapper/EndpointHitMapper.java | 37 --- .../java/ru/practicum/model/EndpointHit.java | 34 --- .../repository/EndpointHitRepository.java | 89 -------- .../ru/practicum/service/StatService.java | 14 -- .../ru/practicum/service/StatServiceImpl.java | 51 ----- .../java/ru/practicum/util/Constants.java | 10 - .../src/main/resources/schema.sql | 9 - 91 files changed, 2714 insertions(+), 1120 deletions(-) rename stats/stats-dto/src/main/java/ru/practicum/dto/ViewStatsDto.java => core/interaction-api/src/main/java/ru/practicum/dto/event/EventRecommendationDto.java (63%) delete mode 100644 infra/config-server/src/main/resources/config/stats.stats-server/application.yml create mode 100644 infra/config-server/src/main/resources/config/stats/aggregator/application.yml create mode 100644 infra/config-server/src/main/resources/config/stats/analyzer/application.yml create mode 100644 infra/config-server/src/main/resources/config/stats/collector/application.yml rename stats/{stats-server => aggregator}/Dockerfile (100%) rename stats/{stats-server => aggregator}/pom.xml (83%) create mode 100644 stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java create mode 100644 stats/aggregator/src/main/resources/application.yml create mode 100644 stats/analyzer/Dockerfile create mode 100644 stats/analyzer/pom.xml rename stats/{stats-server/src/main/java/ru/practicum/StatServer.java => analyzer/src/main/java/ru/practicum/AnalyzerApplication.java} (79%) create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/model/UserAction.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java rename stats/{stats-server => analyzer}/src/main/resources/application.yml (52%) create mode 100644 stats/collector/Dockerfile create mode 100644 stats/collector/pom.xml create mode 100644 stats/collector/src/main/java/ru/practicum/CollectorApplication.java create mode 100644 stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java create mode 100644 stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java create mode 100644 stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java create mode 100644 stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java create mode 100644 stats/collector/src/main/java/ru/practicum/service/MessageProducer.java create mode 100644 stats/collector/src/main/java/ru/practicum/service/UserActionController.java create mode 100644 stats/collector/src/main/resources/application.yml create mode 100644 stats/serialization/avro-schemas/pom.xml create mode 100644 stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl create mode 100644 stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl create mode 100644 stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroDeserializer.java create mode 100644 stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroSerializer.java create mode 100644 stats/serialization/pom.xml create mode 100644 stats/serialization/proto-schemas/pom.xml create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto create mode 100644 stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java create mode 100644 stats/stats-client/src/main/java/ru/practicum/CollectorClient.java delete mode 100644 stats/stats-client/src/main/resources/application.properties create mode 100644 stats/stats-client/src/main/resources/application.yml delete mode 100644 stats/stats-dto/pom.xml delete mode 100644 stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitDto.java delete mode 100644 stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitResponseDto.java delete mode 100644 stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitSaveRequestDto.java delete mode 100644 stats/stats-dto/src/main/java/ru/practicum/dto/SecondaryViewStatsDto.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/ErrorResponse.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/controller/StatController.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/mapper/EndpointHitMapper.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/model/EndpointHit.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/repository/EndpointHitRepository.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/service/StatService.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/service/StatServiceImpl.java delete mode 100644 stats/stats-server/src/main/java/ru/practicum/util/Constants.java delete mode 100644 stats/stats-server/src/main/resources/schema.sql diff --git a/core/event-service/pom.xml b/core/event-service/pom.xml index e38dce8..dc34144 100644 --- a/core/event-service/pom.xml +++ b/core/event-service/pom.xml @@ -1,132 +1,133 @@ - 4.0.0 + ru.practicum core 0.0.1-SNAPSHOT + event-service - 22 - 22 + 21 + 21 UTF-8 + 5.0.0 - - org.springframework.data - spring-data-jpa - - - org.projectlombok - lombok - provided - - - org.apache.tomcat.embed - tomcat-embed-core - - - com.querydsl - querydsl-jpa - jakarta - 5.1.0 - + ru.practicum interaction-api 0.0.1-SNAPSHOT + - org.mapstruct - mapstruct - 1.6.2 - provided - - - org.mapstruct - mapstruct-processor - 1.6.2 - provided + org.springframework.boot + spring-boot-starter-web + - com.netflix.spectator - spectator-api - 1.7.3 - compile + org.springframework.boot + spring-boot-configuration-processor + true + - jakarta.persistence - jakarta.persistence-api + org.springframework.boot + spring-boot-starter-actuator + - org.hibernate.orm - hibernate-core + org.springframework.boot + spring-boot-starter-validation + - org.springframework.retry - spring-retry + org.springframework.boot + spring-boot-starter-data-jpa + - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client - 3.1.8 + org.postgresql + postgresql + runtime + - org.springframework.cloud - spring-cloud-config-client - 4.2.0 + com.h2database + h2 + runtime + - org.springframework.boot - spring-boot-starter-web + org.mapstruct + mapstruct + 1.5.5.Final + - org.springframework.boot - spring-boot-starter-data-jpa + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + - org.postgresql - postgresql + com.querydsl + querydsl-apt + ${querydsl.version} + jakarta + provided + - org.springframework.cloud - spring-cloud-starter-openfeign + com.querydsl + querydsl-jpa + jakarta + ${querydsl.version} - ru.practicum - stats-client - 0.0.1-SNAPSHOT - compile + org.projectlombok + lombok + provided - - - - - org.springframework.cloud - spring-cloud-dependencies - 2024.0.0 - pom - import - - - + org.springframework.boot spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 - - paketobuildpacks/builder-jammy-base:latest - + + + org.mapstruct + mapstruct-processor + 1.5.5.Final + + + org.projectlombok + lombok + 1.18.30 + + + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + + @@ -144,36 +145,8 @@ - - - com.querydsl - querydsl-apt - jakarta - 5.1.0 - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.3.0 - - - add-source - generate-sources - - add-source - - - - ${project.build.directory}/generated-sources/java/ - - - - - \ No newline at end of file diff --git a/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java b/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java index 6e23883..d4ba46c 100644 --- a/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java +++ b/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java @@ -5,14 +5,20 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.web.bind.annotation.*; +import ru.practicum.AnalyzerClient; +import ru.practicum.CollectorClient; import ru.practicum.dto.event.EventFullDto; +import ru.practicum.dto.event.EventRecommendationDto; import ru.practicum.dto.event.EventShortDto; +import ru.practicum.ewm.stats.proto.ActionTypeProto; +import ru.practicum.ewm.stats.proto.RecommendationsMessages; import ru.practicum.exception.IncorrectValueException; import ru.practicum.service.event.EventSearchParams; import ru.practicum.service.event.EventService; import ru.practicum.service.event.PublicSearchParams; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; import static ru.practicum.constant.Constant.PATTERN_DATE; @@ -25,7 +31,13 @@ @RequiredArgsConstructor @Slf4j public class PublicEventController { + + private final CollectorClient collectorClient; + + private final AnalyzerClient analyzerClient; + private final EventService eventService; + private static final String X_EWM_USER_ID_HEADER = "X-EWM-USER-ID"; /** * Gets events. @@ -83,21 +95,32 @@ public List getAll( return eventShortDtoList; } - /** - * Gets event by id. - * - * @param eventId the event id - * @param request the request - * @return the event by id - */ @GetMapping("/{event-id}") - public EventFullDto getEventById(@PathVariable("event-id") Long eventId, HttpServletRequest request) { + public EventFullDto getEventById(@PathVariable("event-id") Long eventId, @RequestHeader(X_EWM_USER_ID_HEADER) long userId) { log.info("Получение информации о событии с id={}", eventId); - // Получение IP клиента - String clientIp = request.getRemoteAddr(); - // Получение события через сервис - return eventService.getEventById(eventId, clientIp); + return eventService.getEventById(eventId, userId); + } + + @GetMapping("/recommendations") + public List getRecommendations(@RequestHeader(X_EWM_USER_ID_HEADER) long userId, + @RequestParam(defaultValue = "10") int maxResults) { + var recommendationStream = analyzerClient.getRecommendationsForUser(userId, maxResults); + var recommendationList = recommendationStream.toList(); + + List result = new ArrayList<>(); + for (RecommendationsMessages.RecommendedEventProto requestProto : recommendationList) { + result.add(new EventRecommendationDto(requestProto.getEventId(), requestProto.getScore())); + } + return result; + } + + @PutMapping("/{event-id}/like") + public void likeEvent(@PathVariable("event-id") Long eventId, + @RequestHeader(X_EWM_USER_ID_HEADER) long userId) { + eventService.addLike(userId, eventId); + + collectorClient.sendUserAction(userId, eventId, ActionTypeProto.ACTION_LIKE); } } diff --git a/core/event-service/src/main/java/ru/practicum/mapper/event/UtilEventClass.java b/core/event-service/src/main/java/ru/practicum/mapper/event/UtilEventClass.java index a70527f..baadf02 100644 --- a/core/event-service/src/main/java/ru/practicum/mapper/event/UtilEventClass.java +++ b/core/event-service/src/main/java/ru/practicum/mapper/event/UtilEventClass.java @@ -128,7 +128,7 @@ public Event updateEvent(Event updatedEvent, UpdateEventAdminRequest request, Ca request.getRequestModeration() : updatedEvent.getRequestModeration()) .state(updatedEvent.getState()) .title(request.getTitle() != null ? request.getTitle() : updatedEvent.getTitle()) - .views(updatedEvent.getViews()) + .rating(updatedEvent.getRating()) .build(); } @@ -160,7 +160,7 @@ public EventFullDto toEventFullDto(Event event) { eventFullDto.setRequestModeration(event.getRequestModeration()); eventFullDto.setState(event.getState()); eventFullDto.setTitle(event.getTitle()); - eventFullDto.setViews(event.getViews()); + eventFullDto.setRating(event.getRating()); eventFullDto.setEventDate(event.getEventDate().format(formatter)); return eventFullDto; diff --git a/core/event-service/src/main/java/ru/practicum/model/Event.java b/core/event-service/src/main/java/ru/practicum/model/Event.java index 93c5b4e..2e7cd3b 100644 --- a/core/event-service/src/main/java/ru/practicum/model/Event.java +++ b/core/event-service/src/main/java/ru/practicum/model/Event.java @@ -55,7 +55,7 @@ public class Event { EventState state; String title; @Transient - Long views; + double rating; @Transient Long likes; } diff --git a/core/event-service/src/main/java/ru/practicum/repository/EventRepository.java b/core/event-service/src/main/java/ru/practicum/repository/EventRepository.java index 723a1c0..d0832d9 100644 --- a/core/event-service/src/main/java/ru/practicum/repository/EventRepository.java +++ b/core/event-service/src/main/java/ru/practicum/repository/EventRepository.java @@ -1,7 +1,9 @@ package ru.practicum.repository; +import jakarta.transaction.Transactional; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.querydsl.QuerydslPredicateExecutor; import org.springframework.data.repository.query.Param; @@ -77,4 +79,18 @@ List findAllEvents(@Param("text") String text, @Query(value = "SELECT COUNT(*) FROM LIKES_EVENTS WHERE EVENT_ID = :eventId", nativeQuery = true) long countLikesByEventId(Long eventId); + + @Query(value = "SELECT EXISTS (" + + "SELECT * FROM LIKES_EVENTS WHERE USER_ID = :userId AND EVENT_ID = :eventId)", nativeQuery = true) + boolean checkLikeExistence(long userId, long eventId); + + @Modifying + @Transactional + @Query(value = "INSERT INTO LIKES_EVENTS (USER_ID, EVENT_ID) values (:userId, :eventId)", nativeQuery = true) + void addLike(Long userId, Long eventId); + + @Modifying + @Transactional + @Query(value = "DELETE FROM LIKES_EVENTS WHERE USER_ID = :userId AND EVENT_ID = :eventId", nativeQuery = true) + void deleteLike(Long userId, Long eventId); } diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventService.java b/core/event-service/src/main/java/ru/practicum/service/event/EventService.java index ac758c6..d03b017 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventService.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventService.java @@ -89,19 +89,16 @@ List getEvents(String text, List categories, Boolean paid, LocalDateTime rangeStart, LocalDateTime rangeEnd, Boolean onlyAvailable, String sort, int from, int size, String clientIp); - /** - * Gets event by id. - * - * @param id the id - * @param clientIp the client ip - * @return the event by id - */ - EventFullDto getEventById(Long id, String clientIp); + EventFullDto getEventById(Long id, long userId); EventFullDto getByIdInternal(long eventId); @Transactional(readOnly = true) List getAllByPublic(EventSearchParams searchParams, Boolean onlyAvailable, String sort, String clientIp); + + EventShortDto addLike(long userId, long eventId); + + void deleteLike(long userId, long eventId); } diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index 4fafb81..71e6bbd 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -9,16 +9,16 @@ import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import ru.practicum.StatClient; +import ru.practicum.AnalyzerClient; +import ru.practicum.CollectorClient; import ru.practicum.client.RequestServiceClient; import ru.practicum.client.UserServiceClient; -import ru.practicum.dto.EndpointHitDto; -import ru.practicum.dto.ViewStatsDto; import ru.practicum.dto.category.CategoryDto; import ru.practicum.dto.event.*; import ru.practicum.enums.AdminStateAction; import ru.practicum.enums.EventState; import ru.practicum.enums.RequestStatus; +import ru.practicum.ewm.stats.proto.ActionTypeProto; import ru.practicum.exception.ConflictException; import ru.practicum.exception.NotFoundException; import ru.practicum.exception.ValidationException; @@ -46,6 +46,8 @@ @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class EventServiceImpl implements EventService { + private final CollectorClient collectorClient; + private final UserServiceClient userServiceClient; private final LocationMapper locationMapper; @@ -59,16 +61,17 @@ public class EventServiceImpl implements EventService { LocationRepository locationRepository; SearchEventRepository searchEventRepository; CategoryRepository categoryRepository; - StatClient statClient; + AnalyzerClient analyzerClient; @Autowired public EventServiceImpl(EventRepository eventRepository, RequestServiceClient requestServiceClient, EventMapper eventMapper, CategoryService categoryService, UtilEventClass utilEventClass, LocationRepository locationRepository, SearchEventRepository searchEventRepository, - CategoryRepository categoryRepository, StatClient statClient, + CategoryRepository categoryRepository, LocationMapper locationMapper, - UserServiceClient userServiceClient) { + UserServiceClient userServiceClient, AnalyzerClient analyzerClient, + CollectorClient collectorClient) { this.eventRepository = eventRepository; this.requestServiceClient = requestServiceClient; this.eventMapper = eventMapper; @@ -77,9 +80,10 @@ public EventServiceImpl(EventRepository eventRepository, this.locationRepository = locationRepository; this.searchEventRepository = searchEventRepository; this.categoryRepository = categoryRepository; - this.statClient = statClient; + this.analyzerClient = analyzerClient; this.locationMapper = locationMapper; this.userServiceClient = userServiceClient; + this.collectorClient = collectorClient; } @Override @@ -213,12 +217,6 @@ public List getEvents(String text, List categories, Boolean if (Boolean.TRUE.equals(text == null && categories == null && paid == null && rangeStart == null && rangeEnd == null && !onlyAvailable && sort == null && from == 0) && size == 10) { - log.info("==> Статистика: вызов метода getEvents с пустыми параметрами от клиента {}", clientIp); - - // Записываем статистику - saveEventsRequestToStats(clientIp); - - // Возвращаем пустой список return Collections.emptyList(); } @@ -256,21 +254,6 @@ public List getEvents(String text, List categories, Boolean .map(event -> "/events/" + event.getId()) // Получаем URI для каждого мероприятия .toList(); - // Запрашиваем статистику просмотров с использованием StatClient - List viewStats = statClient.getStats(rangeStart.toString(), rangeEnd.toString(), uris, true); - - // Обработка случая, если статистика отсутствует - if (viewStats == null || viewStats.isEmpty()) { - log.warn("Сервис статистики вернул пустой результат или null"); - viewStats = Collections.emptyList(); - } - - // Заполняем Map с количеством просмотров - for (ViewStatsDto stat : viewStats) { - Long eventId = Long.valueOf(stat.getUri().substring(stat.getUri().lastIndexOf("/") + 1)); - eventViews.put(eventId, Math.toIntExact(stat.getHits())); - } - // Сортировка if ("VIEWS".equalsIgnoreCase(sort)) { // Сортировка по количеству просмотров @@ -283,10 +266,7 @@ public List getEvents(String text, List categories, Boolean // Сортировка по дате события filteredEvents.sort(Comparator.comparing(Event::getEventDate)); } - log.info("Передаем запрос в статистику"); - // Логируем запрос в статистику - saveEventsRequestToStats(clientIp); // Применяем пагинацию int start = Math.min(from, filteredEvents.size()); @@ -299,7 +279,7 @@ public List getEvents(String text, List categories, Boolean } @Override - public EventFullDto getEventById(Long eventId, String clientIp) { + public EventFullDto getEventById(Long eventId, long userId) { // Проверка существования события Event event = eventRepository.findById(eventId).orElseThrow( () -> new NotFoundException("Event with id=" + eventId + " not found!", "") @@ -310,84 +290,79 @@ public EventFullDto getEventById(Long eventId, String clientIp) { throw new NotFoundException("Event with id=" + eventId + " is not published yet!", ""); } - // Увеличение количества просмотров - saveEventRequestToStats(event, clientIp); - - // Получение количества просмотров из статистики - long views = getViewsFromStats(event); - - event.setViews(views); eventRepository.save(event); + collectorClient.sendUserAction(userId, eventId, ActionTypeProto.ACTION_VIEW); + // Подсчет подтвержденных запросов long confirmedRequests = requestServiceClient.countByStatusAndEventId(RequestStatus.CONFIRMED, eventId); // Создание DTO EventFullDto eventFullDto = utilEventClass.toEventFullDto(event); - eventFullDto.setViews(views); eventFullDto.setConfirmedRequests(confirmedRequests); - return eventFullDto; - } - - private void saveEventsRequestToStats(String clientIp) { - try { - // Создание объекта для статистики - log.info("Создание объекта для статистики"); - EndpointHitDto hitDto = new EndpointHitDto(); - hitDto.setApp("ewm-main-service"); - hitDto.setUri("/events"); - hitDto.setIp(clientIp); - hitDto.setTimestamp(LocalDateTime.now().format(dateTimeFormatter)); - - // Логируем успешный запрос - log.info("Логируем запрос в статистику: URI={}, IP={}", hitDto.getUri(), hitDto.getIp()); - - // Отправка статистики - statClient.saveHit(hitDto); - } catch (Exception e) { - log.error("Ошибка при сохранении статистики для URI=/events, IP=" + clientIp, e); - } - } - private void saveEventRequestToStats(Event event, String clientIp) { - try { - EndpointHitDto hitDto = new EndpointHitDto(); - hitDto.setApp("ewm-main-service"); - hitDto.setUri("/events/" + event.getId()); - hitDto.setIp(clientIp); - hitDto.setTimestamp(LocalDateTime.now().format(dateTimeFormatter)); - - statClient.saveHit(hitDto); - } catch (Exception e) { - log.error("Ошибка при сохранении статистики для события id=" + event.getId(), e); - } + return eventFullDto; } - private long getViewsFromStats(Event event) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - try { - String uri = "/events/" + event.getId(); - // Добавляем одну секунду к началу и завершению диапазона - String start = event.getCreatedOn().minusSeconds(1).format(formatter); - String end = LocalDateTime.now().plusSeconds(1).format(formatter); - - List stats = statClient.getStats( - start, - end, - List.of(uri), - true - ); - - return stats.stream() - .filter(stat -> stat.getUri().equals(uri)) - .mapToLong(ViewStatsDto::getHits) - .sum(); - } catch (Exception e) { - log.error("Ошибка при получении статистики просмотров для события id=" + event.getId(), e); - return 0; - } - } +// private void saveEventsRequestToStats(String clientIp) { +// try { +// // Создание объекта для статистики +// log.info("Создание объекта для статистики"); +// EndpointHitDto hitDto = new EndpointHitDto(); +// hitDto.setApp("ewm-main-service"); +// hitDto.setUri("/events"); +// hitDto.setIp(clientIp); +// hitDto.setTimestamp(LocalDateTime.now().format(dateTimeFormatter)); +// +// // Логируем успешный запрос +// log.info("Логируем запрос в статистику: URI={}, IP={}", hitDto.getUri(), hitDto.getIp()); +// +// // Отправка статистики +// statClient.saveHit(hitDto); +// } catch (Exception e) { +// log.error("Ошибка при сохранении статистики для URI=/events, IP=" + clientIp, e); +// } +// } + +// private void saveEventRequestToStats(Event event, String clientIp) { +// try { +// EndpointHitDto hitDto = new EndpointHitDto(); +// hitDto.setApp("ewm-main-service"); +// hitDto.setUri("/events/" + event.getId()); +// hitDto.setIp(clientIp); +// hitDto.setTimestamp(LocalDateTime.now().format(dateTimeFormatter)); +// +// statClient.saveHit(hitDto); +// } catch (Exception e) { +// log.error("Ошибка при сохранении статистики для события id=" + event.getId(), e); +// } +// } + +// private long getViewsFromStats(Event event) { +// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); +// try { +// String uri = "/events/" + event.getId(); +// // Добавляем одну секунду к началу и завершению диапазона +// String start = event.getCreatedOn().minusSeconds(1).format(formatter); +// String end = LocalDateTime.now().plusSeconds(1).format(formatter); +// +// List stats = statClient.getStats( +// start, +// end, +// List.of(uri), +// true +// ); +// +// return stats.stream() +// .filter(stat -> stat.getUri().equals(uri)) +// .mapToLong(ViewStatsDto::getHits) +// .sum(); +// } catch (Exception e) { +// log.error("Ошибка при получении статистики просмотров для события id=" + event.getId(), e); +// return 0; +// } +// } private void checkDateTime(LocalDateTime rangeStart, LocalDateTime rangeEnd) { @@ -484,7 +459,7 @@ public List getAllByPublic(EventSearchParams searchParams, Boolea Long view = 0L; - event.setViews(view); + event.setRating(view); event.setConfirmedRequests( requestServiceClient.countByStatusAndEventId(RequestStatus.CONFIRMED, event.getId())); event.setLikes(eventRepository.countLikesByEventId(event.getId())); @@ -509,18 +484,6 @@ public List getAllByPublic(EventSearchParams searchParams, Boolea .toList(); - - List viewStats = statClient.getStats(rangeStart.toString(), rangeEnd.toString(), uris, true); - - if (viewStats == null || viewStats.isEmpty()) { - log.warn("Сервис статистики вернул пустой результат или null"); - viewStats = Collections.emptyList(); - } - for (ViewStatsDto stat : viewStats) { - Long eventId = Long.valueOf(stat.getUri().substring(stat.getUri().lastIndexOf("/") + 1)); - eventViews.put(eventId, stat.getHits()); - } - if ("VIEWS".equalsIgnoreCase(sort)) { // Сортировка по количеству просмотров filteredEvents.sort((e1, e2) -> { @@ -534,11 +497,35 @@ public List getAllByPublic(EventSearchParams searchParams, Boolea } log.info("Передаем запрос в статистику"); - // Логируем запрос в статистику - saveEventsRequestToStats(clientIp); - return eventListBySearch.stream() .map(eventMapper::toEventShortDto) .toList(); } + + @Override + public EventShortDto addLike(long userId, long eventId) { + Event event = eventRepository.findById(eventId).orElseThrow( + () -> new NotFoundException("Event with id = " + eventId, " not found") + ); + if (event.getState() != EventState.PUBLISHED) { + throw new ConflictException("Event with id = ", eventId + " is not published"); + } + eventRepository.addLike(userId, eventId); + event.setLikes(eventRepository.countLikesByEventId(eventId)); + return eventMapper.toEventShortDto(event); + } + + @Override + public void deleteLike(long userId, long eventId) { + Event event = eventRepository.findById(eventId).orElseThrow( + () -> new NotFoundException("Event with id = ", eventId + " not found") + ); + boolean isLikeExists = eventRepository.checkLikeExistence(userId, eventId); + if (isLikeExists) { + eventRepository.deleteLike(userId, eventId); + } else { + throw new NotFoundException("Like for event: ", eventId + + " by user: " + userId + " not exists"); + } + } } diff --git a/core/interaction-api/src/main/java/ru/practicum/dto/event/EventFullDto.java b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventFullDto.java index d53ede0..7b22ecf 100644 --- a/core/interaction-api/src/main/java/ru/practicum/dto/event/EventFullDto.java +++ b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventFullDto.java @@ -36,6 +36,6 @@ public class EventFullDto { boolean requestModeration; EventState state; String title; - Long views; + double rating; Long likes; } diff --git a/stats/stats-dto/src/main/java/ru/practicum/dto/ViewStatsDto.java b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventRecommendationDto.java similarity index 63% rename from stats/stats-dto/src/main/java/ru/practicum/dto/ViewStatsDto.java rename to core/interaction-api/src/main/java/ru/practicum/dto/event/EventRecommendationDto.java index 99f252e..b88ffe3 100644 --- a/stats/stats-dto/src/main/java/ru/practicum/dto/ViewStatsDto.java +++ b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventRecommendationDto.java @@ -1,4 +1,5 @@ -package ru.practicum.dto; +package ru.practicum.dto.event; + import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -6,15 +7,11 @@ import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; -/** - * The type View stats dto. - */ @Data -@NoArgsConstructor @AllArgsConstructor +@NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class ViewStatsDto { - String app; - String uri; - Long hits; -} +public class EventRecommendationDto { + long eventId; + double score; +} \ No newline at end of file diff --git a/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java index a4366a2..aa7eb2f 100644 --- a/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java +++ b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java @@ -24,7 +24,7 @@ public class EventShortDto { UserShortDto initiator; Boolean paid; String title; - Long views; + Long rating; diff --git a/core/pom.xml b/core/pom.xml index eb6fb25..a0a3336 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -17,4 +17,69 @@ interaction-api request-service + + + 21 + 21 + UTF-8 + + + + org.springframework.cloud + spring-cloud-starter-config + + + + org.springframework.retry + spring-retry + + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + + ru.practicum + aggregator + 0.0.1-SNAPSHOT + compile + + + ru.practicum + stats-client + 0.0.1-SNAPSHOT + compile + + + org.springframework + spring-tx + + + com.querydsl + querydsl-core + 5.0.0 + compile + + + org.springframework.data + spring-data-commons + + + org.projectlombok + lombok + + + + org.springframework.boot + spring-boot-starter-actuator + + + + \ No newline at end of file diff --git a/core/request-service/pom.xml b/core/request-service/pom.xml index 620d13e..9728d2f 100644 --- a/core/request-service/pom.xml +++ b/core/request-service/pom.xml @@ -1,8 +1,9 @@ - 4.0.0 + ru.practicum core @@ -12,97 +13,147 @@ request-service - 22 - 22 + 21 + 21 UTF-8 + 5.0.0 + + org.springframework.boot - spring-boot-starter + spring-boot-starter-web + - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client - 3.1.8 + org.springframework.boot + spring-boot-starter-actuator + org.springframework.boot - spring-boot-starter-web + spring-boot-starter-data-jpa + - org.projectlombok - lombok - provided + org.postgresql + postgresql + runtime + + + com.h2database + h2 + runtime + + org.mapstruct mapstruct - 1.6.2 - compile + 1.5.5.Final + + + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + + + + jakarta.validation + jakarta.validation-api + + + + org.springframework.boot + spring-boot-starter-validation + + org.mapstruct mapstruct-processor - 1.6.2 + 1.5.5.Final provided + - org.springframework.boot - spring-boot-starter-data-jpa + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + - com.querydsl - querydsl-jpa - jakarta - 5.1.0 + org.springframework.cloud + spring-cloud-starter-config + - ru.practicum - interaction-api - 0.0.1-SNAPSHOT - compile + org.springframework.cloud + spring-cloud-starter-openfeign + - org.postgresql - postgresql + org.springframework.retry + spring-retry + - org.springframework.boot - spring-boot-starter-actuator + org.projectlombok + lombok + true + - org.springframework.cloud - spring-cloud-starter-openfeign + com.querydsl + querydsl-apt + ${querydsl.version} + jakarta + provided + - org.springframework.cloud - spring-cloud-config-client - 4.2.0 + com.querydsl + querydsl-jpa + jakarta + ${querydsl.version} + + + ru.practicum + interaction-api + 0.0.1-SNAPSHOT + compile - - - - - org.springframework.cloud - spring-cloud-dependencies - 2024.0.0 - pom - import - - - + org.springframework.boot spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 - - paketobuildpacks/builder-jammy-base:latest - + + + org.mapstruct + mapstruct-processor + 1.5.5.Final + + + org.projectlombok + lombok + 1.18.30 + + + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + + @@ -120,33 +171,6 @@ - - - com.querydsl - querydsl-apt - jakarta - 5.1.0 - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.3.0 - - - add-source - generate-sources - - add-source - - - - ${project.build.directory}/generated-sources/java/ - - - - diff --git a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java index 95a059f..e52ba3f 100644 --- a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java +++ b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java @@ -4,12 +4,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import ru.practicum.CollectorClient; import ru.practicum.client.EventServiceClient; import ru.practicum.client.UserServiceClient; import ru.practicum.dto.request.EventRequestStatusUpdateRequest; import ru.practicum.dto.request.EventRequestStatusUpdateResult; import ru.practicum.dto.request.ParticipationRequestDto; import ru.practicum.enums.RequestStatus; +import ru.practicum.ewm.stats.proto.ActionTypeProto; import ru.practicum.exception.ConflictException; import ru.practicum.exception.NotFoundException; import ru.practicum.mapper.RequestMapper; @@ -30,6 +32,7 @@ public class RequestServiceImpl implements RequestService { private final RequestMapper requestMapper; private final UserServiceClient userServiceClient; private final RequestRepository requestRepository; + private final CollectorClient collectorClient; @Override public List getRequestByUserId(Long userId) { @@ -46,6 +49,7 @@ public ParticipationRequestDto createRequest(Long userId, Long eventId) { requestToEventVerification(userId, eventId); Request request = requestMapper.formUserAndEventToRequest(userId, eventId); requestRepository.save(request); + collectorClient.sendUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER); return requestMapper.toParticipationRequestDto(request); } diff --git a/core/user-service/pom.xml b/core/user-service/pom.xml index 47b9445..e610891 100644 --- a/core/user-service/pom.xml +++ b/core/user-service/pom.xml @@ -1,142 +1,179 @@ - 4.0.0 + ru.practicum core 0.0.1-SNAPSHOT + user-service - 22 - 22 + 21 + 21 UTF-8 + 5.0.0 + + + + ru.practicum + stats-client + 0.0.1-SNAPSHOT + + + + ru.practicum + aggregator + 0.0.1-SNAPSHOT + + org.springframework.boot spring-boot-starter-web + - org.zalando - logbook-spring-boot-starter - 3.9.0 + org.springframework.boot + spring-boot-starter-actuator + org.springframework.boot - spring-boot-starter-test + spring-boot-starter-validation + - org.springframework - spring-web + org.springframework.boot + spring-boot-starter-data-jpa + - org.projectlombok - lombok - provided + org.postgresql + postgresql + runtime + - org.springframework.data - spring-data-commons + com.h2database + h2 + runtime + org.mapstruct mapstruct - 1.6.2 + 1.5.5.Final + - org.mapstruct - mapstruct-processor - 1.6.2 - provided + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + jakarta.validation jakarta.validation-api + - jakarta.transaction - jakarta.transaction-api + org.springframework.boot + spring-boot-starter-validation + - org.springframework.data - spring-data-jpa + org.mapstruct + mapstruct-processor + 1.5.5.Final + provided + - com.querydsl - querydsl-core - 5.1.0 + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + - javax.persistence - javax.persistence-api - 2.2 + org.springframework.cloud + spring-cloud-starter-config + - org.springframework.boot - spring-boot-starter-data-jpa + org.springframework.cloud + spring-cloud-starter-openfeign + - ru.practicum - interaction-api - 0.0.1-SNAPSHOT - compile + org.springframework.retry + spring-retry + - org.springframework.boot - spring-boot-starter-actuator + org.projectlombok + lombok + true + + + com.querydsl - querydsl-jpa + querydsl-apt + ${querydsl.version} jakarta - 5.1.0 - - - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client - 3.1.8 - - - org.springframework.cloud - spring-cloud-config-client - 4.2.0 + provided + - org.postgresql - postgresql + com.querydsl + querydsl-jpa + jakarta + ${querydsl.version} - org.springframework.cloud - spring-cloud-starter-config + ru.practicum + interaction-api + 0.0.1-SNAPSHOT + compile - - - - - org.springframework.cloud - spring-cloud-dependencies - 2024.0.0 - pom - import - - - + org.springframework.boot spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 - - paketobuildpacks/builder-jammy-base:latest - + + + org.mapstruct + mapstruct-processor + 1.5.5.Final + + + org.projectlombok + lombok + 1.18.30 + + + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + + @@ -154,33 +191,6 @@ - - - com.querydsl - querydsl-apt - jakarta - 5.1.0 - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.3.0 - - - add-source - generate-sources - - add-source - - - - ${project.build.directory}/generated-sources/java/ - - - - diff --git a/docker-compose.yml b/docker-compose.yml index b7f2a57..71fe4d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,108 +37,104 @@ services: depends_on: config-server: condition: service_healthy + event-service: + condition: service_healthy user-service: condition: service_healthy request-service: condition: service_healthy - event-service: - condition: service_healthy - stats-server: - condition: service_healthy + networks: - ewm-net environment: - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - stats-server: - build: stats/stats-server - container_name: ewm-stats-server - ports: - - "9090:9090" + event-service: + build: core/event-service + container_name: event-service depends_on: - stats-db: + event-db: condition: service_healthy config-server: condition: service_healthy + networks: - ewm-net environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://stats-db:5432/ewm-stats + - SPRING_DATASOURCE_URL=jdbc:postgresql://event-db:5432/ewm-event - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=root - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=9090 + - SERVER_PORT=8081 healthcheck: - test: "curl --fail --silent localhost:9090/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8081/actuator/health | grep UP || exit 1" timeout: 5s - interval: 15s + interval: 25s retries: 10 - stats-db: + event-db: image: postgres:16.1 - container_name: postgres-ewm-stats-db + container_name: postgres-ewm-event-db + networks: + - ewm-net environment: - POSTGRES_PASSWORD=root - POSTGRES_USER=root - - POSTGRES_DB=ewm-stats - networks: - - ewm-net + - POSTGRES_DB=ewm-event + ports: + - 5434:5433 healthcheck: test: pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER timeout: 5s interval: 10s retries: 15 - # Event-service - event-service: - build: core/event-service - container_name: event-service + + request-service: + build: core/request-service + container_name: ewm-request-service depends_on: - event-db: + request-db: condition: service_healthy config-server: condition: service_healthy - stats-server: - condition: service_healthy networks: - ewm-net environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://event-db:5432/ewm-event + - SPRING_DATASOURCE_URL=jdbc:postgresql://request-db:5432/ewm-request - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=root - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=8081 + - SERVER_PORT=8083 healthcheck: - test: "curl --fail --silent localhost:8081/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8083/actuator/health | grep UP || exit 1" timeout: 5s interval: 25s retries: 10 - event-db: + + request-db: image: postgres:16.1 - container_name: postgres-ewm-event-db + container_name: postgres-ewm-request-db networks: - ewm-net environment: - POSTGRES_PASSWORD=root - POSTGRES_USER=root - - POSTGRES_DB=ewm-event + - POSTGRES_DB=ewm-request healthcheck: test: pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER timeout: 5s interval: 10s retries: 15 - # USER-SERVICE user-service: build: core/user-service - container_name: user-service + container_name: ewm-user-service depends_on: - event-db: + user-db: condition: service_healthy config-server: condition: service_healthy - stats-server: - condition: service_healthy networks: - ewm-net environment: @@ -146,9 +142,9 @@ services: - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=root - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=8083 + - SERVER_PORT=8084 healthcheck: - test: "curl --fail --silent localhost:8083/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8084/actuator/health | grep UP || exit 1" timeout: 5s interval: 25s retries: 10 @@ -168,51 +164,113 @@ services: interval: 10s retries: 15 - #REQUEST-SERVICE - request-db: - image: postgres:16.1 - container_name: postgres-ewm-request-db + kafka: + image: confluentinc/confluent-local:7.4.3 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" # for client connections + - "9101:9101" # JMX networks: - ewm-net + restart: unless-stopped environment: - - POSTGRES_PASSWORD=root - - POSTGRES_USER=root - - POSTGRES_DB=ewm-request - volumes: - - ${PWD}/core/request-service/src/main/resources/schema.sql:/docker-entrypoint-initdb.d/schema.sql + KAFKA_NODE_ID: 1 + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_PROCESS_ROLES: 'broker, controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' + KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' + CLUSTER_ID: 'K0EA9p0yEe6MkAAAAkKsEg' + + kafka-init-topics: + image: confluentinc/confluent-local:7.4.3 + container_name: kafka-init-topics + depends_on: + - kafka + networks: + - ewm-net + command: "bash -c \ + 'kafka-topics --create --topic stats.user-actions.v1 \ + --partitions 1 --replication-factor 1 --if-not-exists \ + --bootstrap-server kafka:29092 && \ + kafka-topics --create --topic stats.events-similarity.v1 \ + --partitions 1 --replication-factor 1 --if-not-exists \ + --bootstrap-server kafka:29092'" + init: true + + collector: + build: stats/collector + container_name: ewm-collector + depends_on: + config-server: + condition: service_healthy + networks: + - ewm-net + environment: + - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ + - SERVER_PORT=8085 healthcheck: - test: pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER + test: "curl --fail --silent localhost:8085/actuator/health | grep UP || exit 1" timeout: 5s - interval: 10s - retries: 15 + interval: 15s + retries: 10 - request-service: - build: core/request-service - container_name: request-service + aggregator: + build: stats/aggregator + container_name: ewm-aggregator depends_on: - user-db: - condition: service_healthy config-server: condition: service_healthy - stats-server: - condition: service_healthy - user-service: + networks: + - ewm-net + environment: + - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ + - SERVER_PORT=8086 + healthcheck: + test: "curl --fail --silent localhost:8086/actuator/health | grep UP || exit 1" + timeout: 5s + interval: 15s + retries: 10 + + analyzer: + build: stats/analyzer + container_name: ewm-analyzer + depends_on: + analyzer-db: condition: service_healthy - request-db: + config-server: condition: service_healthy networks: - ewm-net environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://request-db:5432/ewm-request + - SPRING_DATASOURCE_URL=jdbc:postgresql://analyzer-db:5432/ewm-analyzer - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=root - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=8085 + - SERVER_PORT=8087 healthcheck: - test: "curl --fail --silent localhost:8085/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8087/actuator/health | grep UP || exit 1" timeout: 5s - interval: 25s + interval: 15s retries: 10 + analyzer-db: + image: postgres:16.1 + container_name: postgres-ewm-analyzer-db + environment: + - POSTGRES_PASSWORD=root + - POSTGRES_USER=root + - POSTGRES_DB=ewm-analyzer + networks: + - ewm-net + healthcheck: + test: pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER + timeout: 5s + interval: 10s + retries: 15 + + networks: ewm-net: \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats.stats-server/application.yml b/infra/config-server/src/main/resources/config/stats.stats-server/application.yml deleted file mode 100644 index 6886a06..0000000 --- a/infra/config-server/src/main/resources/config/stats.stats-server/application.yml +++ /dev/null @@ -1,20 +0,0 @@ -spring: - datasource: - driverClassName: org.postgresql.Driver - url: jdbc:postgresql://stats-db:5432/ewm-stats - username: root - password: root - - jpa: - hibernate: - ddl-auto: none - database-platform: org.hibernate.dialect.PostgreSQLDialect - generate-ddl: false - properties: - hibernate: - format_sql: true - show-sql: false - - sql: - init: - mode: always \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml new file mode 100644 index 0000000..91d5086 --- /dev/null +++ b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml @@ -0,0 +1,11 @@ +kafka: + bootstrap-servers: kafka:29092 + consumer: + topic: stats.user-actions.v1 + group-id: aggregator-actions + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + producer: + topic: stats.events-similarity.v1 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: ru.practicum.AvroSerializer \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml new file mode 100644 index 0000000..fea45a3 --- /dev/null +++ b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml @@ -0,0 +1,39 @@ +spring: + datasource: + driverClassName: org.postgresql.Driver + url: jdbc:postgresql://analyzer-db:5432/ewm-analyzer + username: root + password: root + + jpa: + hibernate: + ddl-auto: update + properties: + hibernate: + format_sql: true + generate-ddl: false + + sql: + init: + mode: always + +kafka: + bootstrap-servers: kafka:29092 + user-actions-consumer: + group-id: analyzer-user-actions + topic: stats.user-actions.v1 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + events-similarity-consumer: + group-id: analyzer-events-sim + topic: stats.events-similarity.v1 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + +grpc: + client: + analyzer: + address: 'discovery:///analyzer' + enableKeepAlive: true + keepAliveWithoutCalls: true + negotiationType: plaintext \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/collector/application.yml b/infra/config-server/src/main/resources/config/stats/collector/application.yml new file mode 100644 index 0000000..74915bb --- /dev/null +++ b/infra/config-server/src/main/resources/config/stats/collector/application.yml @@ -0,0 +1,20 @@ +spring: + kafka: + bootstrap-servers: kafka:29092 + producer: + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: ru.practicum.AvroSerializer + +collector: + kafka: + topic: stats.user-actions.v1 + +grpc: + server: + port: 0 + client: + analyzer: + address: 'discovery:///collector' + enableKeepAlive: true + keepAliveWithoutCalls: true + negotiationType: plaintext \ No newline at end of file diff --git a/pom.xml b/pom.xml index 599095a..8ee149f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.0 + 3.3.4 @@ -15,7 +15,7 @@ core infra stats - + ru.practicum explore-with-me @@ -25,7 +25,14 @@ 21 UTF-8 - 2024.0.0 + 2023.0.3 + 1.12.0 + 3.25.1 + 1.63.0 + ${avro.version} + 2.4.0 + 3.11.0 + 3.1.0.RELEASE @@ -37,6 +44,29 @@ pom import + + net.devh + grpc-spring-boot-starter + ${grpc-spring-boot-starter.version} + + + + net.devh + grpc-server-spring-boot-starter + ${grpc-spring-boot-starter.version} + + + + io.grpc + grpc-stub + ${grpc.version} + + + + io.grpc + grpc-protobuf + ${grpc.version} + diff --git a/stats/stats-server/Dockerfile b/stats/aggregator/Dockerfile similarity index 100% rename from stats/stats-server/Dockerfile rename to stats/aggregator/Dockerfile diff --git a/stats/stats-server/pom.xml b/stats/aggregator/pom.xml similarity index 83% rename from stats/stats-server/pom.xml rename to stats/aggregator/pom.xml index d126e6a..480b505 100644 --- a/stats/stats-server/pom.xml +++ b/stats/aggregator/pom.xml @@ -3,22 +3,23 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - ru.practicum stats 0.0.1-SNAPSHOT - stats-server + aggregator - 17 - 17 + 21 + 21 UTF-8 + 0.0.1-SNAPSHOT + org.springframework.boot spring-boot-starter-web @@ -57,12 +58,6 @@ spring-boot-starter-test test - - ru.practicum - stats-dto - 0.0.1-SNAPSHOT - compile - org.projectlombok @@ -93,6 +88,22 @@ spring-cloud-openfeign-core + + org.apache.kafka + kafka-clients + + + + org.springframework.kafka + spring-kafka + + + + ru.practicum + avro-schemas + ${avro-schemas.version} + + diff --git a/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java b/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java new file mode 100644 index 0000000..0897186 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java @@ -0,0 +1,15 @@ +package ru.practicum; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; + +@SpringBootApplication +@EnableDiscoveryClient +@ConfigurationPropertiesScan +public class AggregatorApplication { + public static void main(String[] args) { + SpringApplication.run(AggregatorApplication.class, args); + } +} diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java new file mode 100644 index 0000000..ff3c0ab --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java @@ -0,0 +1,53 @@ +package ru.practicum.config; + +import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.*; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.Map; + +@Configuration +@RequiredArgsConstructor +public class KafkaConfig { + private final KafkaProperties props; + + @Bean + public ConsumerFactory consumerFactory() { + final Map config = Map.of( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), + ConsumerConfig.GROUP_ID_CONFIG, props.getConsumer().getGroupId(), + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, props.getConsumer().getKeyDeserializer(), + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, props.getConsumer().getValueDeserializer() + ); + return new DefaultKafkaConsumerFactory<>(config); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory()); + return factory; + } + + @Bean + public ProducerFactory producerFactory() { + Map config = Map.of( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, props.getProducer().getKeySerializer(), + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, props.getProducer().getValueSerializer() + ); + return new DefaultKafkaProducerFactory<>(config); + } + + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } +} diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java new file mode 100644 index 0000000..fa8a503 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java @@ -0,0 +1,33 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties(prefix = "kafka") +@Getter +@Setter +public class KafkaProperties { + private String bootstrapServers; + private Consumer consumer = new Consumer(); + private Producer producer = new Producer(); + + @Getter + @Setter + public static class Consumer { + private String groupId; + private String topic; + private String keyDeserializer; + private String valueDeserializer; + } + + @Getter + @Setter + public static class Producer { + private String topic; + private String keySerializer; + private String valueSerializer; + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java new file mode 100644 index 0000000..2c5591d --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java @@ -0,0 +1,25 @@ +package ru.practicum.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Component; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.service.SimilarityService; + +@Slf4j +@Component +@RequiredArgsConstructor +public class UserActionsConsumer { + + private final SimilarityService similarityService; + + @KafkaListener( + topics = "#{kafkaProperties.consumer.topic}", + containerFactory = "kafkaListenerContainerFactory" + ) + public void consumeUserAction(UserActionAvro message) { + log.info("consume user action Kafka: {}", message); + similarityService.processUserAction(message); + } +} diff --git a/stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java b/stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java new file mode 100644 index 0000000..aa49613 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java @@ -0,0 +1,22 @@ +package ru.practicum.service; + +import java.util.HashMap; +import java.util.Map; + +public class MinWeightsMatrix { + + private final Map> minWeightsSums = new HashMap<>(); + + public void put(long eventA, long eventB, double sum) { + long first = Math.min(eventA, eventB); + long second = Math.max(eventA, eventB); + minWeightsSums.computeIfAbsent(first, k -> new HashMap<>()).put(second, sum); + } + + public double get(long eventA, long eventB) { + long first = Math.min(eventA, eventB); + long second = Math.max(eventA, eventB); + return minWeightsSums.getOrDefault(first, Map.of()) + .getOrDefault(second, 0.0); + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java b/stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java new file mode 100644 index 0000000..dd51e96 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java @@ -0,0 +1,110 @@ +package ru.practicum.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Service; +import ru.practicum.config.KafkaProperties; +import ru.practicum.ewm.stats.avro.ActionTypeAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +@Service +public class SimilarityService { + + private final Map> weights = new HashMap<>(); + + private final Map eventWeightsSum = new HashMap<>(); + + private final MinWeightsMatrix minWeightsMatrix = new MinWeightsMatrix(); + + private final KafkaTemplate kafkaTemplate; + private final KafkaProperties props; + + public SimilarityService(KafkaTemplate kafkaTemplate, + KafkaProperties props) { + this.kafkaTemplate = kafkaTemplate; + this.props = props; + } + + public void processUserAction(UserActionAvro action) { + long userId = action.getUserId(); + long eventId = action.getEventId(); + int newWeight = convertActionType(action.getActionType()); + long timestampMillis = action.getTimestamp(); + Instant timestamp = Instant.ofEpochMilli(timestampMillis); + + Map userMap = weights.computeIfAbsent(eventId, e -> new HashMap<>()); + int oldWeight = userMap.getOrDefault(userId, 0); + + if (newWeight <= oldWeight) { + log.debug("Обновление не требуется: userId={}, eventId={}, weight={} <= oldWeight={}", + userId, eventId, newWeight, oldWeight); + return; + } + + userMap.put(userId, newWeight); + + int oldSum = eventWeightsSum.getOrDefault(eventId, 0); + int diff = newWeight - oldWeight; + int updatedSum = oldSum + diff; + eventWeightsSum.put(eventId, updatedSum); + + weights.keySet() + .stream() + .filter(otherEvent -> otherEvent.equals(eventId)) + .forEach(otherEvent -> updatePairSimilarity(eventId, otherEvent, timestamp)); + } + + private void updatePairSimilarity(long eventA, long eventB, Instant timestamp) { + double sMin = calcSMin(eventA, eventB); + minWeightsMatrix.put(eventA, eventB, sMin); + + double sA = eventWeightsSum.getOrDefault(eventA, 0); + double sB = eventWeightsSum.getOrDefault(eventB, 0); + if (sA == 0 || sB == 0) { + + log.debug("Обнаружена нулевая сумма (sA={}, sB={}), пропускающая сходство для событий {} и {}", + sA, sB, eventA, eventB); + return; + } + + float similarity = (float) (sMin / (sA * sB)); + + long first = Math.min(eventA, eventB); + long second = Math.max(eventA, eventB); + + EventSimilarityAvro similarityMsg = EventSimilarityAvro.newBuilder() + .setEventA(first) + .setEventB(second) + .setScore(similarity) + .setTimestamp(timestamp) + .build(); + + kafkaTemplate.send(props.getProducer().getTopic(), similarityMsg); + + log.debug("Обновлено сходство для (A={}, B={}) => {}", first, second, similarity); + } + + private double calcSMin(long eventA, long eventB) { + Map userMapA = weights.getOrDefault(eventA, Map.of()); + Map userMapB = weights.getOrDefault(eventB, Map.of()); + + return userMapA.entrySet().stream() + .filter(e -> userMapB.get(e.getKey()) != null) + .mapToDouble(e -> Math.min(e.getValue(), userMapB.get(e.getKey()))) + .sum(); + } + + private int convertActionType(ActionTypeAvro actionType) { + return switch (actionType) { + case REGISTER -> 2; + case LIKE -> 3; + default -> 1; + }; + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/resources/application.yml b/stats/aggregator/src/main/resources/application.yml new file mode 100644 index 0000000..2b1f593 --- /dev/null +++ b/stats/aggregator/src/main/resources/application.yml @@ -0,0 +1,28 @@ +spring: + application: + name: aggregator + config: + import: "configserver:" + cloud: + config: + fail-fast: true + retry: + useRandomPolicy: true + max-interval: 6000 + discovery: + enabled: true + service-id: config-server + +eureka: + client: + register-with-eureka: true + fetch-registry: true + serviceUrl: + defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ + instance: + preferIpAddress: true + hostname: localhost + instance-id: "${spring.application.name}:${random.value}" + leaseRenewalIntervalInSeconds: 10 +server: + port: 0 \ No newline at end of file diff --git a/stats/analyzer/Dockerfile b/stats/analyzer/Dockerfile new file mode 100644 index 0000000..0ff1817 --- /dev/null +++ b/stats/analyzer/Dockerfile @@ -0,0 +1,5 @@ +FROM eclipse-temurin:21-jre-jammy +VOLUME /tmp +ARG JAR_FILE=target/*.jar +COPY ${JAR_FILE} app.jar +ENTRYPOINT ["sh", "-c", "java ${JAVA_OPTS} -jar /app.jar"] \ No newline at end of file diff --git a/stats/analyzer/pom.xml b/stats/analyzer/pom.xml new file mode 100644 index 0000000..b77554d --- /dev/null +++ b/stats/analyzer/pom.xml @@ -0,0 +1,143 @@ + + + 4.0.0 + + ru.practicum + stats + 0.0.1-SNAPSHOT + + + analyzer + + + 21 + 21 + UTF-8 + 0.0.1-SNAPSHOT + + + + + + net.devh + grpc-server-spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + org.springframework.boot + spring-boot-starter-actuator + + + + org.postgresql + postgresql + runtime + + + + com.h2database + h2 + runtime + + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + org.springframework.boot + spring-boot-starter-test + test + + + ru.practicum + aggregator + 0.0.1-SNAPSHOT + compile + + + + org.projectlombok + lombok + true + + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + + org.springframework.cloud + spring-cloud-starter-config + + + + org.springframework.retry + spring-retry + + + org.springframework.cloud + spring-cloud-openfeign-core + + + org.springframework.cloud + spring-cloud-commons + + + + + org.apache.kafka + kafka-clients + + + + org.springframework.kafka + spring-kafka + + + + ru.practicum + avro-schemas + ${avro-schemas.version} + + + + ru.practicum + proto-schemas + ${avro-schemas.version} + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 16 + 16 + + + + + + \ No newline at end of file diff --git a/stats/stats-server/src/main/java/ru/practicum/StatServer.java b/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java similarity index 79% rename from stats/stats-server/src/main/java/ru/practicum/StatServer.java rename to stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java index 1066613..12c8dd6 100644 --- a/stats/stats-server/src/main/java/ru/practicum/StatServer.java +++ b/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java @@ -1,3 +1,4 @@ + package ru.practicum; import org.springframework.boot.SpringApplication; @@ -8,9 +9,9 @@ @SpringBootApplication @EnableDiscoveryClient @EnableFeignClients -public class StatServer { +public class AnalyzerApplication { public static void main(String[] args) { - SpringApplication.run(StatServer.class, args); + SpringApplication.run(AnalyzerApplication.class, args); } -} \ No newline at end of file +} diff --git a/stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java b/stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java new file mode 100644 index 0000000..b0fa188 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java @@ -0,0 +1,25 @@ +package ru.practicum.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Component; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.service.event.EventSimilarityService; + +@Slf4j +@Component +@RequiredArgsConstructor +public class EventsSimilarityConsumer { + + private final EventSimilarityService eventSimilarityService; + + @KafkaListener( + topics = "${kafka.events-similarity-consumer.topic}", + containerFactory = "eventSimilarityKafkaListenerFactory" + ) + public void consumeEventSimilarity(EventSimilarityAvro msg) { + log.info("Consumed event similarity: {}", msg); + eventSimilarityService.updateEventSimilarity(msg); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java new file mode 100644 index 0000000..9a86cee --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java @@ -0,0 +1,65 @@ +package ru.practicum.config; + +import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; + +import java.util.Map; + +@Configuration +@RequiredArgsConstructor +public class KafkaConfig { + private final KafkaProperties props; + + @Bean + public ConsumerFactory userActionsConsumerFactory() { + final Map config = Map.of( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), + ConsumerConfig.GROUP_ID_CONFIG, props.getUserActionsConsumer().getGroupId(), + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, props.getUserActionsConsumer().getKeyDeserializer(), + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, props.getUserActionsConsumer().getValueDeserializer() + ); + return new DefaultKafkaConsumerFactory<>(config); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory userActionsKafkaListenerFactory() { + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(userActionsConsumerFactory()); + return factory; + } + + @Bean + public ConsumerFactory eventSimilarityConsumerFactory() { + Map config = Map.of( + ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), + ConsumerConfig.GROUP_ID_CONFIG, props.getEventsSimilarityConsumer().getGroupId(), + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, getClassFromString(props.getEventsSimilarityConsumer().getKeyDeserializer()), + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, getClassFromString(props.getEventsSimilarityConsumer().getValueDeserializer()) + ); + return new DefaultKafkaConsumerFactory<>(config); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory eventSimilarityKafkaListenerFactory() { + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(eventSimilarityConsumerFactory()); + return factory; + } + + private Class getClassFromString(String className) { + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Unable to load class: " + className, e); + } + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java new file mode 100644 index 0000000..0e3d8d0 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java @@ -0,0 +1,22 @@ +package ru.practicum.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties(prefix = "kafka") +public class KafkaProperties { + private String bootstrapServers; + private final ConsumerProps userActionsConsumer = new ConsumerProps(); + private final ConsumerProps eventsSimilarityConsumer = new ConsumerProps(); + + @Data + public static class ConsumerProps { + private String topic; + private String groupId; + private String keyDeserializer; + private String valueDeserializer; + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java b/stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java new file mode 100644 index 0000000..a0cca01 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java @@ -0,0 +1,25 @@ +package ru.practicum.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Component; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.service.user.UserActionService; + +@Slf4j +@RequiredArgsConstructor +@Component +public class UserActionsConsumer { + + private final UserActionService userActionService; + + @KafkaListener( + topics = "${kafka.user-actions-consumer.topic}", + containerFactory = "userActionsKafkaListenerFactory" + ) + public void consumeUserActions(UserActionAvro message) { + log.info("user action: {}", message); + userActionService.updateUserAction(message); + } +} diff --git a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java new file mode 100644 index 0000000..27fcb07 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java @@ -0,0 +1,106 @@ +package ru.practicum.controller; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.server.service.GrpcService; +import ru.practicum.ewm.stats.proto.RecommendationsControllerGrpc; +import ru.practicum.ewm.stats.proto.RecommendationsMessages; +import ru.practicum.model.RecommendedEvent; +import ru.practicum.service.RecommendationService; + +import java.util.List; + +@GrpcService +@RequiredArgsConstructor +@Slf4j +public class RecommendationsController extends RecommendationsControllerGrpc.RecommendationsControllerImplBase { + + private final RecommendationService recommendationService; + + @Override + public void getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto requestProto, + StreamObserver streamObserver) { + try { + List recommendedEvents = recommendationService.getSimilarEvents(requestProto); + for (RecommendedEvent recommendedEvent : recommendedEvents) { + RecommendationsMessages.RecommendedEventProto recommendedEventProto = + RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(recommendedEvent.eventId()) + .setScore(recommendedEvent.score()) + .build(); + streamObserver.onNext(recommendedEventProto); + } + streamObserver.onCompleted(); + } catch (IllegalArgumentException e) { + log.error("Illegal argument in getSimilarEvents: {}", e.getMessage(), e); + streamObserver.onError( + new StatusRuntimeException( + Status.INVALID_ARGUMENT.withDescription("unexpected error occurred").withCause(e)) + ); + } catch (Exception e) { + log.error("unexpected error occurred in getSimilarEvents: {}", e.getMessage(), e); + streamObserver.onError( + new StatusRuntimeException(Status.UNKNOWN.withDescription("unexpected error occurred").withCause(e)) + ); + } + } + + @Override + public void getRecommendationsForUser( + RecommendationsMessages.UserPredictionsRequestProto requestProto, + StreamObserver streamObserver + ) { + try { + List events = recommendationService.getRecommendationsForUser(requestProto); + for (RecommendedEvent event : events) { + RecommendationsMessages.RecommendedEventProto recommendedEventProto = + RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(event.eventId()) + .setScore(event.score()) + .build(); + streamObserver.onNext(recommendedEventProto); + } + streamObserver.onCompleted(); + } catch (IllegalArgumentException e) { + log.error("Illegal argument in getRecommendationsForUser: {}", e.getMessage(), e); + streamObserver.onError( + new StatusRuntimeException( + Status.INVALID_ARGUMENT.withDescription("unexpected error occurred").withCause(e)) + ); + } catch (Exception e) { + log.error("unexpected error occurred in getRecommendationsForUser: {}", e.getMessage(), e); + streamObserver.onError( + new StatusRuntimeException(Status.UNKNOWN.withDescription("unexpected error occurred").withCause(e)) + ); + } + } + + @Override + public void getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto requestProto, + StreamObserver streamObserver) { + try { + List events = recommendationService.getInteractionsCount(requestProto); + for (RecommendedEvent event : events) { + RecommendationsMessages.RecommendedEventProto eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(event.eventId()) + .setScore(event.score()) + .build(); + streamObserver.onNext(eventProto); + } + streamObserver.onCompleted(); + } catch (IllegalArgumentException e) { + log.error("Illegal argument in getInteractionsCount: {}", e.getMessage(), e); + streamObserver.onError( + new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e)) + ); + } catch (Exception e) { + log.error("unexpected error occurred in getInteractionsCount: {}", e.getMessage(), e); + streamObserver.onError( + new StatusRuntimeException(Status.UNKNOWN.withDescription("unexpected error occurred").withCause(e)) + ); + } + } +} diff --git a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java new file mode 100644 index 0000000..3976ebd --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java @@ -0,0 +1,28 @@ +package ru.practicum.model; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Entity +@Table(name = "events_similarity") +public class EventSimilarity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private Long eventA; + private Long eventB; + private Float score; + private Instant timestamp; +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java b/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java new file mode 100644 index 0000000..47cd704 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java @@ -0,0 +1,7 @@ +package ru.practicum.model; + +public record RecommendedEvent( + long eventId, + double score +) { +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java new file mode 100644 index 0000000..b7f6f32 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java @@ -0,0 +1,35 @@ +package ru.practicum.model; + + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Instant; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@Builder +@Entity +@Table(name = "user_actions") +public class UserAction { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private Long userId; + private Long eventId; + + private Double maxWeight; + + private Instant lastInteraction; +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java b/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java new file mode 100644 index 0000000..e5a121d --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java @@ -0,0 +1,11 @@ +package ru.practicum.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import ru.practicum.model.EventSimilarity; + +import java.util.List; + +public interface EventSimilarityRepository extends JpaRepository { + + List findByEventAOrEventB(Long eventA, Long eventB); +} diff --git a/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java b/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java new file mode 100644 index 0000000..3e346a3 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java @@ -0,0 +1,15 @@ +package ru.practicum.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import ru.practicum.model.UserAction; + +import java.util.List; + +public interface UserActionRepository extends JpaRepository { + + UserAction findByUserIdAndEventId(Long userId, Long eventId); + + List findByUserId(Long userId); + + List findByEventId(Long eventId); +} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java new file mode 100644 index 0000000..4eec7f0 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java @@ -0,0 +1,112 @@ +package ru.practicum.service; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import org.springframework.stereotype.Service; +import ru.practicum.ewm.stats.proto.RecommendationsMessages; +import ru.practicum.model.EventSimilarity; +import ru.practicum.model.RecommendedEvent; +import ru.practicum.model.UserAction; +import ru.practicum.repository.EventSimilarityRepository; +import ru.practicum.repository.UserActionRepository; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RecommendationService { + + final UserActionRepository userActionRepo; + final EventSimilarityRepository similarityRepo; + + public List getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto requestProto) { + long eventId = requestProto.getEventId(); + long userId = requestProto.getUserId(); + int maxResults = requestProto.getMaxResults(); + + Set interacted = userInteracted(userId); + List result, recList = new ArrayList<>(); + + similarityRepo.findByEventAOrEventB(eventId, eventId) + .forEach(e -> { + long other = (e.getEventA() == eventId) ? e.getEventB() : e.getEventA(); + if (!interacted.contains(other)) { + recList.add(new RecommendedEvent(other, e.getScore())); + } + }); + result = recList.stream() + .sorted(Comparator.comparingDouble(RecommendedEvent::score).reversed()).toList(); + + return result.size() <= maxResults ? result : result.subList(0, maxResults); + } + + public List getRecommendationsForUser(RecommendationsMessages.UserPredictionsRequestProto request) { + long userId = request.getUserId(); + int maxRes = request.getMaxResults(); + + List all = userActionRepo.findByUserId(userId); + if (all.isEmpty()) { + return Collections.emptyList(); + } + + all.sort((a,b) -> b.getLastInteraction().compareTo(a.getLastInteraction())); + + int min = Math.min(5, all.size()); + List recent = all.subList(0, min); + + Set interacted = userInteracted(userId); + + Map bestScoreMap = new HashMap<>(); + for (UserAction r : recent) { + long ev = r.getEventId(); + List simList = similarityRepo.findByEventAOrEventB(ev, ev); + for (EventSimilarity e : simList) { + long other = (e.getEventA() == ev) ? e.getEventB() : e.getEventA(); + if (interacted.contains(other)) { + continue; + } + float oldVal = bestScoreMap.getOrDefault(other, 0f); + if (e.getScore() > oldVal) { + bestScoreMap.put(other, e.getScore()); + } + } + } + + return bestScoreMap.entrySet().stream() + .map(e -> new RecommendedEvent(e.getKey(), e.getValue())) + .sorted(Comparator.comparingDouble(RecommendedEvent::score).reversed()) + .limit(maxRes) + .collect(Collectors.toList()); + } + + public List getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto request) { + List events = request.getEventIdList(); + List result = new ArrayList<>(); + + for (Long e : events) { + List list = userActionRepo.findByEventId(e); + double sum = 0.0; + for (UserAction uae : list) { + sum += uae.getMaxWeight(); + } + result.add(new RecommendedEvent(e, (float) sum)); + } + return result; + } + + private Set userInteracted(long userId) { + return userActionRepo.findByUserId(userId) + .stream() + .map(UserAction::getEventId) + .collect(Collectors.toSet()); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java new file mode 100644 index 0000000..0686ad7 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java @@ -0,0 +1,7 @@ +package ru.practicum.service.event; + +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; + +public interface EventSimilarityService { + void updateEventSimilarity(EventSimilarityAvro eventSimilarityAvro); +} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java new file mode 100644 index 0000000..814fdd0 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java @@ -0,0 +1,50 @@ +package ru.practicum.service.event; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.model.EventSimilarity; +import ru.practicum.repository.EventSimilarityRepository; + +import java.time.Instant; + +@Slf4j +@Service +@RequiredArgsConstructor +public class EventSimilarityServiceImpl implements EventSimilarityService { + + private final EventSimilarityRepository eventSimilarityRepository; + + @Override + public void updateEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { + long eventA = eventSimilarityAvro.getEventA(); + long eventB = eventSimilarityAvro.getEventB(); + float score = eventSimilarityAvro.getScore(); + Instant timestamp = eventSimilarityAvro.getTimestamp(); + + EventSimilarity existingEventSimilarity = findPair(eventA, eventB); + + if (existingEventSimilarity == null) { + existingEventSimilarity = new EventSimilarity(); + existingEventSimilarity.setEventA(eventA); + existingEventSimilarity.setEventB(eventB); + existingEventSimilarity.setScore(score); + existingEventSimilarity.setTimestamp(timestamp); + eventSimilarityRepository.save(existingEventSimilarity); + } else { + existingEventSimilarity.setScore(score); + existingEventSimilarity.setTimestamp(timestamp); + eventSimilarityRepository.save(existingEventSimilarity); + } + } + + private EventSimilarity findPair(long eventA, long eventB) { + return eventSimilarityRepository.findByEventAOrEventB(eventA, eventB) + .stream() + .filter(e -> (e.getEventA().equals(eventA) && e.getEventB().equals(eventB)) + || (e.getEventA().equals(eventB) && e.getEventB().equals(eventA))) + .findFirst() + .orElse(null); + } +} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java new file mode 100644 index 0000000..59ecda9 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java @@ -0,0 +1,7 @@ +package ru.practicum.service.user; + +import ru.practicum.ewm.stats.avro.UserActionAvro; + +public interface UserActionService { + void updateUserAction(UserActionAvro userActionAvro); +} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java new file mode 100644 index 0000000..0e46784 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java @@ -0,0 +1,58 @@ +package ru.practicum.service.user; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import ru.practicum.ewm.stats.avro.ActionTypeAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.model.UserAction; +import ru.practicum.repository.UserActionRepository; + +import java.time.Instant; + +@Service +@Slf4j +@RequiredArgsConstructor +public class UserActionServiceImpl implements UserActionService { + + private final UserActionRepository userActionRepository; + + @Override + public void updateUserAction(UserActionAvro userActionAvro) { + long userId = userActionAvro.getUserId(); + long eventId = userActionAvro.getEventId(); + double newWeight = convertWeight(userActionAvro.getActionType()); + long timestamp = userActionAvro.getTimestamp(); + + Instant interactionTime = Instant.ofEpochMilli(timestamp); + + UserAction userAction = userActionRepository.findByUserIdAndEventId(userId, eventId); + + if (userAction == null) { + userAction = new UserAction(); + userAction.setUserId(userId); + userAction.setEventId(eventId); + userAction.setMaxWeight(newWeight); + userAction.setLastInteraction(interactionTime); + userActionRepository.save(userAction); + return; + } + if (newWeight > userAction.getMaxWeight()) { + userAction.setMaxWeight(newWeight); + } + if (interactionTime.isAfter(userAction.getLastInteraction())) { + userAction.setLastInteraction(interactionTime); + } + userActionRepository.save(userAction); + + + } + + private double convertWeight(ActionTypeAvro actionType) { + return switch (actionType) { + case REGISTER -> 0.8; + case LIKE -> 1; + default -> 0.4; + }; + } +} diff --git a/stats/stats-server/src/main/resources/application.yml b/stats/analyzer/src/main/resources/application.yml similarity index 52% rename from stats/stats-server/src/main/resources/application.yml rename to stats/analyzer/src/main/resources/application.yml index e5a978d..3d0ef3b 100644 --- a/stats/stats-server/src/main/resources/application.yml +++ b/stats/analyzer/src/main/resources/application.yml @@ -1,28 +1,17 @@ spring: application: - name: stats-server + name: analyzer config: import: optional:configserver:http://config-server:9091 cloud: config: - enabled: false - datasource: - driverClassName: org.postgresql.Driver - url: jdbc:postgresql://ewm-db:5432/ewm-stats - username: root - password: root - jpa: - hibernate: - ddl-auto: none - database-platform: org.hibernate.dialect.PostgreSQLDialect - generate-ddl: false - properties: - hibernate: - format_sql: true - show-sql: false - sql: - init: - mode: always + fail-fast: true + retry: + useRandomPolicy: true + max-interval: 6000 + discovery: + enabled: true + service-id: config-server eureka: client: @@ -36,4 +25,4 @@ eureka: instance-id: "${spring.application.name}:${random.value}" lease-renewal-interval-in-seconds: 10 server: - port: 9090 \ No newline at end of file + port: 0 \ No newline at end of file diff --git a/stats/collector/Dockerfile b/stats/collector/Dockerfile new file mode 100644 index 0000000..0ff1817 --- /dev/null +++ b/stats/collector/Dockerfile @@ -0,0 +1,5 @@ +FROM eclipse-temurin:21-jre-jammy +VOLUME /tmp +ARG JAR_FILE=target/*.jar +COPY ${JAR_FILE} app.jar +ENTRYPOINT ["sh", "-c", "java ${JAVA_OPTS} -jar /app.jar"] \ No newline at end of file diff --git a/stats/collector/pom.xml b/stats/collector/pom.xml new file mode 100644 index 0000000..d251941 --- /dev/null +++ b/stats/collector/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + ru.practicum + stats + 0.0.1-SNAPSHOT + + + collector + + + 21 + 21 + UTF-8 + 0.0.1-SNAPSHOT + 0.0.1-SNAPSHOT + + + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-starter-actuator + + + + net.devh + grpc-server-spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-validation + + + + org.springframework.boot + spring-boot-starter-aop + + + + org.projectlombok + lombok + provided + + + + org.springframework.cloud + spring-cloud-starter-config + + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + + org.springframework.kafka + spring-kafka + + + + ru.practicum + avro-schemas + ${avro-schemas.version} + + + + ru.practicum + proto-schemas + ${proto-schemas.version} + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/CollectorApplication.java b/stats/collector/src/main/java/ru/practicum/CollectorApplication.java new file mode 100644 index 0000000..deb088a --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/CollectorApplication.java @@ -0,0 +1,15 @@ +package ru.practicum; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; + +@SpringBootApplication +@EnableDiscoveryClient +@ConfigurationPropertiesScan +public class CollectorApplication { + public static void main(String[] args) { + SpringApplication.run(CollectorApplication.class, args); + } +} diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java new file mode 100644 index 0000000..d8e861b --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java @@ -0,0 +1,38 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.Map; + +@Getter +@Setter +@Configuration +@RequiredArgsConstructor +public class KafkaProducerConfig { + + private final KafkaProperties kafkaProperties; + + @Bean + public ProducerFactory kafkaProducer() { + Map props = Map.of( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers(), + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getKeySerializer(), + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getValueSerializer() + ); + return new DefaultKafkaProducerFactory<>(props); + } + + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(kafkaProducer()); + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java new file mode 100644 index 0000000..fe43fd3 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java @@ -0,0 +1,29 @@ +package ru.practicum.config; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties("spring.kafka") +@FieldDefaults(level = AccessLevel.PRIVATE) +public class KafkaProperties { + + String bootstrapServers; + + Producer producer = new Producer(); + + @Value("${collector.kafka.topic}") + String userActionsTopic; + + @Data + @FieldDefaults(level = AccessLevel.PRIVATE) + public static class Producer { + String keySerializer; + String valueSerializer; + } +} diff --git a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java new file mode 100644 index 0000000..9d0e3e6 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java @@ -0,0 +1,29 @@ +package ru.practicum.mapper; + +import ru.practicum.ewm.stats.avro.ActionTypeAvro; +import ru.practicum.ewm.stats.proto.ActionTypeProto; +import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +public class UserActionMapper { + + public static UserActionAvro toAvro(UserActionProto userActionProto) { + long timestampMillis = userActionProto.getTimestamp().getSeconds() * 1000 + + userActionProto.getTimestamp().getNanos() / 1_000_000; + + return UserActionAvro.newBuilder() + .setUserId(userActionProto.getUserId()) + .setEventId(userActionProto.getEventId()) + .setActionType(toAvroActionType(userActionProto.getActionType())) + .setTimestamp(timestampMillis) + .build(); + } + + private static ActionTypeAvro toAvroActionType(ActionTypeProto protoType) { + return switch (protoType) { + case ACTION_REGISTER -> ActionTypeAvro.REGISTER; + case ACTION_LIKE -> ActionTypeAvro.LIKE; + default -> ActionTypeAvro.VIEW; + }; + } +} diff --git a/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java b/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java new file mode 100644 index 0000000..26f5e8b --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java @@ -0,0 +1,20 @@ +package ru.practicum.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.kafka.core.KafkaTemplate; +import ru.practicum.config.KafkaProperties; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class KafkaMessageProducer implements MessageProducer { + + private final KafkaTemplate kafkaTemplate; + private final KafkaProperties properties; + + @Override + public void sendUserAction(UserActionAvro userActionAvro) { + kafkaTemplate.send(properties.getUserActionsTopic(), userActionAvro); + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/MessageProducer.java b/stats/collector/src/main/java/ru/practicum/service/MessageProducer.java new file mode 100644 index 0000000..e259c37 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/service/MessageProducer.java @@ -0,0 +1,7 @@ +package ru.practicum.service; + +import ru.practicum.ewm.stats.avro.UserActionAvro; + +public interface MessageProducer { + void sendUserAction(UserActionAvro userActionAvro); +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/UserActionController.java b/stats/collector/src/main/java/ru/practicum/service/UserActionController.java new file mode 100644 index 0000000..ab25fe8 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/service/UserActionController.java @@ -0,0 +1,41 @@ +package ru.practicum.service; + +import com.google.protobuf.Empty; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.server.service.GrpcService; +import ru.practicum.ewm.stats.proto.UserActionControllerGrpc; +import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.mapper.UserActionMapper; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +@Slf4j +@GrpcService +@RequiredArgsConstructor +public class UserActionController extends UserActionControllerGrpc.UserActionControllerImplBase { + + private final MessageProducer messageProducer; + + @Override + public void collectUserAction(UserActionProto request, StreamObserver responseObserver) { + try { + UserActionAvro userActionAvro = UserActionMapper.toAvro(request); + messageProducer.sendUserAction(userActionAvro); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } catch (IllegalArgumentException e) { + log.error("IllegalArgumentException collectUserAction: {}", e.getMessage(), e); + responseObserver.onError( + new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e)) + ); + } catch (Exception e) { + log.error("error collectUserAction: {}", e.getMessage(), e); + responseObserver.onError( + new StatusRuntimeException(Status.UNKNOWN.withDescription("error").withCause(e)) + ); + } + } +} \ No newline at end of file diff --git a/stats/collector/src/main/resources/application.yml b/stats/collector/src/main/resources/application.yml new file mode 100644 index 0000000..e1855d9 --- /dev/null +++ b/stats/collector/src/main/resources/application.yml @@ -0,0 +1,28 @@ +spring: + application: + name: collector + config: + import: optional:configserver:http://config-server:9091 + cloud: + config: + fail-fast: true + retry: + useRandomPolicy: true + max-interval: 6000 + discovery: + enabled: true + service-id: config-server + +eureka: + client: + register-with-eureka: true + fetch-registry: true + serviceUrl: + defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ + instance: + prefer-ip-address: true + hostname: localhost + instance-id: "${spring.application.name}:${random.value}" + lease-renewal-interval-in-seconds: 10 +server: + port: 0 \ No newline at end of file diff --git a/stats/pom.xml b/stats/pom.xml index 321657b..7c3a106 100644 --- a/stats/pom.xml +++ b/stats/pom.xml @@ -17,8 +17,10 @@ stats-client - stats-dto - stats-server + collector + aggregator + serialization + analyzer diff --git a/stats/serialization/avro-schemas/pom.xml b/stats/serialization/avro-schemas/pom.xml new file mode 100644 index 0000000..27077af --- /dev/null +++ b/stats/serialization/avro-schemas/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + ru.practicum + serialization + 0.0.1-SNAPSHOT + + + avro-schemas + + + 21 + 21 + UTF-8 + + + + + org.apache.avro + avro + ${avro.version} + + + + org.apache.kafka + kafka-clients + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.avro + avro-maven-plugin + + + schemas + generate-sources + + idl-protocol + + + ${project.basedir}/src/main/avro + ${project.build.directory}/generated-sources/avro + String + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.5.0 + + + add-source + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources + + + + + + + + + \ No newline at end of file diff --git a/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl b/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl new file mode 100644 index 0000000..60267df --- /dev/null +++ b/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl @@ -0,0 +1,10 @@ +@namespace("ru.practicum.ewm.stats.avro") +protocol EventSimilarityProtocol { + + record EventSimilarityAvro { + long eventA; + long eventB; + float score; + timestamp_ms timestamp; + } +} \ No newline at end of file diff --git a/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl b/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl new file mode 100644 index 0000000..9920089 --- /dev/null +++ b/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl @@ -0,0 +1,16 @@ +@namespace("ru.practicum.ewm.stats.avro") +protocol UserActionProtocol { + + enum ActionTypeAvro { + VIEW, + REGISTER, + LIKE + } + + record UserActionAvro { + long userId; + long eventId; + ActionTypeAvro actionType; + long timestamp; + } +} \ No newline at end of file diff --git a/stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroDeserializer.java b/stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroDeserializer.java new file mode 100644 index 0000000..2431732 --- /dev/null +++ b/stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroDeserializer.java @@ -0,0 +1,51 @@ +package ru.practicum; + +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.common.serialization.Deserializer; + +import java.io.ByteArrayInputStream; +import java.util.Map; + +public class AvroDeserializer implements Deserializer { + + private final Class targetType; + + public AvroDeserializer(Class targetType) { + this.targetType = targetType; + } + + public AvroDeserializer() { + this.targetType = null; + } + + @Override + public void configure(Map configs, boolean isKey) { + + } + + @Override + public T deserialize(String topic, byte[] data) { + if (data == null) { + return null; + } + + if (targetType == null) { + throw new IllegalStateException("targetType is undefined in AvroDeserializer"); + } + + try { + SpecificDatumReader datumReader = new SpecificDatumReader<>(targetType); + ByteArrayInputStream in = new ByteArrayInputStream(data); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(in, null); + return datumReader.read(null, decoder); + } catch (Exception e) { + throw new RuntimeException("Couldn't deserialize avro message for topic " + topic, e); + } + } + + @Override + public void close(){} +} diff --git a/stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroSerializer.java b/stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroSerializer.java new file mode 100644 index 0000000..e958572 --- /dev/null +++ b/stats/serialization/avro-schemas/src/main/java/ru/practicum/AvroSerializer.java @@ -0,0 +1,39 @@ +package ru.practicum; + +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Serializer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; + +public class AvroSerializer implements Serializer { + + @Override + public void configure(Map configs, boolean isKey) {} + + @Override + public byte[] serialize(String topic, T data) { + if (data == null) { + return null; + } + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + DatumWriter writer = new SpecificDatumWriter<>(data.getSchema()); + writer.write(data, encoder); + encoder.flush(); + return out.toByteArray(); + } catch (IOException e) { + throw new SerializationException("Error has occurred during serialization avro-message", e); + } + } + + @Override + public void close(){} +} diff --git a/stats/serialization/pom.xml b/stats/serialization/pom.xml new file mode 100644 index 0000000..3f32dfb --- /dev/null +++ b/stats/serialization/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + ru.practicum + stats + 0.0.1-SNAPSHOT + + + serialization + pom + + avro-schemas + proto-schemas + + + + 21 + 21 + UTF-8 + + + + + \ No newline at end of file diff --git a/stats/serialization/proto-schemas/pom.xml b/stats/serialization/proto-schemas/pom.xml new file mode 100644 index 0000000..e1dd02e --- /dev/null +++ b/stats/serialization/proto-schemas/pom.xml @@ -0,0 +1,93 @@ + + + 4.0.0 + + ru.practicum + serialization + 0.0.1-SNAPSHOT + + + proto-schemas + + + 21 + 21 + UTF-8 + + + + + io.grpc + grpc-stub + + + + io.grpc + grpc-protobuf + + + + jakarta.annotation + jakarta.annotation-api + 1.3.5 + true + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + io.github.ascopes + protobuf-maven-plugin + + + ${protobuf.version} + + + + io.grpc + protoc-gen-grpc-java + ${grpc.version} + + + + + + + + generate + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.5.0 + + + add-source + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/protobuf + + + + + + + + + \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto b/stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto new file mode 100644 index 0000000..c2895c8 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package ru.practicum.ewm.stats.proto; + +import "messages/recommendations_messages.proto"; +import "google/protobuf/empty.proto"; + +service RecommendationsController { + rpc GetRecommendationsForUser(UserPredictionsRequestProto) + returns (stream RecommendedEventProto); + + rpc GetSimilarEvents(SimilarEventsRequestProto) + returns (stream RecommendedEventProto); + + rpc GetInteractionsCount(InteractionsCountRequestProto) + returns (stream RecommendedEventProto); +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto b/stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto new file mode 100644 index 0000000..0108be8 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package ru.practicum.ewm.stats.proto; + +import "google/protobuf/empty.proto"; +import "messages/user_action.proto"; + +service UserActionController { + rpc CollectUserAction(UserActionProto) returns (google.protobuf.Empty); +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto b/stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto new file mode 100644 index 0000000..f4c7216 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package ru.practicum.ewm.stats.proto; + +import "google/protobuf/timestamp.proto"; + +message UserPredictionsRequestProto { + int64 user_id = 1; + int32 max_results = 2; +} + +message SimilarEventsRequestProto { + int64 event_id = 1; + int64 user_id = 2; + int32 max_results = 3; +} + +message InteractionsCountRequestProto { + repeated int64 event_id = 1; +} + +message RecommendedEventProto { + int64 event_id = 1; + double score = 2; +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto b/stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto new file mode 100644 index 0000000..2f91b96 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package ru.practicum.ewm.stats.proto; + +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "ru.practicum.ewm.stats.proto"; + +enum ActionTypeProto { + ACTION_VIEW = 0; + ACTION_REGISTER = 1; + ACTION_LIKE = 2; +} + +message UserActionProto { + int64 user_id = 1; + int64 event_id = 2; + ActionTypeProto action_type = 3; + google.protobuf.Timestamp timestamp = 4; +} \ No newline at end of file diff --git a/stats/stats-client/pom.xml b/stats/stats-client/pom.xml index b110bde..c53c1d0 100644 --- a/stats/stats-client/pom.xml +++ b/stats/stats-client/pom.xml @@ -16,35 +16,48 @@ 21 21 UTF-8 + 2.13.1.RELEASE + - ru.practicum - stats-dto - 0.0.1-SNAPSHOT + org.springframework.boot + spring-boot-starter-web org.springframework.boot - spring-boot-starter-web + spring-boot-starter-actuator + + + + net.devh + grpc-client-spring-boot-starter + ${grpc.spring.boot.starter.version} - org.springframework - spring-web + io.grpc + grpc-stub + + + + org.springframework.boot + spring-boot-starter-aop org.projectlombok lombok - true + provided + - org.springframework.cloud - spring-cloud-openfeign-core + ru.practicum + proto-schemas + 0.0.1-SNAPSHOT - \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java b/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java new file mode 100644 index 0000000..0916c5d --- /dev/null +++ b/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java @@ -0,0 +1,78 @@ +package ru.practicum; + +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.client.inject.GrpcClient; +import org.springframework.stereotype.Service; +import ru.practicum.ewm.stats.proto.RecommendationsControllerGrpc; +import ru.practicum.ewm.stats.proto.RecommendationsMessages; + +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +@Slf4j +@Service +public class AnalyzerClient { + + @GrpcClient("analyzer") + private RecommendationsControllerGrpc.RecommendationsControllerBlockingStub analyzerStub; + + public Stream getSimilarEvents( + long eventId, long userId, int maxResults) { + try { + log.info("Fetching similar events: eventId={}, userId={}, maxResults={}", eventId, userId, maxResults); + RecommendationsMessages.SimilarEventsRequestProto requestProto = + RecommendationsMessages.SimilarEventsRequestProto.newBuilder() + .setEventId(eventId) + .setUserId(userId) + .setMaxResults(maxResults) + .build(); + Iterator iterator = analyzerStub.getSimilarEvents(requestProto); + return toStream(iterator); + } catch (Exception e) { + log.error("Error occurred while fetching similar events: eventId={}, userId={}, maxResults={}", + eventId, userId, maxResults); + return Stream.empty(); + } + } + + public Stream getRecommendationsForUser(long userId, int maxResults) { + try { + log.info("Fetching recommendations for user : userId={}, maxResults={}", userId, maxResults); + RecommendationsMessages.UserPredictionsRequestProto requestProto = + RecommendationsMessages.UserPredictionsRequestProto.newBuilder() + .setUserId(userId) + .setMaxResults(maxResults) + .build(); + Iterator iterator = analyzerStub.getRecommendationsForUser(requestProto); + return toStream(iterator); + } catch (Exception e) { + log.error("Error occurred while fetching recommendations for user : userId={}, maxResults={}", userId, maxResults); + return Stream.empty(); + } + } + + public Stream getInteractionsCount(Iterable eventIds) { + try { + log.info("Fetching interactions count for events"); + RecommendationsMessages.InteractionsCountRequestProto.Builder builder = + RecommendationsMessages.InteractionsCountRequestProto.newBuilder(); + eventIds.forEach(builder::addEventId); + RecommendationsMessages.InteractionsCountRequestProto requestProto = builder.build(); + Iterator iterator = analyzerStub.getInteractionsCount(requestProto); + return toStream(iterator); + } catch (Exception e) { + log.error("Error occurred while fetching interactions count", e); + return Stream.empty(); + } + } + + private Stream toStream(Iterator iterator) { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), + false + ); + } +} diff --git a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java new file mode 100644 index 0000000..c4a3ac1 --- /dev/null +++ b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java @@ -0,0 +1,55 @@ +package ru.practicum; + +import com.google.protobuf.Empty; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.client.inject.GrpcClient; +import org.springframework.stereotype.Service; +import ru.practicum.ewm.stats.proto.ActionTypeProto; +import ru.practicum.ewm.stats.proto.UserActionControllerGrpc; +import ru.practicum.ewm.stats.proto.UserActionProto; + +import java.time.Instant; + +@Slf4j +@Service +public class CollectorClient { + + @GrpcClient("collector") + private UserActionControllerGrpc.UserActionControllerBlockingStub collectorStub; + + public void sendUserAction(long userId, long eventId, ActionTypeProto actionTypeProto) { + try { + log.info("Sending user action: userId={}, eventId={}, actionType={}", userId, eventId, actionTypeProto); + long secondes = Instant.now().getEpochSecond(); + int nanos = Instant.now().getNano(); + + UserActionProto userActionProto = UserActionProto.newBuilder() + .setUserId(userId) + .setEventId(eventId) + .setActionType(actionTypeProto) + .setTimestamp( + com.google.protobuf.Timestamp.newBuilder() + .setSeconds(secondes) + .setNanos(nanos) + ) + .build(); + collectorStub.collectUserAction(userActionProto); + log.info("sendUserAction -> Collector answered"); + } catch (Exception e) { + log.error("Ошибка при отправке действия пользователя: userId={}, eventId={}, actionType={}", + userId, eventId, actionTypeProto, e); + } + } + + public void sendEventView(long userId, long eventId) { + sendUserAction(userId, eventId, ActionTypeProto.ACTION_VIEW); + } + + public void sendEventLike(long userId, long eventId) { + sendUserAction(userId, eventId, ActionTypeProto.ACTION_LIKE); + } + + public void sendEventRegistration(long userId, long eventId) { + sendUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER); + } +} diff --git a/stats/stats-client/src/main/java/ru/practicum/StatClient.java b/stats/stats-client/src/main/java/ru/practicum/StatClient.java index b371b8d..c95620a 100644 --- a/stats/stats-client/src/main/java/ru/practicum/StatClient.java +++ b/stats/stats-client/src/main/java/ru/practicum/StatClient.java @@ -1,24 +1,24 @@ -package ru.practicum; - -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; -import ru.practicum.dto.EndpointHitDto; -import ru.practicum.dto.ViewStatsDto; - -import java.util.List; - -@FeignClient(name = "stats-server") -public interface StatClient { - - @PostMapping("/hit") - void saveHit(@RequestBody EndpointHitDto hitDto); - - @GetMapping("stats") - List getStats(@RequestParam(defaultValue = "") String start, - @RequestParam(defaultValue = "") String end, - @RequestParam(defaultValue = "") List uris, - @RequestParam(defaultValue = "false") Boolean unique); -} \ No newline at end of file +//package ru.practicum; +// +//import org.springframework.cloud.openfeign.FeignClient; +//import org.springframework.web.bind.annotation.GetMapping; +//import org.springframework.web.bind.annotation.PostMapping; +//import org.springframework.web.bind.annotation.RequestBody; +//import org.springframework.web.bind.annotation.RequestParam; +//import ru.practicum.dto.EndpointHitDto; +//import ru.practicum.dto.ViewStatsDto; +// +//import java.util.List; +// +//@FeignClient(name = "stats-server") +//public interface StatClient { +// +// @PostMapping("/hit") +// void saveHit(@RequestBody EndpointHitDto hitDto); +// +// @GetMapping("stats") +// List getStats(@RequestParam(defaultValue = "") String start, +// @RequestParam(defaultValue = "") String end, +// @RequestParam(defaultValue = "") List uris, +// @RequestParam(defaultValue = "false") Boolean unique); +//} \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java b/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java index 40afb07..2ced547 100644 --- a/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java +++ b/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java @@ -1,34 +1,34 @@ -package ru.practicum; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import ru.practicum.dto.EndpointHitDto; -import ru.practicum.dto.ViewStatsDto; - -import java.util.Collections; -import java.util.List; - -@Slf4j -@Component -@RequiredArgsConstructor -public class StatServiceClient { - - private final StatClient statClient; - - public void saveHit(EndpointHitDto dto) { - statClient.saveHit(dto); - } - - public List getStats(String start, - String end, - List uris, - Boolean unique) { - try { - return statClient.getStats(start, end, uris, unique); - } catch (Exception e) { - log.warn("Failed to get stats: {}", e.getMessage()); - } - return Collections.emptyList(); - } -} \ No newline at end of file +//package ru.practicum; +// +//import lombok.RequiredArgsConstructor; +//import lombok.extern.slf4j.Slf4j; +//import org.springframework.stereotype.Component; +//import ru.practicum.dto.EndpointHitDto; +//import ru.practicum.dto.ViewStatsDto; +// +//import java.util.Collections; +//import java.util.List; +// +//@Slf4j +//@Component +//@RequiredArgsConstructor +//public class StatServiceClient { +// +// private final StatClient statClient; +// +// public void saveHit(EndpointHitDto dto) { +// statClient.saveHit(dto); +// } +// +// public List getStats(String start, +// String end, +// List uris, +// Boolean unique) { +// try { +// return statClient.getStats(start, end, uris, unique); +// } catch (Exception e) { +// log.warn("Failed to get stats: {}", e.getMessage()); +// } +// return Collections.emptyList(); +// } +//} \ No newline at end of file diff --git a/stats/stats-client/src/main/resources/application.properties b/stats/stats-client/src/main/resources/application.properties deleted file mode 100644 index f181fe5..0000000 --- a/stats/stats-client/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -client.url=http://stats-server:9090 \ No newline at end of file diff --git a/stats/stats-client/src/main/resources/application.yml b/stats/stats-client/src/main/resources/application.yml new file mode 100644 index 0000000..cb75a5d --- /dev/null +++ b/stats/stats-client/src/main/resources/application.yml @@ -0,0 +1,13 @@ +grpc: + client: + collector: + address: 'discovery:///collector' + enableKeepAlive: true + keepAliveWithoutCalls: true + negotiationType: plaintext + + analyzer: + address: 'discovery:///analyzer' + enableKeepAlive: true + keepAliveWithoutCalls: true + negotiationType: plaintext \ No newline at end of file diff --git a/stats/stats-dto/pom.xml b/stats/stats-dto/pom.xml deleted file mode 100644 index e810cb1..0000000 --- a/stats/stats-dto/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - 4.0.0 - - ru.practicum - stats - 0.0.1-SNAPSHOT - - - stats-dto - - - 21 - 21 - UTF-8 - - - - - org.springframework.boot - spring-boot-starter-validation - - - org.projectlombok - lombok - provided - - - org.springframework.boot - spring-boot-starter-actuator - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - \ No newline at end of file diff --git a/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitDto.java b/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitDto.java deleted file mode 100644 index 79babb3..0000000 --- a/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitDto.java +++ /dev/null @@ -1,48 +0,0 @@ -package ru.practicum.dto; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.Pattern; -import jakarta.validation.constraints.Size; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.FieldDefaults; - -/** - * The type Endpoint hit dto. - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class EndpointHitDto { - @JsonProperty(access = JsonProperty.Access.READ_ONLY) - Integer id; - @Size(max = 255) - String app; - @Size(max = 2048) - String uri; - @Pattern( - regexp = "^((25[0-5]|2[0-4]\\d|1?\\d\\d?)\\.){3}(25[0-5]|2[0-4]\\d|1?\\d\\d?)$", - message = "Неверный формат IP-адреса" - ) - String ip; - String timestamp; - - /** - * Instantiates a new Endpoint hit dto. - * - * @param app the app - * @param uri the uri - * @param ip the ip - * @param timestamp the timestamp - */ -// Дополнительный конструктор без id - public EndpointHitDto(String app, String uri, String ip, String timestamp) { - this.app = app; - this.uri = uri; - this.ip = ip; - this.timestamp = timestamp; - } -} diff --git a/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitResponseDto.java b/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitResponseDto.java deleted file mode 100644 index cd1f8ab..0000000 --- a/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitResponseDto.java +++ /dev/null @@ -1,21 +0,0 @@ -package ru.practicum.dto; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.FieldDefaults; - -/** - * The type Endpoint hit response dto. - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class EndpointHitResponseDto { - - String app; - String uri; - String ip; -} diff --git a/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitSaveRequestDto.java b/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitSaveRequestDto.java deleted file mode 100644 index 81441b9..0000000 --- a/stats/stats-dto/src/main/java/ru/practicum/dto/EndpointHitSaveRequestDto.java +++ /dev/null @@ -1,27 +0,0 @@ -package ru.practicum.dto; - -import com.fasterxml.jackson.annotation.JsonFormat; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.experimental.FieldDefaults; - -import java.time.LocalDateTime; - -/** - * The type Endpoint hit save request dto. - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class EndpointHitSaveRequestDto { - - String app; - String uri; - String ip; - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss") - LocalDateTime timestamp; - -} diff --git a/stats/stats-dto/src/main/java/ru/practicum/dto/SecondaryViewStatsDto.java b/stats/stats-dto/src/main/java/ru/practicum/dto/SecondaryViewStatsDto.java deleted file mode 100644 index 48603cf..0000000 --- a/stats/stats-dto/src/main/java/ru/practicum/dto/SecondaryViewStatsDto.java +++ /dev/null @@ -1,26 +0,0 @@ -package ru.practicum.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * The type Secondary view stats dto. - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class SecondaryViewStatsDto { - /** - * The App. - */ - String app; - /** - * The Uri. - */ - String uri; - /** - * The Hits. - */ - Long hits; -} diff --git a/stats/stats-server/src/main/java/ru/practicum/ErrorResponse.java b/stats/stats-server/src/main/java/ru/practicum/ErrorResponse.java deleted file mode 100644 index 6965644..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/ErrorResponse.java +++ /dev/null @@ -1,32 +0,0 @@ -package ru.practicum; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; -import org.slf4j.Logger; - -import java.io.PrintWriter; -import java.io.StringWriter; - -@Data -public class ErrorResponse { - - @JsonProperty("error") - private String message; - @JsonIgnore - private String stacktrace; - - public ErrorResponse(String message) { - this.message = message; - } - - public static ErrorResponse getErrorResponse(Exception e, Logger log) { - log.info("Error", e); - ErrorResponse errorResponse = new ErrorResponse(e.getMessage()); - StringWriter stringWriter = new StringWriter(); - PrintWriter pw = new PrintWriter(stringWriter); - e.printStackTrace(pw); - errorResponse.setStacktrace(pw.toString()); - return errorResponse; - } -} \ No newline at end of file diff --git a/stats/stats-server/src/main/java/ru/practicum/controller/StatController.java b/stats/stats-server/src/main/java/ru/practicum/controller/StatController.java deleted file mode 100644 index 5bfbc21..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/controller/StatController.java +++ /dev/null @@ -1,67 +0,0 @@ -package ru.practicum.controller; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.format.annotation.DateTimeFormat; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; -import ru.practicum.ErrorResponse; -import ru.practicum.dto.EndpointHitDto; -import ru.practicum.dto.ViewStatsDto; -import ru.practicum.service.StatService; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; - -@RestController -@Slf4j -public class StatController { - - private final StatService statsService; - - @Autowired - public StatController(StatService service) { - this.statsService = service; - } - - - @PostMapping("/hit") - @ResponseStatus(HttpStatus.CREATED) - public EndpointHitDto saveHit(@RequestBody EndpointHitDto hitDto) { - - return statsService.saveHit(hitDto); - } - - @GetMapping("/stats") - public List getHits(@RequestParam(value = "start", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") LocalDateTime start, - @RequestParam(value = "end", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") LocalDateTime end, - @RequestParam(value = "uris", required = false) List uris, - @RequestParam(value = "unique", defaultValue = "false") boolean unique - ) { - if (start == null || end == null) { - throw new IllegalArgumentException("Время не может быть Null"); - } - if (start.isAfter(end)) { - throw new IllegalArgumentException("Дата начала не может быть позже даты конца"); - } - if (uris == null) { - uris = new ArrayList<>(); - } - log.info("/GET запрос на получение статистики"); - return statsService.getStats(start, end, uris, unique); - } - - @ExceptionHandler - @ResponseStatus(HttpStatus.BAD_REQUEST) - public ErrorResponse handleException(final IllegalArgumentException e) { - ErrorResponse errorResponse = new ErrorResponse(e.getMessage()); - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e.printStackTrace(pw); - errorResponse.setStacktrace(pw.toString()); - return errorResponse; - } -} \ No newline at end of file diff --git a/stats/stats-server/src/main/java/ru/practicum/mapper/EndpointHitMapper.java b/stats/stats-server/src/main/java/ru/practicum/mapper/EndpointHitMapper.java deleted file mode 100644 index 7252c50..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/mapper/EndpointHitMapper.java +++ /dev/null @@ -1,37 +0,0 @@ -package ru.practicum.mapper; - -import lombok.experimental.UtilityClass; -import ru.practicum.dto.EndpointHitDto; -import ru.practicum.model.EndpointHit; - -import java.time.LocalDateTime; - -import static ru.practicum.util.Constants.FORMATTER; - -@UtilityClass -public class EndpointHitMapper { - - public static EndpointHitDto toHitDto(EndpointHit hit) { - String dateTime = hit.getTimestamp().format(FORMATTER); - - return new EndpointHitDto( - hit.getId(), - hit.getApp(), - hit.getUri(), - hit.getIp(), - dateTime - ); - } - - public static EndpointHit dtoToHit(EndpointHitDto hitDto) { - - LocalDateTime localDateTime = LocalDateTime.parse(hitDto.getTimestamp(), FORMATTER); - EndpointHit hit = new EndpointHit(); - hit.setId(hitDto.getId()); - hit.setApp(hitDto.getApp()); - hit.setUri(hitDto.getUri()); - hit.setIp(hitDto.getIp()); - hit.setTimestamp(localDateTime); - return hit; - } -} \ No newline at end of file diff --git a/stats/stats-server/src/main/java/ru/practicum/model/EndpointHit.java b/stats/stats-server/src/main/java/ru/practicum/model/EndpointHit.java deleted file mode 100644 index 6b78cc2..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/model/EndpointHit.java +++ /dev/null @@ -1,34 +0,0 @@ -package ru.practicum.model; - -import jakarta.persistence.*; -import lombok.*; -import lombok.experimental.FieldDefaults; - -import java.time.LocalDateTime; - -/** - * The type Endpoint hit. - */ -@Setter -@Getter -@Entity -@Table(name = "endpoint_hit") -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -@NoArgsConstructor -public class EndpointHit { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - Integer id; - - @Column(name = "app") - String app; - @Column(name = "uri") - String uri; - @Column(name = "ip") - String ip; - @Column(name = "timestamp") - LocalDateTime timestamp; - -} \ No newline at end of file diff --git a/stats/stats-server/src/main/java/ru/practicum/repository/EndpointHitRepository.java b/stats/stats-server/src/main/java/ru/practicum/repository/EndpointHitRepository.java deleted file mode 100644 index f786e29..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/repository/EndpointHitRepository.java +++ /dev/null @@ -1,89 +0,0 @@ -package ru.practicum.repository; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Query; - -import ru.practicum.dto.ViewStatsDto; -import ru.practicum.model.EndpointHit; - -import java.time.LocalDateTime; -import java.util.List; - -/** - * The interface Endpoint hit repository. - */ -public interface EndpointHitRepository extends JpaRepository { - - String SELECT_STAT_WITHOUT_UNIQUE_IP_SQL = "SELECT " + - "new ru.practicum.dto.ViewStatsDto(e.app, e.uri, " + - "(SELECT count(ep.ip) FROM EndpointHit AS ep WHERE ep.uri = e.uri) AS hits) " + - "FROM EndpointHit AS e WHERE e.uri IN ( ?3 ) AND e.timestamp BETWEEN ?1 AND ?2 " + - "GROUP BY e.uri, e.app ORDER BY hits DESC "; - - /** - * The constant SELECT_STAT_WITH_UNIQUE_IP_SQL. - */ - String SELECT_STAT_WITH_UNIQUE_IP_SQL = "SELECT " + - "new ru.practicum.dto.ViewStatsDto(e.app, e.uri, " + - "(SELECT count(DISTINCT ep.ip) FROM EndpointHit AS ep WHERE ep.uri = e.uri) AS hits) " + - "FROM EndpointHit AS e WHERE e.uri IN ( ?3 ) AND e.timestamp BETWEEN ?1 AND ?2 " + - "GROUP BY e.uri, e.app ORDER BY hits DESC "; - - /** - * The constant SELECT_STAT_ALL_WITHOUT_UNIQUE_IP_SQL. - */ - String SELECT_STAT_ALL_WITHOUT_UNIQUE_IP_SQL = "SELECT " + - "new ru.practicum.dto.ViewStatsDto(e.app, e.uri, " + - "(SELECT count(ep.ip) FROM EndpointHit AS ep WHERE ep.uri = e.uri) AS hits) " + - "FROM EndpointHit AS e WHERE e.timestamp BETWEEN ?1 AND ?2 GROUP BY e.uri, e.app ORDER BY hits DESC "; - - /** - * The constant SELECT_STAT_ALL_WITH_UNIQUE_IP_SQL. - */ - String SELECT_STAT_ALL_WITH_UNIQUE_IP_SQL = "SELECT " + - "new ru.practicum.dto.ViewStatsDto(e.app, e.uri, " + - "(SELECT count(DISTINCT ep.ip) FROM EndpointHit AS ep WHERE ep.uri = e.uri) AS hits) " + - "FROM EndpointHit AS e WHERE e.timestamp BETWEEN ?1 AND ?2 GROUP BY e.uri, e.app ORDER BY hits DESC "; - - /** - * Find stat without unique ip list. - * - * @param start the start - * @param end the end - * @param uris the uris - * @return the list - */ - @Query(SELECT_STAT_WITHOUT_UNIQUE_IP_SQL) - List findStatWithoutUniqueIp(LocalDateTime start, LocalDateTime end, List uris); - - /** - * Find stat with unique ip list. - * - * @param start the start - * @param end the end - * @param uris the uris - * @return the list - */ - @Query(SELECT_STAT_WITH_UNIQUE_IP_SQL) - List findStatWithUniqueIp(LocalDateTime start, LocalDateTime end, List uris); - - /** - * Find stat all without unique ip list. - * - * @param start the start - * @param end the end - * @return the list - */ - @Query(SELECT_STAT_ALL_WITHOUT_UNIQUE_IP_SQL) - List findStatAllWithoutUniqueIp(LocalDateTime start, LocalDateTime end); - - /** - * Find stat all with unique ip list. - * - * @param start the start - * @param end the end - * @return the list - */ - @Query(SELECT_STAT_ALL_WITH_UNIQUE_IP_SQL) - List findStatAllWithUniqueIp(LocalDateTime start, LocalDateTime end); -} \ No newline at end of file diff --git a/stats/stats-server/src/main/java/ru/practicum/service/StatService.java b/stats/stats-server/src/main/java/ru/practicum/service/StatService.java deleted file mode 100644 index a7d24b9..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/service/StatService.java +++ /dev/null @@ -1,14 +0,0 @@ -package ru.practicum.service; - -import ru.practicum.dto.EndpointHitDto; -import ru.practicum.dto.ViewStatsDto; - -import java.time.LocalDateTime; -import java.util.List; - -public interface StatService { - - EndpointHitDto saveHit(EndpointHitDto hitDto); - - List getStats(LocalDateTime start, LocalDateTime end, List uris, Boolean unique); -} diff --git a/stats/stats-server/src/main/java/ru/practicum/service/StatServiceImpl.java b/stats/stats-server/src/main/java/ru/practicum/service/StatServiceImpl.java deleted file mode 100644 index 25bfb72..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/service/StatServiceImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -package ru.practicum.service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import ru.practicum.dto.*; -import ru.practicum.repository.EndpointHitRepository; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; - -import static ru.practicum.mapper.EndpointHitMapper.dtoToHit; -import static ru.practicum.mapper.EndpointHitMapper.toHitDto; - -/** - * The type Stat service. - */ -@Service -@RequiredArgsConstructor -@Slf4j -public class StatServiceImpl implements StatService { - - private final EndpointHitRepository endpointHitRepository; - - @Override - @Transactional - public EndpointHitDto saveHit(EndpointHitDto hitDto) { - return toHitDto(endpointHitRepository.save(dtoToHit(hitDto))); - } - - @Transactional(readOnly = true) - @Override - public List getStats(LocalDateTime start, LocalDateTime end, List uris, Boolean unique) { - log.info("Получение статистики с параметрами: start={}, end={}, uris={}, unique={}", start, end, uris, unique); - List stats; - - if (unique) { - stats = uris.isEmpty() ? - endpointHitRepository.findStatAllWithUniqueIp(start, end) : - endpointHitRepository.findStatWithUniqueIp(start, end, uris); - } else { - stats = uris.isEmpty() ? - endpointHitRepository.findStatAllWithoutUniqueIp(start, end) : - endpointHitRepository.findStatWithoutUniqueIp(start, end, uris); - } - log.info("Полученные статистические данные: {}", stats); - return new ArrayList<>(stats); - } -} diff --git a/stats/stats-server/src/main/java/ru/practicum/util/Constants.java b/stats/stats-server/src/main/java/ru/practicum/util/Constants.java deleted file mode 100644 index 8766eb6..0000000 --- a/stats/stats-server/src/main/java/ru/practicum/util/Constants.java +++ /dev/null @@ -1,10 +0,0 @@ -package ru.practicum.util; - -import java.time.format.DateTimeFormatter; - -public class Constants { - - public static final String TIMESTAMP_PATTERN = "yyyy-MM-dd HH:mm:ss"; - - public static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern(TIMESTAMP_PATTERN); -} \ No newline at end of file diff --git a/stats/stats-server/src/main/resources/schema.sql b/stats/stats-server/src/main/resources/schema.sql deleted file mode 100644 index 49082a7..0000000 --- a/stats/stats-server/src/main/resources/schema.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP TABLE IF EXISTS endpoint_hit; - -CREATE TABLE IF NOT EXISTS endpoint_hit ( - id SERIAL PRIMARY KEY, - app VARCHAR(255), - uri VARCHAR(255) NOT NULL, - ip VARCHAR(255) NOT NULL, - timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL -); \ No newline at end of file From 68175143870091d361d99b102c25fb51cde4db4f Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Tue, 6 May 2025 10:11:41 +0300 Subject: [PATCH 02/26] feat: refactored stats and released recommendations feature --- stats/analyzer/src/main/resources/application.yml | 2 +- stats/collector/src/main/resources/application.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/analyzer/src/main/resources/application.yml b/stats/analyzer/src/main/resources/application.yml index 3d0ef3b..ccc4033 100644 --- a/stats/analyzer/src/main/resources/application.yml +++ b/stats/analyzer/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: application: name: analyzer config: - import: optional:configserver:http://config-server:9091 + import: "configserver:" cloud: config: fail-fast: true diff --git a/stats/collector/src/main/resources/application.yml b/stats/collector/src/main/resources/application.yml index e1855d9..5f9dddc 100644 --- a/stats/collector/src/main/resources/application.yml +++ b/stats/collector/src/main/resources/application.yml @@ -2,7 +2,7 @@ spring: application: name: collector config: - import: optional:configserver:http://config-server:9091 + import: "configserver:" cloud: config: fail-fast: true From e65142dd6d28f8cbcc503db894e5011033efa467 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Tue, 6 May 2025 10:27:03 +0300 Subject: [PATCH 03/26] feat: refactored stats and released recommendations feature --- .../config/stats/aggregator/application.yml | 23 +++++++++--------- .../config/stats/analyzer/application.yml | 24 +++++++++---------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml index 91d5086..7cd7e6c 100644 --- a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml +++ b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml @@ -1,11 +1,12 @@ -kafka: - bootstrap-servers: kafka:29092 - consumer: - topic: stats.user-actions.v1 - group-id: aggregator-actions - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer - producer: - topic: stats.events-similarity.v1 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.AvroSerializer \ No newline at end of file +spring: + kafka: + bootstrap-servers: kafka:29092 + consumer: + topic: stats.user-actions.v1 + group-id: aggregator-actions + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + producer: + topic: stats.events-similarity.v1 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: ru.practicum.AvroSerializer \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml index fea45a3..f91fdf2 100644 --- a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml +++ b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml @@ -17,18 +17,18 @@ spring: init: mode: always -kafka: - bootstrap-servers: kafka:29092 - user-actions-consumer: - group-id: analyzer-user-actions - topic: stats.user-actions.v1 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer - events-similarity-consumer: - group-id: analyzer-events-sim - topic: stats.events-similarity.v1 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer + kafka: + bootstrap-servers: kafka:29092 + user-actions-consumer: + group-id: analyzer-user-actions + topic: stats.user-actions.v1 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + events-similarity-consumer: + group-id: analyzer-events-sim + topic: stats.events-similarity.v1 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer grpc: client: From 95a01bd41f04ec6c232e9bcf2f75865e227f846a Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Tue, 6 May 2025 10:32:49 +0300 Subject: [PATCH 04/26] feat: refactored stats and released recommendations feature --- .../src/main/java/ru/practicum/config/UserActionsConsumer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java index 2c5591d..1210b03 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java @@ -15,7 +15,7 @@ public class UserActionsConsumer { private final SimilarityService similarityService; @KafkaListener( - topics = "#{kafkaProperties.consumer.topic}", + topics = "${kafkaProperties.consumer.topic}", containerFactory = "kafkaListenerContainerFactory" ) public void consumeUserAction(UserActionAvro message) { From 73d008c835948fbf448293df7094c266d104329f Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Tue, 6 May 2025 10:40:40 +0300 Subject: [PATCH 05/26] feat: refactored stats and released recommendations feature --- .../config/stats/aggregator/application.yml | 23 +++++++++--------- .../config/stats/analyzer/application.yml | 24 +++++++++---------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml index 7cd7e6c..91d5086 100644 --- a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml +++ b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml @@ -1,12 +1,11 @@ -spring: - kafka: - bootstrap-servers: kafka:29092 - consumer: - topic: stats.user-actions.v1 - group-id: aggregator-actions - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer - producer: - topic: stats.events-similarity.v1 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.AvroSerializer \ No newline at end of file +kafka: + bootstrap-servers: kafka:29092 + consumer: + topic: stats.user-actions.v1 + group-id: aggregator-actions + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + producer: + topic: stats.events-similarity.v1 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: ru.practicum.AvroSerializer \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml index f91fdf2..fea45a3 100644 --- a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml +++ b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml @@ -17,18 +17,18 @@ spring: init: mode: always - kafka: - bootstrap-servers: kafka:29092 - user-actions-consumer: - group-id: analyzer-user-actions - topic: stats.user-actions.v1 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer - events-similarity-consumer: - group-id: analyzer-events-sim - topic: stats.events-similarity.v1 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer +kafka: + bootstrap-servers: kafka:29092 + user-actions-consumer: + group-id: analyzer-user-actions + topic: stats.user-actions.v1 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer + events-similarity-consumer: + group-id: analyzer-events-sim + topic: stats.events-similarity.v1 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.AvroDeserializer grpc: client: From 3a03cc7f9636ce57dbe47d4a33e332336004e187 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Thu, 8 May 2025 01:05:49 +0300 Subject: [PATCH 06/26] feat: refactored stats and released recommendations feature --- .../src/main/java/ru/practicum/config/UserActionsConsumer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java index 1210b03..2c5591d 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java @@ -15,7 +15,7 @@ public class UserActionsConsumer { private final SimilarityService similarityService; @KafkaListener( - topics = "${kafkaProperties.consumer.topic}", + topics = "#{kafkaProperties.consumer.topic}", containerFactory = "kafkaListenerContainerFactory" ) public void consumeUserAction(UserActionAvro message) { From 52eb4e319f716f0205aaeade6001118ae8b4d5ee Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Fri, 9 May 2025 17:21:47 +0300 Subject: [PATCH 07/26] feat: refactored stats and released recommendations feature --- docker-compose.yml | 4 - .../config/stats/aggregator/application.yml | 33 ++- .../config/stats/analyzer/application.yml | 73 ++++--- .../config/stats/collector/application.yml | 13 +- stats/aggregator/pom.xml | 81 ++----- .../ru/practicum/AggregatorApplication.java | 2 - .../java/ru/practicum/config/AppConfig.java | 51 +++++ .../java/ru/practicum/config/KafkaConfig.java | 72 +++---- .../ru/practicum/config/KafkaProperties.java | 33 --- .../config/SimilarityEventProducer.java | 11 + .../SimilarityEventProducerConfiguration.java | 47 ++++ .../practicum/config/UserActionsConsumer.java | 25 --- .../deserializer/BaseAvroDeserializer.java | 42 ++++ .../deserializer/UserActionDeserializer.java | 11 + .../practicum/serializer/AvroSerializer.java | 36 ++++ .../practicum/service/AggregationStarter.java | 103 +++++++++ .../practicum/service/AggregatorService.java | 10 + .../service/AggregatorServiceImpl.java | 129 +++++++++++ .../practicum/service/MinWeightsMatrix.java | 22 -- .../practicum/service/SimilarityService.java | 110 ---------- .../src/main/resources/application.yml | 18 +- stats/analyzer/pom.xml | 95 +++------ .../ru/practicum/AnalyzerApplication.java | 4 - .../config/EventsSimilarityConsumer.java | 25 --- .../java/ru/practicum/config/KafkaConfig.java | 65 ------ .../config/KafkaConsumerSettings.java | 19 ++ .../ru/practicum/config/KafkaProperties.java | 22 -- .../java/ru/practicum/config/KafkaTopics.java | 16 ++ .../practicum/config/UserActionsConsumer.java | 25 --- .../controller/RecommendationController.java | 67 ++++++ .../controller/RecommendationsController.java | 106 --------- .../deserializer/BaseAvroDeserializer.java | 41 ++++ .../EventSimilarityDeserializer.java | 11 + .../deserializer/UserActionDeserializer.java | 10 + .../kafka/ConfigKafkaProperties.java | 61 ++++++ .../mapper/EventSimilarityMapper.java | 10 + .../ru/practicum/model/EventSimilarity.java | 18 +- .../java/ru/practicum/model/UserAction.java | 2 +- .../processor/EventSimilarityProcessor.java | 90 ++++++++ .../processor/UserActionEventProcessor.java | 89 ++++++++ .../repository/EventSimilarityRepository.java | 7 +- .../repository/UserActionRepository.java | 5 +- .../service/RecommendationService.java | 201 +++++++++++------- .../service/event/EventSimilarityService.java | 3 +- .../event/EventSimilarityServiceImpl.java | 55 ++--- .../service/user/UserActionService.java | 2 +- .../service/user/UserActionServiceImpl.java | 68 +++--- .../src/main/resources/application.yml | 19 +- .../ru/practicum/CollectorApplication.java | 4 - .../practicum/config/KafkaProducerConfig.java | 76 +++---- .../ru/practicum/config/KafkaProperties.java | 58 ++--- .../practicum/config/UserActionProducer.java | 11 + .../UserActionProducerConfiguration.java | 70 ++++++ .../ru/practicum/handler/ActionsHandlers.java | 7 + .../practicum/handler/UserActionHandler.java | 61 ++++++ .../ru/practicum/mapper/UserActionMapper.java | 4 +- .../serializer/UserActionsAvroSerializer.java | 36 ++++ .../service/KafkaMessageProducer.java | 24 ++- .../service/UserActionController.java | 9 +- .../src/main/resources/application.yml | 19 +- .../main/avro/EventSimilarityProtocol.avdl | 2 +- .../src/main/avro/UserActionAvro.avdl | 2 +- .../java/ru/practicum/CollectorClient.java | 2 +- 63 files changed, 1498 insertions(+), 949 deletions(-) create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java delete mode 100644 stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java delete mode 100644 stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java delete mode 100644 stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java delete mode 100644 stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/deserializer/EventSimilarityDeserializer.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java create mode 100644 stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java create mode 100644 stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java create mode 100644 stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java create mode 100644 stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java create mode 100644 stats/collector/src/main/java/ru/practicum/serializer/UserActionsAvroSerializer.java diff --git a/docker-compose.yml b/docker-compose.yml index 71fe4d9..93a6d0b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,8 +171,6 @@ services: ports: - "9092:9092" # for client connections - "9101:9101" # JMX - networks: - - ewm-net restart: unless-stopped environment: KAFKA_NODE_ID: 1 @@ -189,8 +187,6 @@ services: container_name: kafka-init-topics depends_on: - kafka - networks: - - ewm-net command: "bash -c \ 'kafka-topics --create --topic stats.user-actions.v1 \ --partitions 1 --replication-factor 1 --if-not-exists \ diff --git a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml index 91d5086..52a5a9d 100644 --- a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml +++ b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml @@ -1,11 +1,22 @@ -kafka: - bootstrap-servers: kafka:29092 - consumer: - topic: stats.user-actions.v1 - group-id: aggregator-actions - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer - producer: - topic: stats.events-similarity.v1 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.AvroSerializer \ No newline at end of file +server: + port: 0 + +spring: + kafka: + producer: + bootstrap-servers: localhost:9092 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: ru.practicum.serializer.AvroSerializer + consumer: + bootstrap-servers: localhost:9092 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.deserializer.UserActionDeserializer + client-id: action-consumer + group-id: group-practicum + max-poll-records: 100 + fetch-max-bytes: 3072000 + max-partition-fetch-bytes: 307200 + consume-attempts-timeout-ms: 1000 + topics: + action-topic: stats.user-actions.v1 + similarity-topic: stats.events-similarity.v1 \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml index fea45a3..d06f86c 100644 --- a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml +++ b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml @@ -1,39 +1,56 @@ spring: datasource: - driverClassName: org.postgresql.Driver - url: jdbc:postgresql://analyzer-db:5432/ewm-analyzer + url: jdbc:postgresql://localhost:5432/ewm-stats-analyzer-db username: root password: root - + sql: + init: + mode: always + output: + ansi: + enabled: ALWAYS jpa: hibernate: - ddl-auto: update + ddl-auto: none + show-sql: false properties: hibernate: format_sql: true - generate-ddl: false - - sql: - init: - mode: always - -kafka: - bootstrap-servers: kafka:29092 - user-actions-consumer: - group-id: analyzer-user-actions - topic: stats.user-actions.v1 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer - events-similarity-consumer: - group-id: analyzer-events-sim - topic: stats.events-similarity.v1 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.AvroDeserializer + kafka: + consumer-user-actions: + bootstrap-servers: localhost:9092 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.yandex.practicum.deserializer.UserActionDeserializer + client-id: user-consumer + group-id: group-user + max-poll-records: 100 + fetch-max-bytes: 3072000 + max-partition-fetch-bytes: 307200 + consume-attempts-timeout-ms: 1000 + consumer-events-similarity: + bootstrap-servers: localhost:9092 + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: ru.practicum.deserializer.EventSimilarityDeserializer + client-id: similarity-consumer + group-id: group-similarity + max-poll-records: 100 + fetch-max-bytes: 3072000 + max-partition-fetch-bytes: 307200 + consume-attempts-timeout-ms: 1000 + producer: + bootstrap-servers: localhost:9092 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + value-serializer: ru.practicum.serializer.AvroSerializer + topics: + user-action-topic: stats.user-actions.v1 + events-similarity-topic: stats.events-similarity.v1 grpc: - client: - analyzer: - address: 'discovery:///analyzer' - enableKeepAlive: true - keepAliveWithoutCalls: true - negotiationType: plaintext \ No newline at end of file + analyzer: + address: 'discovery:///analyzer' + enableKeepAlive: true + keepAliveWithoutCalls: true + negotiationType: plaintext +logging: + level: + io.grpc: DEBUG \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/collector/application.yml b/infra/config-server/src/main/resources/config/stats/collector/application.yml index 74915bb..74a5680 100644 --- a/infra/config-server/src/main/resources/config/stats/collector/application.yml +++ b/infra/config-server/src/main/resources/config/stats/collector/application.yml @@ -1,19 +1,16 @@ spring: kafka: - bootstrap-servers: kafka:29092 producer: + bootstrap-servers: localhost:9092 key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.AvroSerializer - -collector: - kafka: - topic: stats.user-actions.v1 - + value-serializer: ru.practicum.serializer.UserActionsAvroSerializer + topics: + actions-topic: stats.user-actions.v1 grpc: server: port: 0 client: - analyzer: + collector: address: 'discovery:///collector' enableKeepAlive: true keepAliveWithoutCalls: true diff --git a/stats/aggregator/pom.xml b/stats/aggregator/pom.xml index 480b505..c8a83ab 100644 --- a/stats/aggregator/pom.xml +++ b/stats/aggregator/pom.xml @@ -1,6 +1,6 @@ - 4.0.0 @@ -15,97 +15,47 @@ 21 21 UTF-8 - 0.0.1-SNAPSHOT - - - org.springframework.boot - spring-boot-starter-web - - - - org.springframework.boot - spring-boot-starter-data-jpa - - - - org.springframework.boot - spring-boot-starter-actuator - - - org.postgresql - postgresql - runtime - - - - com.h2database - h2 - runtime - - - - org.springframework.boot - spring-boot-configuration-processor - true - - - - org.springframework.boot - spring-boot-starter-test - test + ru.practicum + avro-schemas + 0.0.1-SNAPSHOT - org.projectlombok lombok - true - org.springframework.cloud - spring-cloud-commons + org.springframework.boot + spring-boot-starter - - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client + org.apache.kafka + kafka-clients - org.springframework.cloud spring-cloud-starter-config - org.springframework.retry spring-retry - org.springframework.cloud - spring-cloud-openfeign-core - - - - org.apache.kafka - kafka-clients + org.springframework.boot + spring-boot-starter-validation - - org.springframework.kafka - spring-kafka + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client - - ru.practicum - avro-schemas - ${avro-schemas.version} + org.springframework.boot + spring-boot-starter-actuator - - @@ -114,5 +64,4 @@ - \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java b/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java index 0897186..ae72ee5 100644 --- a/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java +++ b/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java @@ -3,10 +3,8 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; @SpringBootApplication -@EnableDiscoveryClient @ConfigurationPropertiesScan public class AggregatorApplication { public static void main(String[] args) { diff --git a/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java b/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java new file mode 100644 index 0000000..31fae33 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java @@ -0,0 +1,51 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.time.Duration; + +@Getter +@Setter +@ToString +@ConfigurationProperties("spring.kafka") +public class AppConfig { + ProducerSettings producer; + ConsumerSettings consumer; + TopicsSettings topics; + + @Setter + @Getter + @ToString + + public static class ProducerSettings { + private String bootstrapServers; + private String keySerializer; + private String valueSerializer; + } + + @Setter + @Getter + @ToString + public static class ConsumerSettings { + private String bootstrapServers; + private String keyDeserializer; + private String valueDeserializer; + private String clientId; + private String groupId; + private String maxPollRecords; + private String fetchMaxBytes; + private String maxPartitionFetchBytes; + private Duration consumeAttemptsTimeoutMs; + } + + @ToString + @Getter + @Setter + public static class TopicsSettings { + private String actionTopic; + private String similarityTopic; + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java index ff3c0ab..e15e007 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java @@ -1,53 +1,45 @@ package ru.practicum.config; -import lombok.RequiredArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.core.*; -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -import ru.practicum.ewm.stats.avro.UserActionAvro; +import org.springframework.stereotype.Component; -import java.util.Map; +import java.util.Properties; -@Configuration -@RequiredArgsConstructor -public class KafkaConfig { - private final KafkaProperties props; +@Slf4j +@Component +@Data - @Bean - public ConsumerFactory consumerFactory() { - final Map config = Map.of( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), - ConsumerConfig.GROUP_ID_CONFIG, props.getConsumer().getGroupId(), - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, props.getConsumer().getKeyDeserializer(), - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, props.getConsumer().getValueDeserializer() - ); - return new DefaultKafkaConsumerFactory<>(config); - } +public class KafkaConfig { + private final AppConfig appConfig; - @Bean - public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory()); - return factory; + public KafkaConfig(AppConfig appConfig) { + this.appConfig = appConfig; } - @Bean - public ProducerFactory producerFactory() { - Map config = Map.of( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, props.getProducer().getKeySerializer(), - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, props.getProducer().getValueSerializer() - ); - return new DefaultKafkaProducerFactory<>(config); + public Properties getConsumerProperties() { + Properties properties = new Properties(); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, appConfig.getConsumer().getGroupId()); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, appConfig.getConsumer().getBootstrapServers()); + properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, appConfig.getConsumer().getKeyDeserializer()); + properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, appConfig.getConsumer().getValueDeserializer()); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, appConfig.getConsumer().getMaxPollRecords()); + properties.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, appConfig.getConsumer().getFetchMaxBytes()); + properties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, appConfig.getConsumer().getMaxPartitionFetchBytes()); + return properties; } - @Bean - public KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(producerFactory()); + public Properties getProducerProperties() { + Properties config = new Properties(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + appConfig.getProducer().getBootstrapServers()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + appConfig.getProducer().getKeySerializer()); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + appConfig.getProducer().getValueSerializer()); + log.info("Kafka producer config is ready = {}", config); + return config; } -} +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java deleted file mode 100644 index fa8a503..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/config/KafkaProperties.java +++ /dev/null @@ -1,33 +0,0 @@ -package ru.practicum.config; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Component -@ConfigurationProperties(prefix = "kafka") -@Getter -@Setter -public class KafkaProperties { - private String bootstrapServers; - private Consumer consumer = new Consumer(); - private Producer producer = new Producer(); - - @Getter - @Setter - public static class Consumer { - private String groupId; - private String topic; - private String keyDeserializer; - private String valueDeserializer; - } - - @Getter - @Setter - public static class Producer { - private String topic; - private String keySerializer; - private String valueSerializer; - } -} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java new file mode 100644 index 0000000..77ba538 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java @@ -0,0 +1,11 @@ +package ru.practicum.config; + +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.Producer; + +public interface SimilarityEventProducer { + + Producer getProducer(); + + void stop(); +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java new file mode 100644 index 0000000..320e425 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java @@ -0,0 +1,47 @@ +package ru.practicum.config; + +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import java.util.Properties; + +@Slf4j +@RequiredArgsConstructor +@Component +public class SimilarityEventProducerConfiguration { + private final KafkaConfig kafkaConfig; + + @Bean + SimilarityEventProducer getClient() { + return new SimilarityEventProducer() { + private Producer producer; + + @Override + public Producer getProducer() { + if (producer == null) { + initProducer(); + } + return producer; + } + + private void initProducer() { + Properties config = kafkaConfig.getProducerProperties(); + producer = new KafkaProducer<>(config); + } + + @PreDestroy + @Override + public void stop() { + if (producer != null) { + producer.close(); + } + } + }; + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java b/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java deleted file mode 100644 index 2c5591d..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/config/UserActionsConsumer.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.practicum.config; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.stereotype.Component; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.service.SimilarityService; - -@Slf4j -@Component -@RequiredArgsConstructor -public class UserActionsConsumer { - - private final SimilarityService similarityService; - - @KafkaListener( - topics = "#{kafkaProperties.consumer.topic}", - containerFactory = "kafkaListenerContainerFactory" - ) - public void consumeUserAction(UserActionAvro message) { - log.info("consume user action Kafka: {}", message); - similarityService.processUserAction(message); - } -} diff --git a/stats/aggregator/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java b/stats/aggregator/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java new file mode 100644 index 0000000..f4a6ca7 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java @@ -0,0 +1,42 @@ +package ru.practicum.deserializer; + +import org.apache.avro.Schema; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.common.serialization.Deserializer; + + +public class BaseAvroDeserializer implements Deserializer { + + private final DecoderFactory decoderFactory; + private final DatumReader reader; + + public BaseAvroDeserializer(Schema schema) { + this(DecoderFactory.get(), schema); + } + + public BaseAvroDeserializer(DecoderFactory decoderFactory, Schema schema) { + this.decoderFactory = decoderFactory; + this.reader = new SpecificDatumReader<>(schema); + + } + + + @Override + public T deserialize(String topic, byte[] data) { + + try { + if (data != null) { + + BinaryDecoder decoder = decoderFactory.binaryDecoder(data, null); + return reader.read(null, decoder); + } + return null; + } catch (Exception e) { + throw new RuntimeException("Data serialization from topic error [" + topic + "]", e); + } + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java b/stats/aggregator/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java new file mode 100644 index 0000000..957ca3d --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java @@ -0,0 +1,11 @@ +package ru.practicum.deserializer; + +import ru.practicum.ewm.stats.avro.UserActionAvro; + + +public class UserActionDeserializer extends BaseAvroDeserializer { + public UserActionDeserializer() { + super(UserActionAvro.getClassSchema()); + } + +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java b/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java new file mode 100644 index 0000000..8f4754a --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java @@ -0,0 +1,36 @@ +package ru.yandex.practicum.serializer; + +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Serializer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +@Slf4j +public class AvroSerializer implements Serializer { + private final EncoderFactory encoderFactory = EncoderFactory.get(); + private BinaryEncoder encoder; + + public byte[] serialize(String topic, SpecificRecordBase data) { + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] result = null; + encoder = encoderFactory.binaryEncoder(out, encoder); + if (data != null) { + DatumWriter writer = new SpecificDatumWriter<>(data.getSchema()); + writer.write(data, encoder); + encoder.flush(); + result = out.toByteArray(); + } + return result; + } catch (IOException ex) { + throw new SerializationException("Data serialization from topic error [" + topic + "]", ex); + } + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java new file mode 100644 index 0000000..b98c70d --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java @@ -0,0 +1,103 @@ +package ru.practicum.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.springframework.stereotype.Component; +import ru.practicum.config.AppConfig; +import ru.practicum.config.KafkaConfig; +import ru.practicum.config.SimilarityEventProducer; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Component + +public class AggregationStarter { + private static final Map currentOffsets = new HashMap<>(); + private final AggregatorService aggregatorService; + private final SimilarityEventProducer similarityEventProducer; + private final KafkaConfig kafkaConfig; + private final AppConfig appConfig; + + public AggregationStarter(AggregatorService aggregatorService, SimilarityEventProducer eventProducer, KafkaConfig kafkaConfig, AppConfig appConfig) { + this.aggregatorService = aggregatorService; + this.similarityEventProducer = eventProducer; + this.kafkaConfig = kafkaConfig; + this.appConfig = appConfig; + } + + private static void manageOffsets(ConsumerRecord record, int count, + KafkaConsumer consumer) { + currentOffsets.put( + new TopicPartition(record.topic(), record.partition()), + new OffsetAndMetadata(record.offset() + 1) + ); + + if (count % 10 == 0) { + consumer.commitAsync(currentOffsets, (offsets, exception) -> { + if (exception != null) { + log.warn("Error occurred while pinning offsets : {}", offsets, exception); + } + }); + } + } + + public void start() { + + KafkaConsumer consumer = new KafkaConsumer<>(kafkaConfig.getConsumerProperties()); + Runtime.getRuntime().addShutdownHook(new Thread(consumer::wakeup)); + + try { + consumer.subscribe(List.of(appConfig.getTopics().getActionTopic())); + while (true) { + ConsumerRecords records = consumer.poll(appConfig.getConsumer().getConsumeAttemptsTimeoutMs()); + + int count = 0; + for (ConsumerRecord record : records) { + handleRecord(record); + manageOffsets(record, count, consumer); + count++; + } + consumer.commitAsync(); + + } + } catch (WakeupException | InterruptedException ignores) { + } catch (Exception e) { + log.error("Error occurred while reading data", e); + } finally { + + try { + consumer.commitSync(currentOffsets); + } finally { + log.info("Closing consumer"); + consumer.close(); + + } + } + } + + private void handleRecord(ConsumerRecord record) throws InterruptedException { + + log.info("топик = {}, партиция = {}, смещение = {}, значение: {}\n", + record.topic(), record.partition(), record.offset(), record.value()); + List result = aggregatorService.getSimilarities(record.value()); + log.info("Сервис aggregatorService.getSimilarities отработал= {}", result); + if (!result.isEmpty()) { + log.info("Отправляем результаты расчета: {}", result); + for (EventSimilarityAvro event : result) { + similarityEventProducer.getProducer().send(new ProducerRecord<>(appConfig.getTopics().getSimilarityTopic(), + event)); + } + } + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java new file mode 100644 index 0000000..221dfba --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java @@ -0,0 +1,10 @@ +package ru.practicum.service; + +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.List; + +public interface AggregatorService { + List getSimilarities(UserActionAvro actionAvro); +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java new file mode 100644 index 0000000..9f9adad --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java @@ -0,0 +1,129 @@ +package ru.practicum.service; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Service +@AllArgsConstructor +public class AggregatorServiceImpl implements AggregatorService { + + private final Map> eventUserWeightMap = new HashMap<>(); + private final Map eventSum = new HashMap<>(); + private final Map> eventMinSum = new HashMap<>(); + + @Override + public List getSimilarities(UserActionAvro actionAvro) { + log.info("Service AggregatorServiceImpl.getSimilarities"); + long eventId = actionAvro.getEventId(); + double newScore = getActionScore(actionAvro); + + double currentWeight = 0.0; + if (eventUserWeightMap.containsKey(eventId)) { + currentWeight = eventUserWeightMap.get(eventId).get(eventId); + } else { + currentWeight = newScore; + eventUserWeightMap.put(eventId, new HashMap<>()); + } + if (currentWeight >= newScore) { + log.info("Old weight equals or greater than new one, returning void"); + return List.of(); + } + + double newEventScoreSum; + + newEventScoreSum = eventSum.getOrDefault(eventId, 0.0) - currentWeight + newScore; + eventSum.put(eventId, newEventScoreSum); + + + Map eventsToRecalculate = getLongDoubleMap(actionAvro, eventId); + + List similarities = new ArrayList<>(); + + for (Map.Entry event2 : eventsToRecalculate.entrySet()) { + double minSum = getMinScore(eventId, event2.getKey()); + double deltaMin = Math.min(newScore, event2.getValue()) - Math.min(currentWeight, event2.getValue()); + if (deltaMin != 0) { + minSum += deltaMin; + putMinWeights(eventId, event2.getKey(), minSum); + } + + double event2Sum = eventSum.get(event2.getKey()); + float score = (float) (minSum / Math.sqrt(newEventScoreSum) / Math.sqrt(event2Sum)); + + similarities.add( + EventSimilarityAvro.newBuilder() + .setEventA(Math.min(eventId, event2.getKey())) + .setEventB(Math.max(eventId, event2.getKey())) + .setTimestamp(actionAvro.getTimestamp()) + .setScore(score) + .build() + ); + } + log.info("New weight {}", similarities); + + return similarities; + } + + private Map getLongDoubleMap(UserActionAvro action, long eventId) { + Map eventsToRecalculate = new HashMap<>(); + + for (Map.Entry> entry : eventUserWeightMap.entrySet()) { + Long currentEventId = entry.getKey(); + Map userWeights = entry.getValue(); + + if (!currentEventId.equals(eventId) && userWeights.containsKey(action.getUserId())) { + Double weight = userWeights.get(action.getUserId()); + eventsToRecalculate.put(currentEventId, weight); + } + } + return eventsToRecalculate; + } + + private void putMinWeights(long eventA, long eventB, double sum) { + long first = Math.min(eventA, eventB); + long second = Math.max(eventA, eventB); + + if (!eventMinSum.containsKey(first)) { + eventMinSum.put(first, new HashMap<>()); + } + + Map innerMap = eventMinSum.get(first); + innerMap.put(second, sum); + } + + private double getMinScore(long eventA, long eventB) { + long first = Math.min(eventA, eventB); + long second = Math.max(eventA, eventB); + + Double value; + Map innerMap; + + if (!eventMinSum.containsKey(first)) { + innerMap = new HashMap<>(); + eventMinSum.put(first, innerMap); + } else { + innerMap = eventMinSum.get(first); + } + + value = innerMap.getOrDefault(second, 0.0); + + return value; + } + + private double getActionScore(UserActionAvro action) { + return switch (action.getActionType()) { + case VIEW -> 0.4; + case REGISTER -> 0.8; + case LIKE -> 1.0; + }; + } +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java b/stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java deleted file mode 100644 index aa49613..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/service/MinWeightsMatrix.java +++ /dev/null @@ -1,22 +0,0 @@ -package ru.practicum.service; - -import java.util.HashMap; -import java.util.Map; - -public class MinWeightsMatrix { - - private final Map> minWeightsSums = new HashMap<>(); - - public void put(long eventA, long eventB, double sum) { - long first = Math.min(eventA, eventB); - long second = Math.max(eventA, eventB); - minWeightsSums.computeIfAbsent(first, k -> new HashMap<>()).put(second, sum); - } - - public double get(long eventA, long eventB) { - long first = Math.min(eventA, eventB); - long second = Math.max(eventA, eventB); - return minWeightsSums.getOrDefault(first, Map.of()) - .getOrDefault(second, 0.0); - } -} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java b/stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java deleted file mode 100644 index dd51e96..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/service/SimilarityService.java +++ /dev/null @@ -1,110 +0,0 @@ -package ru.practicum.service; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.stereotype.Service; -import ru.practicum.config.KafkaProperties; -import ru.practicum.ewm.stats.avro.ActionTypeAvro; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; - -@Slf4j -@Service -public class SimilarityService { - - private final Map> weights = new HashMap<>(); - - private final Map eventWeightsSum = new HashMap<>(); - - private final MinWeightsMatrix minWeightsMatrix = new MinWeightsMatrix(); - - private final KafkaTemplate kafkaTemplate; - private final KafkaProperties props; - - public SimilarityService(KafkaTemplate kafkaTemplate, - KafkaProperties props) { - this.kafkaTemplate = kafkaTemplate; - this.props = props; - } - - public void processUserAction(UserActionAvro action) { - long userId = action.getUserId(); - long eventId = action.getEventId(); - int newWeight = convertActionType(action.getActionType()); - long timestampMillis = action.getTimestamp(); - Instant timestamp = Instant.ofEpochMilli(timestampMillis); - - Map userMap = weights.computeIfAbsent(eventId, e -> new HashMap<>()); - int oldWeight = userMap.getOrDefault(userId, 0); - - if (newWeight <= oldWeight) { - log.debug("Обновление не требуется: userId={}, eventId={}, weight={} <= oldWeight={}", - userId, eventId, newWeight, oldWeight); - return; - } - - userMap.put(userId, newWeight); - - int oldSum = eventWeightsSum.getOrDefault(eventId, 0); - int diff = newWeight - oldWeight; - int updatedSum = oldSum + diff; - eventWeightsSum.put(eventId, updatedSum); - - weights.keySet() - .stream() - .filter(otherEvent -> otherEvent.equals(eventId)) - .forEach(otherEvent -> updatePairSimilarity(eventId, otherEvent, timestamp)); - } - - private void updatePairSimilarity(long eventA, long eventB, Instant timestamp) { - double sMin = calcSMin(eventA, eventB); - minWeightsMatrix.put(eventA, eventB, sMin); - - double sA = eventWeightsSum.getOrDefault(eventA, 0); - double sB = eventWeightsSum.getOrDefault(eventB, 0); - if (sA == 0 || sB == 0) { - - log.debug("Обнаружена нулевая сумма (sA={}, sB={}), пропускающая сходство для событий {} и {}", - sA, sB, eventA, eventB); - return; - } - - float similarity = (float) (sMin / (sA * sB)); - - long first = Math.min(eventA, eventB); - long second = Math.max(eventA, eventB); - - EventSimilarityAvro similarityMsg = EventSimilarityAvro.newBuilder() - .setEventA(first) - .setEventB(second) - .setScore(similarity) - .setTimestamp(timestamp) - .build(); - - kafkaTemplate.send(props.getProducer().getTopic(), similarityMsg); - - log.debug("Обновлено сходство для (A={}, B={}) => {}", first, second, similarity); - } - - private double calcSMin(long eventA, long eventB) { - Map userMapA = weights.getOrDefault(eventA, Map.of()); - Map userMapB = weights.getOrDefault(eventB, Map.of()); - - return userMapA.entrySet().stream() - .filter(e -> userMapB.get(e.getKey()) != null) - .mapToDouble(e -> Math.min(e.getValue(), userMapB.get(e.getKey()))) - .sum(); - } - - private int convertActionType(ActionTypeAvro actionType) { - return switch (actionType) { - case REGISTER -> 2; - case LIKE -> 3; - default -> 1; - }; - } -} \ No newline at end of file diff --git a/stats/aggregator/src/main/resources/application.yml b/stats/aggregator/src/main/resources/application.yml index 2b1f593..5254cd6 100644 --- a/stats/aggregator/src/main/resources/application.yml +++ b/stats/aggregator/src/main/resources/application.yml @@ -5,24 +5,14 @@ spring: import: "configserver:" cloud: config: + discovery: + enabled: true + serviceId: config-server fail-fast: true retry: useRandomPolicy: true max-interval: 6000 - discovery: - enabled: true - service-id: config-server - eureka: client: - register-with-eureka: true - fetch-registry: true serviceUrl: - defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ - instance: - preferIpAddress: true - hostname: localhost - instance-id: "${spring.application.name}:${random.value}" - leaseRenewalIntervalInSeconds: 10 -server: - port: 0 \ No newline at end of file + defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ \ No newline at end of file diff --git a/stats/analyzer/pom.xml b/stats/analyzer/pom.xml index b77554d..4ef8a15 100644 --- a/stats/analyzer/pom.xml +++ b/stats/analyzer/pom.xml @@ -15,129 +15,86 @@ 21 21 UTF-8 - 0.0.1-SNAPSHOT - - - net.devh - grpc-server-spring-boot-starter - - - - org.springframework.boot - spring-boot-starter-web - - org.springframework.boot spring-boot-starter-data-jpa - org.springframework.boot - spring-boot-starter-actuator - - - - org.postgresql - postgresql - runtime + spring-boot-starter-validation - - com.h2database - h2 - runtime + org.projectlombok + lombok - org.springframework.boot - spring-boot-configuration-processor - true + spring-boot-starter - - org.springframework.boot - spring-boot-starter-test - test + org.apache.kafka + kafka-clients ru.practicum - aggregator + avro-schemas 0.0.1-SNAPSHOT - compile - - org.projectlombok - lombok - true + ru.practicum + proto-schemas + 0.0.1-SNAPSHOT - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client + org.postgresql + postgresql - org.springframework.cloud spring-cloud-starter-config - org.springframework.retry spring-retry org.springframework.cloud - spring-cloud-openfeign-core + spring-cloud-starter-netflix-eureka-client - org.springframework.cloud - spring-cloud-commons + org.mapstruct + mapstruct + 1.5.5.Final - - - org.apache.kafka - kafka-clients + org.mapstruct + mapstruct-processor + 1.5.5.Final + provided - - org.springframework.kafka - spring-kafka + net.devh + grpc-server-spring-boot-starter - - ru.practicum - avro-schemas - ${avro-schemas.version} + org.springframework.boot + spring-boot-starter-actuator - - ru.practicum - proto-schemas - ${avro-schemas.version} + io.grpc + grpc-stub - org.springframework.boot spring-boot-maven-plugin - - org.apache.maven.plugins - maven-compiler-plugin - - 16 - 16 - - - \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java b/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java index 12c8dd6..8544ab8 100644 --- a/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java +++ b/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java @@ -3,12 +3,8 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; -import org.springframework.cloud.openfeign.EnableFeignClients; @SpringBootApplication -@EnableDiscoveryClient -@EnableFeignClients public class AnalyzerApplication { public static void main(String[] args) { SpringApplication.run(AnalyzerApplication.class, args); diff --git a/stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java b/stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java deleted file mode 100644 index b0fa188..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/config/EventsSimilarityConsumer.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.practicum.config; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.stereotype.Component; -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -import ru.practicum.service.event.EventSimilarityService; - -@Slf4j -@Component -@RequiredArgsConstructor -public class EventsSimilarityConsumer { - - private final EventSimilarityService eventSimilarityService; - - @KafkaListener( - topics = "${kafka.events-similarity-consumer.topic}", - containerFactory = "eventSimilarityKafkaListenerFactory" - ) - public void consumeEventSimilarity(EventSimilarityAvro msg) { - log.info("Consumed event similarity: {}", msg); - eventSimilarityService.updateEventSimilarity(msg); - } -} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java deleted file mode 100644 index 9a86cee..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java +++ /dev/null @@ -1,65 +0,0 @@ -package ru.practicum.config; - -import lombok.RequiredArgsConstructor; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; - -import java.util.Map; - -@Configuration -@RequiredArgsConstructor -public class KafkaConfig { - private final KafkaProperties props; - - @Bean - public ConsumerFactory userActionsConsumerFactory() { - final Map config = Map.of( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), - ConsumerConfig.GROUP_ID_CONFIG, props.getUserActionsConsumer().getGroupId(), - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, props.getUserActionsConsumer().getKeyDeserializer(), - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, props.getUserActionsConsumer().getValueDeserializer() - ); - return new DefaultKafkaConsumerFactory<>(config); - } - - @Bean - public ConcurrentKafkaListenerContainerFactory userActionsKafkaListenerFactory() { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(userActionsConsumerFactory()); - return factory; - } - - @Bean - public ConsumerFactory eventSimilarityConsumerFactory() { - Map config = Map.of( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, props.getBootstrapServers(), - ConsumerConfig.GROUP_ID_CONFIG, props.getEventsSimilarityConsumer().getGroupId(), - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, getClassFromString(props.getEventsSimilarityConsumer().getKeyDeserializer()), - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, getClassFromString(props.getEventsSimilarityConsumer().getValueDeserializer()) - ); - return new DefaultKafkaConsumerFactory<>(config); - } - - @Bean - public ConcurrentKafkaListenerContainerFactory eventSimilarityKafkaListenerFactory() { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(eventSimilarityConsumerFactory()); - return factory; - } - - private Class getClassFromString(String className) { - try { - return Class.forName(className); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Unable to load class: " + className, e); - } - } -} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java new file mode 100644 index 0000000..eacf240 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java @@ -0,0 +1,19 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +public class KafkaConsumerSettings { + private String bootstrapServers; + private String keyDeserializer; + private String valueDeserializer; + private String clientId; + private String groupId; + private String maxPollRecords; + private int fetchMaxBytes; + private int maxPartitionFetchBytes; +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java deleted file mode 100644 index 0e3d8d0..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/config/KafkaProperties.java +++ /dev/null @@ -1,22 +0,0 @@ -package ru.practicum.config; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Data -@Component -@ConfigurationProperties(prefix = "kafka") -public class KafkaProperties { - private String bootstrapServers; - private final ConsumerProps userActionsConsumer = new ConsumerProps(); - private final ConsumerProps eventsSimilarityConsumer = new ConsumerProps(); - - @Data - public static class ConsumerProps { - private String topic; - private String groupId; - private String keyDeserializer; - private String valueDeserializer; - } -} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java new file mode 100644 index 0000000..d20995e --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java @@ -0,0 +1,16 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Getter +@Setter +@ConfigurationProperties(prefix = "spring.kafka.topics") +public class KafkaTopics { + private String userActionsTopic; + private String eventsSimilarityTopic; + +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java b/stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java deleted file mode 100644 index a0cca01..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/config/UserActionsConsumer.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.practicum.config; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.stereotype.Component; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.service.user.UserActionService; - -@Slf4j -@RequiredArgsConstructor -@Component -public class UserActionsConsumer { - - private final UserActionService userActionService; - - @KafkaListener( - topics = "${kafka.user-actions-consumer.topic}", - containerFactory = "userActionsKafkaListenerFactory" - ) - public void consumeUserActions(UserActionAvro message) { - log.info("user action: {}", message); - userActionService.updateUserAction(message); - } -} diff --git a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java new file mode 100644 index 0000000..b571086 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java @@ -0,0 +1,67 @@ +package ru.practicum.controller; + +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.server.service.GrpcService; +import ru.practicum.ewm.stats.proto.RecommendationsControllerGrpc; +import ru.practicum.ewm.stats.proto.RecommendationsMessages; +import ru.practicum.service.RecommendationService; + +@GrpcService +@Slf4j +@RequiredArgsConstructor +public class RecommendationController extends RecommendationsControllerGrpc.RecommendationsControllerImplBase { + private final RecommendationService recommendationService; + + @Override + public void getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto eventsRequestProto, + StreamObserver responseObserver) { + try { + recommendationService.getSimilarEvents(eventsRequestProto) + .forEach(responseObserver::onNext); + responseObserver.onCompleted(); + } catch (Exception e) { + log.error("Unexpected error occurred in getSimilarEvents: {}", e.getMessage(), e); + responseObserver.onError( + new RuntimeException("Error while trying to complete getSimilarEvents") + ); + } + } + + @Override + public void getRecommendationsForUser(RecommendationsMessages.UserPredictionsRequestProto request, + StreamObserver responseObserver) { + try { + recommendationService.getRecommendationsForUser(request) + .forEach(responseObserver::onNext); + responseObserver.onCompleted(); + } catch (Exception e) { + log.error("Unexpected error occurred in getRecommendationsForUser: {}", e.getMessage(), e); + responseObserver.onError( + new RuntimeException("Error while trying to complete getRecommendationsForUser") + ); + } + } + + @Override + public void getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto request, + StreamObserver responseObserver) { + try { + log.info("Received request for getting number of activities about event. Stage 1"); + recommendationService.getInteractionsCount(request) + .forEach(responseObserver::onNext); + log.info("Received request for getting number of activities about event. Stage 2"); + responseObserver.onCompleted(); + } catch (StatusRuntimeException e) { + log.error("Unexpected error occurred StatusRuntimeException in getSimilarEvents: {}", e.getMessage(), e); + + } catch (Exception e) { + log.error("Unexpected error occurred in getSimilarEvents: {}", e.getMessage(), e); + responseObserver.onError( + new RuntimeException("Error while trying to complete GetSimilarEvents") + ); + } + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java deleted file mode 100644 index 27fcb07..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationsController.java +++ /dev/null @@ -1,106 +0,0 @@ -package ru.practicum.controller; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import ru.practicum.ewm.stats.proto.RecommendationsControllerGrpc; -import ru.practicum.ewm.stats.proto.RecommendationsMessages; -import ru.practicum.model.RecommendedEvent; -import ru.practicum.service.RecommendationService; - -import java.util.List; - -@GrpcService -@RequiredArgsConstructor -@Slf4j -public class RecommendationsController extends RecommendationsControllerGrpc.RecommendationsControllerImplBase { - - private final RecommendationService recommendationService; - - @Override - public void getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto requestProto, - StreamObserver streamObserver) { - try { - List recommendedEvents = recommendationService.getSimilarEvents(requestProto); - for (RecommendedEvent recommendedEvent : recommendedEvents) { - RecommendationsMessages.RecommendedEventProto recommendedEventProto = - RecommendationsMessages.RecommendedEventProto.newBuilder() - .setEventId(recommendedEvent.eventId()) - .setScore(recommendedEvent.score()) - .build(); - streamObserver.onNext(recommendedEventProto); - } - streamObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal argument in getSimilarEvents: {}", e.getMessage(), e); - streamObserver.onError( - new StatusRuntimeException( - Status.INVALID_ARGUMENT.withDescription("unexpected error occurred").withCause(e)) - ); - } catch (Exception e) { - log.error("unexpected error occurred in getSimilarEvents: {}", e.getMessage(), e); - streamObserver.onError( - new StatusRuntimeException(Status.UNKNOWN.withDescription("unexpected error occurred").withCause(e)) - ); - } - } - - @Override - public void getRecommendationsForUser( - RecommendationsMessages.UserPredictionsRequestProto requestProto, - StreamObserver streamObserver - ) { - try { - List events = recommendationService.getRecommendationsForUser(requestProto); - for (RecommendedEvent event : events) { - RecommendationsMessages.RecommendedEventProto recommendedEventProto = - RecommendationsMessages.RecommendedEventProto.newBuilder() - .setEventId(event.eventId()) - .setScore(event.score()) - .build(); - streamObserver.onNext(recommendedEventProto); - } - streamObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal argument in getRecommendationsForUser: {}", e.getMessage(), e); - streamObserver.onError( - new StatusRuntimeException( - Status.INVALID_ARGUMENT.withDescription("unexpected error occurred").withCause(e)) - ); - } catch (Exception e) { - log.error("unexpected error occurred in getRecommendationsForUser: {}", e.getMessage(), e); - streamObserver.onError( - new StatusRuntimeException(Status.UNKNOWN.withDescription("unexpected error occurred").withCause(e)) - ); - } - } - - @Override - public void getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto requestProto, - StreamObserver streamObserver) { - try { - List events = recommendationService.getInteractionsCount(requestProto); - for (RecommendedEvent event : events) { - RecommendationsMessages.RecommendedEventProto eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() - .setEventId(event.eventId()) - .setScore(event.score()) - .build(); - streamObserver.onNext(eventProto); - } - streamObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal argument in getInteractionsCount: {}", e.getMessage(), e); - streamObserver.onError( - new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e)) - ); - } catch (Exception e) { - log.error("unexpected error occurred in getInteractionsCount: {}", e.getMessage(), e); - streamObserver.onError( - new StatusRuntimeException(Status.UNKNOWN.withDescription("unexpected error occurred").withCause(e)) - ); - } - } -} diff --git a/stats/analyzer/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java b/stats/analyzer/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java new file mode 100644 index 0000000..52d1c4d --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/deserializer/BaseAvroDeserializer.java @@ -0,0 +1,41 @@ +package ru.practicum.deserializer; + +import org.apache.avro.Schema; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.common.serialization.Deserializer; + +public class BaseAvroDeserializer implements Deserializer { + + private final DecoderFactory decoderFactory; + private final DatumReader reader; + + public BaseAvroDeserializer(Schema schema) { + this(DecoderFactory.get(), schema); + } + + public BaseAvroDeserializer(DecoderFactory decoderFactory, Schema schema) { + this.decoderFactory = decoderFactory; + this.reader = new SpecificDatumReader<>(schema); + + } + + + @Override + public T deserialize(String topic, byte[] data) { + + try { + if (data != null) { + + BinaryDecoder decoder = decoderFactory.binaryDecoder(data, null); + return reader.read(null, decoder); + } + return null; + } catch (Exception e) { + throw new RuntimeException("Data serialization from topic error [" + topic + "]", e); + } + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/deserializer/EventSimilarityDeserializer.java b/stats/analyzer/src/main/java/ru/practicum/deserializer/EventSimilarityDeserializer.java new file mode 100644 index 0000000..68326e6 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/deserializer/EventSimilarityDeserializer.java @@ -0,0 +1,11 @@ +package ru.practicum.deserializer; + +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; + + +public class EventSimilarityDeserializer extends BaseAvroDeserializer { + + public EventSimilarityDeserializer() { + super(EventSimilarityAvro.getClassSchema()); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java b/stats/analyzer/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java new file mode 100644 index 0000000..6be984d --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/deserializer/UserActionDeserializer.java @@ -0,0 +1,10 @@ +package ru.practicum.deserializer; + +import ru.practicum.ewm.stats.avro.UserActionAvro; + +public class UserActionDeserializer extends BaseAvroDeserializer { + public UserActionDeserializer() { + super(UserActionAvro.getClassSchema()); + } + +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java b/stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java new file mode 100644 index 0000000..e7f5876 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java @@ -0,0 +1,61 @@ +package ru.practicum.kafka; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import ru.practicum.config.KafkaConsumerSettings; + +import java.util.Properties; + +@Getter +@Configuration +public class ConfigKafkaProperties { + KafkaConsumerSettings kafkaSettings; + + @Bean(name = "user-actions") + @Qualifier("user-actions") + @ConfigurationProperties(prefix = "spring.kafka.consumer-user-actions") + protected KafkaConsumerSettings kafkaSnapshotKafkaConfig() { + return new KafkaConsumerSettings(); + } + + @Bean(name = "events-similarity") + @Qualifier("events-similarity") + @ConfigurationProperties(prefix = "spring.kafka.consumer-events-similarity") + protected KafkaConsumerSettings kafkaHubKafkaConfig() { + return new KafkaConsumerSettings(); + } + + public Properties getSnapshotProperties() { + kafkaSettings = kafkaSnapshotKafkaConfig(); + return getProperties(kafkaSettings); + } + + public Properties getHubProperties() { + kafkaSettings = kafkaHubKafkaConfig(); + return getProperties(kafkaSettings); + } + + private Properties getProperties(KafkaConsumerSettings kafkaSettings) { + Properties properties = new Properties(); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.CLIENT_ID_CONFIG, + kafkaSettings.getClientId()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG, + kafkaSettings.getGroupId()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + kafkaSettings.getBootstrapServers()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + kafkaSettings.getKeyDeserializer()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + kafkaSettings.getValueDeserializer()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.MAX_POLL_RECORDS_CONFIG, + kafkaSettings.getMaxPollRecords()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.FETCH_MAX_BYTES_CONFIG, + kafkaSettings.getFetchMaxBytes()); + properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, + kafkaSettings.getMaxPartitionFetchBytes()); + return properties; + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java b/stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java new file mode 100644 index 0000000..37be0c7 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java @@ -0,0 +1,10 @@ +package ru.practicum.mapper; + +import org.mapstruct.Mapper; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.model.EventSimilarity; + +@Mapper(componentModel = "spring") +public interface EventSimilarityMapper { + EventSimilarity map(EventSimilarityAvro avro); +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java index 3976ebd..b9dfdab 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java @@ -1,11 +1,8 @@ package ru.practicum.model; import jakarta.persistence.*; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.*; +import lombok.experimental.FieldDefaults; import java.time.Instant; @@ -16,13 +13,14 @@ @Builder @Entity @Table(name = "events_similarity") +@FieldDefaults(level = AccessLevel.PRIVATE) public class EventSimilarity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; + Long id; - private Long eventA; - private Long eventB; - private Float score; - private Instant timestamp; + Long eventA; + Long eventB; + Double score; + Instant timestamp; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java index b7f6f32..686c4dc 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java @@ -29,7 +29,7 @@ public class UserAction { private Long userId; private Long eventId; - private Double maxWeight; + private Double score; private Instant lastInteraction; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java b/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java new file mode 100644 index 0000000..6e27c7b --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java @@ -0,0 +1,90 @@ +package ru.practicum.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.config.KafkaTopics; +import ru.practicum.kafka.ConfigKafkaProperties; +import ru.practicum.service.event.EventSimilarityService; + +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +@Slf4j +@Component +@RequiredArgsConstructor +public class EventSimilarityProcessor implements Runnable { + @Value(value = "${spring.kafka.consumer-events-similarity.consume-attempts-timeout-ms}") + private Duration consumeAttemptTimeout; + private final Map currentOffsets = new HashMap<>();// снимок состояния + private final ConfigKafkaProperties configClass; + private final EventSimilarityService eventSimilarityService; + private final KafkaTopics kafkaTopics; + + @Override + public void run() { + Properties config = configClass.getSnapshotProperties(); + KafkaConsumer consumer = new KafkaConsumer<>(config); + Runtime.getRuntime().addShutdownHook(new Thread(consumer::wakeup)); + + try { + consumer.subscribe(List.of(kafkaTopics.getEventsSimilarityTopic())); + while (true) { + ConsumerRecords records = consumer.poll(consumeAttemptTimeout); + int count = 0; + for (ConsumerRecord record : records) { + handleRecord(record); + manageOffsets(record, count, consumer); + count++; + } + consumer.commitAsync(); + } + } catch (WakeupException | InterruptedException ignores) { + } catch (Exception e) { + log.error("Error while reading", e); + } finally { + + try { + consumer.commitSync(currentOffsets); + } finally { + log.info("Closing consumer"); + consumer.close(); + + } + } + } + + private void manageOffsets(ConsumerRecord record, int count, + KafkaConsumer consumer) { + currentOffsets.put( + new TopicPartition(record.topic(), record.partition()), + new OffsetAndMetadata(record.offset() + 1) + ); + + if (count % 10 == 0) { + consumer.commitAsync(currentOffsets, (offsets, exception) -> { + if (exception != null) { + log.warn("Ошибка во время фиксации оффсетов: {}", offsets, exception); + } + }); + } + } + + private void handleRecord(ConsumerRecord record) throws InterruptedException { + + log.info("топик = {}, партиция = {}, смещение = {}, значение: {}\n", + record.topic(), record.partition(), record.offset(), record.value()); + eventSimilarityService.handleEventSimilarity(record.value()); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java b/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java new file mode 100644 index 0000000..54ec9ad --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java @@ -0,0 +1,89 @@ +package ru.practicum.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.config.KafkaTopics; +import ru.practicum.kafka.ConfigKafkaProperties; +import ru.practicum.service.user.UserActionService; + +import java.time.Duration; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + + +@Slf4j +@Component +@RequiredArgsConstructor +public class UserActionEventProcessor implements Runnable { + + @Value(value = "${spring.kafka.consumer-user-actions.consume-attempts-timeout-ms}") + private Duration consumeAttemptTimeout; + private final ConcurrentHashMap currentOffsets = new ConcurrentHashMap<>();// снимок состояния + private final ConfigKafkaProperties consumerConfig; + private final UserActionService userActionService; + private final KafkaTopics kafkaTopics; + + @Override + public void run() { + Properties config = consumerConfig.getHubProperties(); + KafkaConsumer consumer = new KafkaConsumer<>(config); + Runtime.getRuntime().addShutdownHook(new Thread(consumer::wakeup)); + + try { + consumer.subscribe(List.of(kafkaTopics.getUserActionsTopic())); + while (true) { + ConsumerRecords records = consumer.poll(consumeAttemptTimeout); + int count = 0; + for (ConsumerRecord record : records) { + handleRecord(record); + manageOffsets(record, count, consumer); + count++; + } + consumer.commitAsync(); + } + } catch (WakeupException | InterruptedException ignores) { + } catch (Exception e) { + log.error("Error occurred while reading from hubs", e); + } finally { + try { + consumer.commitSync(currentOffsets); + } finally { + log.info("Closing consumers"); + consumer.close(); + } + } + } + + private void manageOffsets(ConsumerRecord record, int count, + KafkaConsumer consumer) { + currentOffsets.put( + new TopicPartition(record.topic(), record.partition()), + new OffsetAndMetadata(record.offset() + 1) + ); + + if (count % 10 == 0) { + consumer.commitAsync(currentOffsets, (offsets, exception) -> { + if (exception != null) { + log.warn("Error with offset fixation: {}", offsets, exception); + } + }); + } + } + + private void handleRecord(ConsumerRecord record) throws InterruptedException { + + log.info("topic = {}, partition = {}, changing = {}, value: {}\n", + record.topic(), record.partition(), record.offset(), record.value()); + userActionService.handleUserAction(record.value()); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java b/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java index e5a121d..1cbb5ba 100644 --- a/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java +++ b/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java @@ -4,8 +4,13 @@ import ru.practicum.model.EventSimilarity; import java.util.List; +import java.util.Optional; public interface EventSimilarityRepository extends JpaRepository { - List findByEventAOrEventB(Long eventA, Long eventB); + Optional findEventSimilaritiesByEventAAndEventB(long eventA, long eventB); + + List findAllByEventAOrEventB(long eventA, long eventB); + + List findAllByEventAInOrEventBIn(List eventIdsA, List eventIdsB); } diff --git a/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java b/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java index 3e346a3..fac174f 100644 --- a/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java +++ b/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java @@ -4,12 +4,15 @@ import ru.practicum.model.UserAction; import java.util.List; +import java.util.Optional; public interface UserActionRepository extends JpaRepository { - UserAction findByUserIdAndEventId(Long userId, Long eventId); + Optional findByUserIdAndEventId(Long userId, Long eventId); List findByUserId(Long userId); List findByEventId(Long eventId); + + List findByEventIdIsIn(List eventIds); } diff --git a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java index 4eec7f0..8337464 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java @@ -3,6 +3,7 @@ import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import ru.practicum.ewm.stats.proto.RecommendationsMessages; import ru.practicum.model.EventSimilarity; @@ -11,102 +12,156 @@ import ru.practicum.repository.EventSimilarityRepository; import ru.practicum.repository.UserActionRepository; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; +@Slf4j @Service @RequiredArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) public class RecommendationService { - final UserActionRepository userActionRepo; - final EventSimilarityRepository similarityRepo; + private final UserActionRepository userActionRepository; + private final EventSimilarityRepository eventSimilarityRepository; - public List getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto requestProto) { - long eventId = requestProto.getEventId(); - long userId = requestProto.getUserId(); - int maxResults = requestProto.getMaxResults(); + public List getRecommendationsForUser(RecommendationsMessages.UserPredictionsRequestProto requestProto) { + log.info("Recommendations for user: {}", requestProto.getUserId()); + long userId = requestProto.getUserId(); + int maxResult = requestProto.getMaxResults(); - Set interacted = userInteracted(userId); - List result, recList = new ArrayList<>(); + List userActionList = new ArrayList<>(userActionRepository.findByUserId(userId)); + if (userActionList.isEmpty()) { + return Collections.emptyList(); + } - similarityRepo.findByEventAOrEventB(eventId, eventId) - .forEach(e -> { - long other = (e.getEventA() == eventId) ? e.getEventB() : e.getEventA(); - if (!interacted.contains(other)) { - recList.add(new RecommendedEvent(other, e.getScore())); - } - }); - result = recList.stream() - .sorted(Comparator.comparingDouble(RecommendedEvent::score).reversed()).toList(); + userActionList.sort((a, b) -> b.getLastInteraction().compareTo(a.getLastInteraction())); - return result.size() <= maxResults ? result : result.subList(0, maxResults); - } + int n = 10; + List userActionListLimited = userActionList.stream().limit(n).toList(); - public List getRecommendationsForUser(RecommendationsMessages.UserPredictionsRequestProto request) { - long userId = request.getUserId(); - int maxRes = request.getMaxResults(); + Set interactedByUserLimitedEvents = userActionListLimited + .stream() + .map(UserAction::getEventId) + .collect(Collectors.toSet()); - List all = userActionRepo.findByUserId(userId); - if (all.isEmpty()) { - return Collections.emptyList(); - } - all.sort((a,b) -> b.getLastInteraction().compareTo(a.getLastInteraction())); - - int min = Math.min(5, all.size()); - List recent = all.subList(0, min); - - Set interacted = userInteracted(userId); - - Map bestScoreMap = new HashMap<>(); - for (UserAction r : recent) { - long ev = r.getEventId(); - List simList = similarityRepo.findByEventAOrEventB(ev, ev); - for (EventSimilarity e : simList) { - long other = (e.getEventA() == ev) ? e.getEventB() : e.getEventA(); - if (interacted.contains(other)) { - continue; - } - float oldVal = bestScoreMap.getOrDefault(other, 0f); - if (e.getScore() > oldVal) { - bestScoreMap.put(other, e.getScore()); - } - } - } + Set interactedByUserAllEvents = userActionList + .stream() + .map(UserAction::getEventId) + .collect(Collectors.toSet()); - return bestScoreMap.entrySet().stream() - .map(e -> new RecommendedEvent(e.getKey(), e.getValue())) - .sorted(Comparator.comparingDouble(RecommendedEvent::score).reversed()) - .limit(maxRes) - .collect(Collectors.toList()); - } + List eventsToRequest = interactedByUserLimitedEvents.stream().toList(); + List eventSimilarityList = eventSimilarityRepository + .findAllByEventAInOrEventBIn(eventsToRequest, eventsToRequest); + + Set uniqueEvents = new HashSet<>(eventSimilarityList); + + List sortedList = uniqueEvents.stream() + .sorted(Comparator.comparing(EventSimilarity::getScore).reversed()) + .toList(); + + List unwatchedSotedLimitedEventsList = eventSimilarityList.stream() + .collect(Collectors.flatMapping( + es -> Stream.of(es.getEventA(), es.getEventB()), + Collectors.toList() + )).stream() + .filter(interactedByUserAllEvents::contains) + .limit(n) + .toList(); + + + Map weightedScoreForUnwatchedEvent = new HashMap<>(); - public List getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto request) { - List events = request.getEventIdList(); - List result = new ArrayList<>(); + for (EventSimilarity eventSimilarity : sortedList) { + long eventId = eventsToRequest.contains(eventSimilarity.getEventA()) + ? eventSimilarity.getEventA() : eventSimilarity.getEventB(); - for (Long e : events) { - List list = userActionRepo.findByEventId(e); - double sum = 0.0; - for (UserAction uae : list) { - sum += uae.getMaxWeight(); + if (!weightedScoreForUnwatchedEvent.containsKey(eventId)) { + double score = eventSimilarity.getScore() * userActionList.stream() + .filter(event -> event.getEventId() == eventId) + .findFirst().map(UserAction::getScore).orElse(1.0); + + weightedScoreForUnwatchedEvent.put(eventId, score); + } else { + double score = eventSimilarity.getScore() * userActionList.stream() + .filter(event -> event.getEventId() == eventId) + .findFirst().map(UserAction::getScore).orElse(1.0); + double oldScore = weightedScoreForUnwatchedEvent.get(eventId); + weightedScoreForUnwatchedEvent.put(eventId, score + oldScore); } - result.add(new RecommendedEvent(e, (float) sum)); } - return result; + + List eventProtoList = new ArrayList<>(); + for (Long eventId : weightedScoreForUnwatchedEvent.keySet()) { + + RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(eventId) + .setScore(weightedScoreForUnwatchedEvent.get(eventId) / weightedScoreForUnwatchedEvent.size()); + } + + return eventProtoList; } - private Set userInteracted(long userId) { - return userActionRepo.findByUserId(userId) + public List getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto eventsRequestProto) { + long eventId = eventsRequestProto.getEventId(); + List similarEventitsList = eventSimilarityRepository.findAllByEventAOrEventB(eventId, eventId); + Set watchedByUserEvents = userActionRepository.findByUserId(eventsRequestProto.getUserId()) .stream() .map(UserAction::getEventId) .collect(Collectors.toSet()); + List finalEventList = new ArrayList<>(similarEventitsList); + for (EventSimilarity eventSimilarity : similarEventitsList) { + if (watchedByUserEvents.contains(eventSimilarity.getEventA()) && watchedByUserEvents.contains(eventSimilarity.getEventB())) { + finalEventList.remove(eventSimilarity); + } + } + List recommendedEventList = new ArrayList<>(); + for (EventSimilarity eventSimilarity : finalEventList) { + RecommendationsMessages.RecommendedEventProto eventProto; + if (eventsRequestProto.getEventId() != eventSimilarity.getEventA()) { + eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(eventSimilarity.getEventA()) + .setScore(eventSimilarity.getScore()) + .build(); + } else { + eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(eventSimilarity.getEventB()) + .setScore(eventSimilarity.getScore()) + .build(); + } + recommendedEventList.add(eventProto); + } + + return recommendedEventList.stream().sorted(Comparator.comparingDouble(RecommendationsMessages.RecommendedEventProto::getScore) + .reversed()).limit(eventsRequestProto.getMaxResults()).toList(); } + + public List getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto request) { + log.info("Method getInteractionsCount began its work"); + List userActionList = userActionRepository.findByEventIdIsIn(request.getEventIdList()); + Map recommendedEventMap = new HashMap<>(); + List recommendedEventList = new ArrayList<>(); + + for (UserAction userAction : userActionList) { + if (!recommendedEventMap.containsKey(userAction.getEventId())) { + + recommendedEventMap.put(userAction.getEventId(), userAction.getScore()); + } else { + + Double newScore = recommendedEventMap.get(userAction.getEventId()) + userAction.getScore(); + recommendedEventMap.put(userAction.getEventId(), newScore); + } + } + for (long eventId : recommendedEventMap.keySet()) { + RecommendationsMessages.RecommendedEventProto eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + .setEventId(eventId) + .setScore(recommendedEventMap.get(eventId)) + .build(); + recommendedEventList.add(eventProto); + } + log.info("Method getInteractionsCount ended its work"); + return recommendedEventList; + } + } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java index 0686ad7..cf89782 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java @@ -3,5 +3,6 @@ import ru.practicum.ewm.stats.avro.EventSimilarityAvro; public interface EventSimilarityService { - void updateEventSimilarity(EventSimilarityAvro eventSimilarityAvro); + + void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro); } diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java index 814fdd0..2a043ff 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java @@ -1,50 +1,39 @@ -package ru.practicum.service.event; +package ru.practicum.service; -import lombok.RequiredArgsConstructor; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.mapper.EventSimilarityMapper; import ru.practicum.model.EventSimilarity; import ru.practicum.repository.EventSimilarityRepository; +import ru.practicum.service.event.EventSimilarityService; -import java.time.Instant; +import java.util.Optional; -@Slf4j @Service -@RequiredArgsConstructor +@Slf4j +@AllArgsConstructor public class EventSimilarityServiceImpl implements EventSimilarityService { private final EventSimilarityRepository eventSimilarityRepository; + private final EventSimilarityMapper eventSimilarityMapper; + @Override - public void updateEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { - long eventA = eventSimilarityAvro.getEventA(); - long eventB = eventSimilarityAvro.getEventB(); - float score = eventSimilarityAvro.getScore(); - Instant timestamp = eventSimilarityAvro.getTimestamp(); - - EventSimilarity existingEventSimilarity = findPair(eventA, eventB); - - if (existingEventSimilarity == null) { - existingEventSimilarity = new EventSimilarity(); - existingEventSimilarity.setEventA(eventA); - existingEventSimilarity.setEventB(eventB); - existingEventSimilarity.setScore(score); - existingEventSimilarity.setTimestamp(timestamp); - eventSimilarityRepository.save(existingEventSimilarity); + public void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { + log.info("сервис EventSimilarityServiceImpl начал обработку eventSimilarityAvro {}", eventSimilarityAvro); + EventSimilarity eventSimilarity = eventSimilarityMapper.map(eventSimilarityAvro); + Optional eventSimilarityOptional = eventSimilarityRepository + .findEventSimilaritiesByEventAAndEventB(eventSimilarityAvro.getEventA(), eventSimilarityAvro.getEventB()); + + if (eventSimilarityOptional.isPresent()) { + eventSimilarityOptional.get().setScore(eventSimilarity.getScore()); + eventSimilarityOptional.get().setTimestamp(eventSimilarity.getTimestamp()); + eventSimilarityRepository.save(eventSimilarityOptional.get()); + } else { - existingEventSimilarity.setScore(score); - existingEventSimilarity.setTimestamp(timestamp); - eventSimilarityRepository.save(existingEventSimilarity); + eventSimilarityRepository.save(eventSimilarity); } } - - private EventSimilarity findPair(long eventA, long eventB) { - return eventSimilarityRepository.findByEventAOrEventB(eventA, eventB) - .stream() - .filter(e -> (e.getEventA().equals(eventA) && e.getEventB().equals(eventB)) - || (e.getEventA().equals(eventB) && e.getEventB().equals(eventA))) - .findFirst() - .orElse(null); - } -} +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java index 59ecda9..b462ac1 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java @@ -3,5 +3,5 @@ import ru.practicum.ewm.stats.avro.UserActionAvro; public interface UserActionService { - void updateUserAction(UserActionAvro userActionAvro); + void handleUserAction(UserActionAvro userActionAvro); } diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java index 0e46784..d885aaf 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java @@ -1,6 +1,6 @@ package ru.practicum.service.user; -import lombok.RequiredArgsConstructor; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import ru.practicum.ewm.stats.avro.ActionTypeAvro; @@ -8,51 +8,49 @@ import ru.practicum.model.UserAction; import ru.practicum.repository.UserActionRepository; -import java.time.Instant; +import java.util.Optional; + @Service @Slf4j -@RequiredArgsConstructor -public class UserActionServiceImpl implements UserActionService { +@AllArgsConstructor +public class +UserActionServiceImpl implements UserActionService { private final UserActionRepository userActionRepository; @Override - public void updateUserAction(UserActionAvro userActionAvro) { - long userId = userActionAvro.getUserId(); - long eventId = userActionAvro.getEventId(); - double newWeight = convertWeight(userActionAvro.getActionType()); - long timestamp = userActionAvro.getTimestamp(); - - Instant interactionTime = Instant.ofEpochMilli(timestamp); - - UserAction userAction = userActionRepository.findByUserIdAndEventId(userId, eventId); - - if (userAction == null) { - userAction = new UserAction(); - userAction.setUserId(userId); - userAction.setEventId(eventId); - userAction.setMaxWeight(newWeight); - userAction.setLastInteraction(interactionTime); + public void handleUserAction(UserActionAvro userActionAvro) { + + Optional userActionOptional = userActionRepository.findByUserIdAndEventId(userActionAvro.getUserId(), + userActionAvro.getEventId()); + if (userActionOptional.isPresent()) { + if (userActionOptional.get().getScore() <= calcInteractionScore(userActionAvro.getActionType())) { + UserAction userAction = UserAction.builder() + .id(userActionOptional.get().getId()) + .userId(userActionAvro.getUserId()) + .lastInteraction(userActionAvro.getTimestamp()) + .eventId(userActionAvro.getEventId()) + .score(calcInteractionScore(userActionAvro.getActionType())) + .build(); + userActionRepository.save(userAction); + } + } else { + UserAction userAction = UserAction.builder() + .userId(userActionAvro.getUserId()) + .lastInteraction(userActionAvro.getTimestamp()) + .eventId(userActionAvro.getEventId()) + .score(calcInteractionScore(userActionAvro.getActionType())) + .build(); userActionRepository.save(userAction); - return; - } - if (newWeight > userAction.getMaxWeight()) { - userAction.setMaxWeight(newWeight); - } - if (interactionTime.isAfter(userAction.getLastInteraction())) { - userAction.setLastInteraction(interactionTime); } - userActionRepository.save(userAction); - - } - private double convertWeight(ActionTypeAvro actionType) { - return switch (actionType) { + private double calcInteractionScore(ActionTypeAvro type) { + return switch (type) { + case VIEW -> 0.4; case REGISTER -> 0.8; - case LIKE -> 1; - default -> 0.4; + case LIKE -> 1.0; }; } -} +} \ No newline at end of file diff --git a/stats/analyzer/src/main/resources/application.yml b/stats/analyzer/src/main/resources/application.yml index ccc4033..ecaf492 100644 --- a/stats/analyzer/src/main/resources/application.yml +++ b/stats/analyzer/src/main/resources/application.yml @@ -5,24 +5,15 @@ spring: import: "configserver:" cloud: config: + discovery: + enabled: true + serviceId: config-server + enabled: true fail-fast: true retry: useRandomPolicy: true max-interval: 6000 - discovery: - enabled: true - service-id: config-server - eureka: client: - register-with-eureka: true - fetch-registry: true serviceUrl: - defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ - instance: - prefer-ip-address: true - hostname: localhost - instance-id: "${spring.application.name}:${random.value}" - lease-renewal-interval-in-seconds: 10 -server: - port: 0 \ No newline at end of file + defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/CollectorApplication.java b/stats/collector/src/main/java/ru/practicum/CollectorApplication.java index deb088a..e424df4 100644 --- a/stats/collector/src/main/java/ru/practicum/CollectorApplication.java +++ b/stats/collector/src/main/java/ru/practicum/CollectorApplication.java @@ -2,12 +2,8 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.ConfigurationPropertiesScan; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; @SpringBootApplication -@EnableDiscoveryClient -@ConfigurationPropertiesScan public class CollectorApplication { public static void main(String[] args) { SpringApplication.run(CollectorApplication.class, args); diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java index d8e861b..cd28f2b 100644 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java @@ -1,38 +1,38 @@ -package ru.practicum.config; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.core.ProducerFactory; -import ru.practicum.ewm.stats.avro.UserActionAvro; - -import java.util.Map; - -@Getter -@Setter -@Configuration -@RequiredArgsConstructor -public class KafkaProducerConfig { - - private final KafkaProperties kafkaProperties; - - @Bean - public ProducerFactory kafkaProducer() { - Map props = Map.of( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers(), - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getKeySerializer(), - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getValueSerializer() - ); - return new DefaultKafkaProducerFactory<>(props); - } - - @Bean - public KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(kafkaProducer()); - } -} \ No newline at end of file +//package ru.practicum.config; +// +//import lombok.Getter; +//import lombok.RequiredArgsConstructor; +//import lombok.Setter; +//import org.apache.kafka.clients.producer.ProducerConfig; +//import org.springframework.context.annotation.Bean; +//import org.springframework.context.annotation.Configuration; +//import org.springframework.kafka.core.DefaultKafkaProducerFactory; +//import org.springframework.kafka.core.KafkaTemplate; +//import org.springframework.kafka.core.ProducerFactory; +//import ru.practicum.ewm.stats.avro.UserActionAvro; +// +//import java.util.Map; +// +//@Getter +//@Setter +//@Configuration +//@RequiredArgsConstructor +//public class KafkaProducerConfig { +// +// private final KafkaProperties kafkaProperties; +// +// @Bean +// public ProducerFactory kafkaProducer() { +// Map props = Map.of( +// ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers(), +// ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getKeySerializer(), +// ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getValueSerializer() +// ); +// return new DefaultKafkaProducerFactory<>(props); +// } +// +// @Bean +// public KafkaTemplate kafkaTemplate() { +// return new KafkaTemplate<>(kafkaProducer()); +// } +//} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java index fe43fd3..2fad01a 100644 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java @@ -1,29 +1,29 @@ -package ru.practicum.config; - -import lombok.AccessLevel; -import lombok.Data; -import lombok.experimental.FieldDefaults; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Data -@Component -@ConfigurationProperties("spring.kafka") -@FieldDefaults(level = AccessLevel.PRIVATE) -public class KafkaProperties { - - String bootstrapServers; - - Producer producer = new Producer(); - - @Value("${collector.kafka.topic}") - String userActionsTopic; - - @Data - @FieldDefaults(level = AccessLevel.PRIVATE) - public static class Producer { - String keySerializer; - String valueSerializer; - } -} +//package ru.practicum.config; +// +//import lombok.AccessLevel; +//import lombok.Data; +//import lombok.experimental.FieldDefaults; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.boot.context.properties.ConfigurationProperties; +//import org.springframework.stereotype.Component; +// +//@Data +//@Component +//@ConfigurationProperties("spring.kafka") +//@FieldDefaults(level = AccessLevel.PRIVATE) +//public class KafkaProperties { +// +// String bootstrapServers; +// +// Producer producer = new Producer(); +// +// @Value("${collector.kafka.topic}") +// String userActionsTopic; +// +// @Data +// @FieldDefaults(level = AccessLevel.PRIVATE) +// public static class Producer { +// String keySerializer; +// String valueSerializer; +// } +//} diff --git a/stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java b/stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java new file mode 100644 index 0000000..60016fc --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java @@ -0,0 +1,11 @@ +package ru.practicum.config; + +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.Producer; + +public interface UserActionProducer { + + Producer getProducer(); + + void stop(); +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java b/stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java new file mode 100644 index 0000000..269bb62 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java @@ -0,0 +1,70 @@ +package ru.practicum.config; + +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Properties; + +@Slf4j +@RequiredArgsConstructor +@Configuration +public class UserActionProducerConfiguration { + @Value("${spring.kafka.producer.bootstrap-servers}") + private String bootstrapServers; + @Value("${spring.kafka.producer.key-serializer}") + private String keySerializer; + @Value("${spring.kafka.producer.value-serializer}") + private String valueSerializer; + + @Bean + UserActionProducer getClient() { + return new UserActionProducer() { + + private Producer producer; + + @Override + public Producer getProducer() { + if (producer == null) { + log.info(" Producer пустой, начинает создавтаь новый"); + initProducer(); + } + log.info("Возвращаем готовый продьюсер = {}", producer); + return producer; + } + + private void initProducer() { + log.info("Начало инициализации продьюсера"); + Properties config = new Properties(); + log.info("BOOTSTRAP_SERVERS_CONFIG: {}",bootstrapServers); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + bootstrapServers); + log.info("KEY_SERIALIZER_CLASS_CONFIG: {}",keySerializer); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + keySerializer); + log.info("VALUE_SERIALIZER_CLASS_CONFIG: {}",valueSerializer); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + valueSerializer); + log.info("Подготовили конфиг для продьюсера = {}", config.toString()); + + producer = new KafkaProducer<>(config); + log.info("Закончили инициализацию продьюсера = {}", producer.toString()); + } + + @PreDestroy + @Override + public void stop() { + if (producer != null) { + producer.close(); + } + } + }; + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java b/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java new file mode 100644 index 0000000..a46e7a1 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java @@ -0,0 +1,7 @@ +package ru.practicum.handler; + +import ru.practicum.ewm.stats.proto.UserActionProto; + +public interface ActionsHandlers { + void handle(UserActionProto userActionProto); +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java b/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java new file mode 100644 index 0000000..91e4048 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java @@ -0,0 +1,61 @@ +package ru.practicum.handler; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import ru.practicum.ewm.stats.avro.ActionTypeAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.ewm.stats.proto.ActionTypeProto; +import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.service.KafkaMessageProducer; + +import java.time.Instant; + +@Slf4j +@Component +public class UserActionHandler implements ActionsHandlers { + KafkaMessageProducer kafkaMessageProducer; + + public UserActionHandler(KafkaMessageProducer kafkaMessageProducer) { + this.kafkaMessageProducer = kafkaMessageProducer; + } + + @Override + public void handle(UserActionProto userActionProto) { + log.info("Обработчик UserActionHandler начал работать"); + + log.info("На вход:{}", userActionProto.toString()); + UserActionAvro userActionAvro = new UserActionAvro(); + + userActionAvro.setUserId(userActionProto.getUserId()); + log.info("Установили userId={}", userActionAvro.getUserId()); + + userActionAvro.setActionType(getActionType(userActionProto.getActionType())); + log.info("Установили setActionType={}", userActionAvro.getActionType()); + + userActionAvro.setEventId(userActionProto.getEventId()); + log.info("Установили EventId={}", userActionAvro.getEventId()); + + userActionAvro.setTimestamp(Instant.ofEpochSecond(userActionProto.getTimestamp().getSeconds(), + userActionProto.getTimestamp().getNanos())); + log.info("Установили timestamp={}", userActionAvro.getTimestamp()); + + + log.info("Смапили действие пользователя в AVRO {}", userActionAvro.toString()); + kafkaMessageProducer.sendUserAction(userActionAvro); + + + } + + private ActionTypeAvro getActionType(ActionTypeProto actionTypeProto) { + if (actionTypeProto.equals(ActionTypeProto.ACTION_LIKE)) { + return ActionTypeAvro.LIKE; + } + if (actionTypeProto.equals(ActionTypeProto.ACTION_REGISTER)) { + return ActionTypeAvro.REGISTER; + } + if (actionTypeProto.equals(ActionTypeProto.ACTION_VIEW)) { + return ActionTypeAvro.VIEW; + } + return null; + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java index 9d0e3e6..3534ee1 100644 --- a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java +++ b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java @@ -5,6 +5,8 @@ import ru.practicum.ewm.stats.proto.UserActionProto; import ru.practicum.ewm.stats.avro.UserActionAvro; +import java.time.Instant; + public class UserActionMapper { public static UserActionAvro toAvro(UserActionProto userActionProto) { @@ -15,7 +17,7 @@ public static UserActionAvro toAvro(UserActionProto userActionProto) { .setUserId(userActionProto.getUserId()) .setEventId(userActionProto.getEventId()) .setActionType(toAvroActionType(userActionProto.getActionType())) - .setTimestamp(timestampMillis) + .setTimestamp(Instant.ofEpochSecond(timestampMillis)) .build(); } diff --git a/stats/collector/src/main/java/ru/practicum/serializer/UserActionsAvroSerializer.java b/stats/collector/src/main/java/ru/practicum/serializer/UserActionsAvroSerializer.java new file mode 100644 index 0000000..f6d1e49 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/serializer/UserActionsAvroSerializer.java @@ -0,0 +1,36 @@ +package ru.practicum.serializer; + +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Serializer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +@Slf4j +public class UserActionsAvroSerializer implements Serializer { + private final EncoderFactory encoderFactory = EncoderFactory.get(); + private BinaryEncoder encoder; + + public byte[] serialize(String topic, SpecificRecordBase data) { + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] result = null; + encoder = encoderFactory.binaryEncoder(out, encoder); + if (data != null) { + DatumWriter writer = new SpecificDatumWriter<>(data.getSchema()); + writer.write(data, encoder); + encoder.flush(); + result = out.toByteArray(); + } + return result; + } catch (IOException ex) { + throw new SerializationException("Error with serialization data for topic [" + topic + "]", ex); + } + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java b/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java index 26f5e8b..3422794 100644 --- a/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java +++ b/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java @@ -1,20 +1,32 @@ package ru.practicum.service; import lombok.RequiredArgsConstructor; -import org.springframework.kafka.core.KafkaTemplate; -import ru.practicum.config.KafkaProperties; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.beans.factory.annotation.Value; +import ru.practicum.config.UserActionProducer; import ru.practicum.ewm.stats.avro.UserActionAvro; import org.springframework.stereotype.Component; @Component +@Slf4j @RequiredArgsConstructor public class KafkaMessageProducer implements MessageProducer { - private final KafkaTemplate kafkaTemplate; - private final KafkaProperties properties; + @Value("${spring.kafka.topics.actions-topic}") + private String actionsTopic; + private final UserActionProducer eventClient; + @Override - public void sendUserAction(UserActionAvro userActionAvro) { - kafkaTemplate.send(properties.getUserActionsTopic(), userActionAvro); + public void sendUserAction(UserActionAvro userAction) { + log.info("Готовим сообщение UserActionAvro к отправке: {}", getClass()); + log.info("Кафка топик = {}", actionsTopic); + log.info("eventClient = {}", eventClient.getProducer()); + eventClient.getProducer().send(new ProducerRecord<>(actionsTopic, + userAction)); + log.info("Топик: {}", actionsTopic); + log.info("Обработка UserActionAvro завершена, в KAFKA ушло: {}", userAction); } + } \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/UserActionController.java b/stats/collector/src/main/java/ru/practicum/service/UserActionController.java index ab25fe8..f7569c6 100644 --- a/stats/collector/src/main/java/ru/practicum/service/UserActionController.java +++ b/stats/collector/src/main/java/ru/practicum/service/UserActionController.java @@ -9,6 +9,7 @@ import net.devh.boot.grpc.server.service.GrpcService; import ru.practicum.ewm.stats.proto.UserActionControllerGrpc; import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.handler.ActionsHandlers; import ru.practicum.mapper.UserActionMapper; import ru.practicum.ewm.stats.avro.UserActionAvro; @@ -16,15 +17,13 @@ @GrpcService @RequiredArgsConstructor public class UserActionController extends UserActionControllerGrpc.UserActionControllerImplBase { - - private final MessageProducer messageProducer; + private final ActionsHandlers actionHandler; @Override public void collectUserAction(UserActionProto request, StreamObserver responseObserver) { try { - UserActionAvro userActionAvro = UserActionMapper.toAvro(request); - messageProducer.sendUserAction(userActionAvro); - responseObserver.onNext(Empty.getDefaultInstance()); + actionHandler.handle(request); + responseObserver.onNext(Empty.newBuilder().build()); responseObserver.onCompleted(); } catch (IllegalArgumentException e) { log.error("IllegalArgumentException collectUserAction: {}", e.getMessage(), e); diff --git a/stats/collector/src/main/resources/application.yml b/stats/collector/src/main/resources/application.yml index 5f9dddc..c3b5a2b 100644 --- a/stats/collector/src/main/resources/application.yml +++ b/stats/collector/src/main/resources/application.yml @@ -5,24 +5,15 @@ spring: import: "configserver:" cloud: config: + discovery: + enabled: true + serviceId: config-server + enabled: true fail-fast: true retry: useRandomPolicy: true max-interval: 6000 - discovery: - enabled: true - service-id: config-server - eureka: client: - register-with-eureka: true - fetch-registry: true serviceUrl: - defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ - instance: - prefer-ip-address: true - hostname: localhost - instance-id: "${spring.application.name}:${random.value}" - lease-renewal-interval-in-seconds: 10 -server: - port: 0 \ No newline at end of file + defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ \ No newline at end of file diff --git a/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl b/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl index 60267df..117946e 100644 --- a/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl +++ b/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl @@ -4,7 +4,7 @@ protocol EventSimilarityProtocol { record EventSimilarityAvro { long eventA; long eventB; - float score; + double score; timestamp_ms timestamp; } } \ No newline at end of file diff --git a/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl b/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl index 9920089..b8ea932 100644 --- a/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl +++ b/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl @@ -11,6 +11,6 @@ protocol UserActionProtocol { long userId; long eventId; ActionTypeAvro actionType; - long timestamp; + timestamp_ms timestamp; } } \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java index c4a3ac1..166f30c 100644 --- a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java +++ b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java @@ -33,7 +33,7 @@ public void sendUserAction(long userId, long eventId, ActionTypeProto actionType .setNanos(nanos) ) .build(); - collectorStub.collectUserAction(userActionProto); + Empty response = collectorStub.collectUserAction(userActionProto); log.info("sendUserAction -> Collector answered"); } catch (Exception e) { log.error("Ошибка при отправке действия пользователя: userId={}, eventId={}, actionType={}", From b019e6970130b26b4df1992de0d125234a358f7f Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Fri, 9 May 2025 17:29:25 +0300 Subject: [PATCH 08/26] feat: refactored stats and released recommendations feature --- .../src/main/resources/config/stats/collector/application.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infra/config-server/src/main/resources/config/stats/collector/application.yml b/infra/config-server/src/main/resources/config/stats/collector/application.yml index 74a5680..fa9f262 100644 --- a/infra/config-server/src/main/resources/config/stats/collector/application.yml +++ b/infra/config-server/src/main/resources/config/stats/collector/application.yml @@ -1,3 +1,6 @@ +server: + port: 0 + spring: kafka: producer: From c3a4c4daaa7ffb466ff8f3e85864e2d7b3af63cf Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Fri, 9 May 2025 17:38:57 +0300 Subject: [PATCH 09/26] feat: refactored stats and released recommendations feature --- stats/aggregator/src/main/resources/application.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/stats/aggregator/src/main/resources/application.yml b/stats/aggregator/src/main/resources/application.yml index 5254cd6..3e252c1 100644 --- a/stats/aggregator/src/main/resources/application.yml +++ b/stats/aggregator/src/main/resources/application.yml @@ -8,6 +8,7 @@ spring: discovery: enabled: true serviceId: config-server + enabled: true fail-fast: true retry: useRandomPolicy: true From 2c9225517170016df19bf190163ef084fb196d8d Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Fri, 9 May 2025 17:49:13 +0300 Subject: [PATCH 10/26] feat: refactored stats and released recommendations feature --- stats/aggregator/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stats/aggregator/pom.xml b/stats/aggregator/pom.xml index c8a83ab..d34ad44 100644 --- a/stats/aggregator/pom.xml +++ b/stats/aggregator/pom.xml @@ -18,6 +18,11 @@ + + org.springframework.boot + spring-boot-starter-web + + ru.practicum avro-schemas From b3951ebc0db2da536b66a89876c41ff771d81d47 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Fri, 9 May 2025 18:33:40 +0300 Subject: [PATCH 11/26] feat: refactored stats and released recommendations feature --- .../controller/PublicEventController.java | 8 ++--- .../service/event/EventServiceImpl.java | 4 +-- .../practicum/service/RequestServiceImpl.java | 4 +-- .../controller/RecommendationController.java | 16 +++++----- .../service/RecommendationService.java | 27 ++++++++-------- .../ru/practicum/handler/ActionsHandlers.java | 5 +-- .../practicum/handler/UserActionHandler.java | 13 ++++---- .../ru/practicum/mapper/UserActionMapper.java | 7 ++-- ...ntroller.java => CollectorController.java} | 14 ++++---- .../recommendations_controller.proto | 17 ---------- .../controller/user_action_controller.proto | 10 ------ .../message/recommendation_message.proto} | 5 ++- .../message/user_action_message.proto} | 12 ++++--- .../service/recommendations_service.proto | 12 +++++++ .../stats/service/user_action_service.proto | 11 +++++++ .../java/ru/practicum/AnalyzerClient.java | 32 +++++++++---------- .../java/ru/practicum/CollectorClient.java | 18 +++++------ 17 files changed, 104 insertions(+), 111 deletions(-) rename stats/collector/src/main/java/ru/practicum/service/{UserActionController.java => CollectorController.java} (69%) delete mode 100644 stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto delete mode 100644 stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto rename stats/serialization/proto-schemas/src/main/protobuf/{messages/recommendations_messages.proto => stats/message/recommendation_message.proto} (80%) rename stats/serialization/proto-schemas/src/main/protobuf/{messages/user_action.proto => stats/message/user_action_message.proto} (61%) create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto diff --git a/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java b/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java index d4ba46c..3076047 100644 --- a/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java +++ b/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java @@ -10,9 +10,9 @@ import ru.practicum.dto.event.EventFullDto; import ru.practicum.dto.event.EventRecommendationDto; import ru.practicum.dto.event.EventShortDto; -import ru.practicum.ewm.stats.proto.ActionTypeProto; -import ru.practicum.ewm.stats.proto.RecommendationsMessages; import ru.practicum.exception.IncorrectValueException; +import ru.practicum.grpc.stats.action.UserActionMessage; +import ru.practicum.grpc.stats.recommendation.RecommendationMessage; import ru.practicum.service.event.EventSearchParams; import ru.practicum.service.event.EventService; import ru.practicum.service.event.PublicSearchParams; @@ -110,7 +110,7 @@ public List getRecommendations(@RequestHeader(X_EWM_USER var recommendationList = recommendationStream.toList(); List result = new ArrayList<>(); - for (RecommendationsMessages.RecommendedEventProto requestProto : recommendationList) { + for (RecommendationMessage.RecommendedEventProto requestProto : recommendationList) { result.add(new EventRecommendationDto(requestProto.getEventId(), requestProto.getScore())); } return result; @@ -121,6 +121,6 @@ public void likeEvent(@PathVariable("event-id") Long eventId, @RequestHeader(X_EWM_USER_ID_HEADER) long userId) { eventService.addLike(userId, eventId); - collectorClient.sendUserAction(userId, eventId, ActionTypeProto.ACTION_LIKE); + collectorClient.sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_LIKE); } } diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index 71e6bbd..253e3f4 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -18,10 +18,10 @@ import ru.practicum.enums.AdminStateAction; import ru.practicum.enums.EventState; import ru.practicum.enums.RequestStatus; -import ru.practicum.ewm.stats.proto.ActionTypeProto; import ru.practicum.exception.ConflictException; import ru.practicum.exception.NotFoundException; import ru.practicum.exception.ValidationException; +import ru.practicum.grpc.stats.action.UserActionMessage; import ru.practicum.mapper.event.EventMapper; import ru.practicum.mapper.event.UtilEventClass; import ru.practicum.mapper.location.LocationMapper; @@ -292,7 +292,7 @@ public EventFullDto getEventById(Long eventId, long userId) { eventRepository.save(event); - collectorClient.sendUserAction(userId, eventId, ActionTypeProto.ACTION_VIEW); + collectorClient.sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_VIEW); // Подсчет подтвержденных запросов long confirmedRequests = requestServiceClient.countByStatusAndEventId(RequestStatus.CONFIRMED, eventId); diff --git a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java index e52ba3f..995df48 100644 --- a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java +++ b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java @@ -11,9 +11,9 @@ import ru.practicum.dto.request.EventRequestStatusUpdateResult; import ru.practicum.dto.request.ParticipationRequestDto; import ru.practicum.enums.RequestStatus; -import ru.practicum.ewm.stats.proto.ActionTypeProto; import ru.practicum.exception.ConflictException; import ru.practicum.exception.NotFoundException; +import ru.practicum.grpc.stats.action.UserActionMessage; import ru.practicum.mapper.RequestMapper; import ru.practicum.model.Request; import ru.practicum.repository.RequestRepository; @@ -49,7 +49,7 @@ public ParticipationRequestDto createRequest(Long userId, Long eventId) { requestToEventVerification(userId, eventId); Request request = requestMapper.formUserAndEventToRequest(userId, eventId); requestRepository.save(request); - collectorClient.sendUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER); + collectorClient.sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_REGISTER); return requestMapper.toParticipationRequestDto(request); } diff --git a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java index b571086..814cfef 100644 --- a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java +++ b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java @@ -5,8 +5,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; -import ru.practicum.ewm.stats.proto.RecommendationsControllerGrpc; -import ru.practicum.ewm.stats.proto.RecommendationsMessages; +import ru.practicum.grpc.stats.recommendation.RecommendationMessage; +import ru.practicum.grpc.stats.recommendation.RecommendationsControllerGrpc; import ru.practicum.service.RecommendationService; @GrpcService @@ -16,8 +16,8 @@ public class RecommendationController extends RecommendationsControllerGrpc.Reco private final RecommendationService recommendationService; @Override - public void getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto eventsRequestProto, - StreamObserver responseObserver) { + public void getSimilarEvents(RecommendationMessage.SimilarEventsRequestProto eventsRequestProto, + StreamObserver responseObserver) { try { recommendationService.getSimilarEvents(eventsRequestProto) .forEach(responseObserver::onNext); @@ -31,8 +31,8 @@ public void getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto e } @Override - public void getRecommendationsForUser(RecommendationsMessages.UserPredictionsRequestProto request, - StreamObserver responseObserver) { + public void getRecommendationsForUser(RecommendationMessage.UserPredictionsRequestProto request, + StreamObserver responseObserver) { try { recommendationService.getRecommendationsForUser(request) .forEach(responseObserver::onNext); @@ -46,8 +46,8 @@ public void getRecommendationsForUser(RecommendationsMessages.UserPredictionsReq } @Override - public void getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto request, - StreamObserver responseObserver) { + public void getInteractionsCount(RecommendationMessage.InteractionsCountRequestProto request, + StreamObserver responseObserver) { try { log.info("Received request for getting number of activities about event. Stage 1"); recommendationService.getInteractionsCount(request) diff --git a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java index 8337464..3129bac 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java @@ -5,9 +5,8 @@ import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import ru.practicum.ewm.stats.proto.RecommendationsMessages; +import ru.practicum.grpc.stats.recommendation.RecommendationMessage; import ru.practicum.model.EventSimilarity; -import ru.practicum.model.RecommendedEvent; import ru.practicum.model.UserAction; import ru.practicum.repository.EventSimilarityRepository; import ru.practicum.repository.UserActionRepository; @@ -25,7 +24,7 @@ public class RecommendationService { private final UserActionRepository userActionRepository; private final EventSimilarityRepository eventSimilarityRepository; - public List getRecommendationsForUser(RecommendationsMessages.UserPredictionsRequestProto requestProto) { + public List getRecommendationsForUser(RecommendationMessage.UserPredictionsRequestProto requestProto) { log.info("Recommendations for user: {}", requestProto.getUserId()); long userId = requestProto.getUserId(); int maxResult = requestProto.getMaxResults(); @@ -92,10 +91,10 @@ public List getRecommendationsFor } } - List eventProtoList = new ArrayList<>(); + List eventProtoList = new ArrayList<>(); for (Long eventId : weightedScoreForUnwatchedEvent.keySet()) { - RecommendationsMessages.RecommendedEventProto.newBuilder() + RecommendationMessage.RecommendedEventProto.newBuilder() .setEventId(eventId) .setScore(weightedScoreForUnwatchedEvent.get(eventId) / weightedScoreForUnwatchedEvent.size()); } @@ -103,7 +102,7 @@ public List getRecommendationsFor return eventProtoList; } - public List getSimilarEvents(RecommendationsMessages.SimilarEventsRequestProto eventsRequestProto) { + public List getSimilarEvents(RecommendationMessage.SimilarEventsRequestProto eventsRequestProto) { long eventId = eventsRequestProto.getEventId(); List similarEventitsList = eventSimilarityRepository.findAllByEventAOrEventB(eventId, eventId); Set watchedByUserEvents = userActionRepository.findByUserId(eventsRequestProto.getUserId()) @@ -116,16 +115,16 @@ public List getSimilarEvents(Reco finalEventList.remove(eventSimilarity); } } - List recommendedEventList = new ArrayList<>(); + List recommendedEventList = new ArrayList<>(); for (EventSimilarity eventSimilarity : finalEventList) { - RecommendationsMessages.RecommendedEventProto eventProto; + RecommendationMessage.RecommendedEventProto eventProto; if (eventsRequestProto.getEventId() != eventSimilarity.getEventA()) { - eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + eventProto = RecommendationMessage.RecommendedEventProto.newBuilder() .setEventId(eventSimilarity.getEventA()) .setScore(eventSimilarity.getScore()) .build(); } else { - eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + eventProto = RecommendationMessage.RecommendedEventProto.newBuilder() .setEventId(eventSimilarity.getEventB()) .setScore(eventSimilarity.getScore()) .build(); @@ -133,15 +132,15 @@ public List getSimilarEvents(Reco recommendedEventList.add(eventProto); } - return recommendedEventList.stream().sorted(Comparator.comparingDouble(RecommendationsMessages.RecommendedEventProto::getScore) + return recommendedEventList.stream().sorted(Comparator.comparingDouble(RecommendationMessage.RecommendedEventProto::getScore) .reversed()).limit(eventsRequestProto.getMaxResults()).toList(); } - public List getInteractionsCount(RecommendationsMessages.InteractionsCountRequestProto request) { + public List getInteractionsCount(RecommendationMessage.InteractionsCountRequestProto request) { log.info("Method getInteractionsCount began its work"); List userActionList = userActionRepository.findByEventIdIsIn(request.getEventIdList()); Map recommendedEventMap = new HashMap<>(); - List recommendedEventList = new ArrayList<>(); + List recommendedEventList = new ArrayList<>(); for (UserAction userAction : userActionList) { if (!recommendedEventMap.containsKey(userAction.getEventId())) { @@ -154,7 +153,7 @@ public List getInteractionsCount( } } for (long eventId : recommendedEventMap.keySet()) { - RecommendationsMessages.RecommendedEventProto eventProto = RecommendationsMessages.RecommendedEventProto.newBuilder() + RecommendationMessage.RecommendedEventProto eventProto = RecommendationMessage.RecommendedEventProto.newBuilder() .setEventId(eventId) .setScore(recommendedEventMap.get(eventId)) .build(); diff --git a/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java b/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java index a46e7a1..9e2cf78 100644 --- a/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java +++ b/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java @@ -1,7 +1,8 @@ package ru.practicum.handler; -import ru.practicum.ewm.stats.proto.UserActionProto; + +import ru.practicum.grpc.stats.action.UserActionMessage; public interface ActionsHandlers { - void handle(UserActionProto userActionProto); + void handle(UserActionMessage.UserActionRequest userActionProto); } \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java b/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java index 91e4048..1fb2d31 100644 --- a/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java +++ b/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java @@ -4,8 +4,7 @@ import org.springframework.stereotype.Component; import ru.practicum.ewm.stats.avro.ActionTypeAvro; import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.ewm.stats.proto.ActionTypeProto; -import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.grpc.stats.action.UserActionMessage; import ru.practicum.service.KafkaMessageProducer; import java.time.Instant; @@ -20,7 +19,7 @@ public UserActionHandler(KafkaMessageProducer kafkaMessageProducer) { } @Override - public void handle(UserActionProto userActionProto) { + public void handle(UserActionMessage.UserActionRequest userActionProto) { log.info("Обработчик UserActionHandler начал работать"); log.info("На вход:{}", userActionProto.toString()); @@ -46,14 +45,14 @@ public void handle(UserActionProto userActionProto) { } - private ActionTypeAvro getActionType(ActionTypeProto actionTypeProto) { - if (actionTypeProto.equals(ActionTypeProto.ACTION_LIKE)) { + private ActionTypeAvro getActionType(UserActionMessage.ActionTypeProto actionTypeProto) { + if (actionTypeProto.equals(UserActionMessage.ActionTypeProto.ACTION_LIKE)) { return ActionTypeAvro.LIKE; } - if (actionTypeProto.equals(ActionTypeProto.ACTION_REGISTER)) { + if (actionTypeProto.equals(UserActionMessage.ActionTypeProto.ACTION_REGISTER)) { return ActionTypeAvro.REGISTER; } - if (actionTypeProto.equals(ActionTypeProto.ACTION_VIEW)) { + if (actionTypeProto.equals(UserActionMessage.ActionTypeProto.ACTION_VIEW)) { return ActionTypeAvro.VIEW; } return null; diff --git a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java index 3534ee1..c80a12e 100644 --- a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java +++ b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java @@ -1,15 +1,14 @@ package ru.practicum.mapper; import ru.practicum.ewm.stats.avro.ActionTypeAvro; -import ru.practicum.ewm.stats.proto.ActionTypeProto; -import ru.practicum.ewm.stats.proto.UserActionProto; import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.grpc.stats.action.UserActionMessage; import java.time.Instant; public class UserActionMapper { - public static UserActionAvro toAvro(UserActionProto userActionProto) { + public static UserActionAvro toAvro(UserActionMessage.UserActionRequest userActionProto) { long timestampMillis = userActionProto.getTimestamp().getSeconds() * 1000 + userActionProto.getTimestamp().getNanos() / 1_000_000; @@ -21,7 +20,7 @@ public static UserActionAvro toAvro(UserActionProto userActionProto) { .build(); } - private static ActionTypeAvro toAvroActionType(ActionTypeProto protoType) { + private static ActionTypeAvro toAvroActionType(UserActionMessage.ActionTypeProto protoType) { return switch (protoType) { case ACTION_REGISTER -> ActionTypeAvro.REGISTER; case ACTION_LIKE -> ActionTypeAvro.LIKE; diff --git a/stats/collector/src/main/java/ru/practicum/service/UserActionController.java b/stats/collector/src/main/java/ru/practicum/service/CollectorController.java similarity index 69% rename from stats/collector/src/main/java/ru/practicum/service/UserActionController.java rename to stats/collector/src/main/java/ru/practicum/service/CollectorController.java index f7569c6..a84b8f0 100644 --- a/stats/collector/src/main/java/ru/practicum/service/UserActionController.java +++ b/stats/collector/src/main/java/ru/practicum/service/CollectorController.java @@ -7,23 +7,21 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; -import ru.practicum.ewm.stats.proto.UserActionControllerGrpc; -import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.grpc.stats.action.UserActionControllerGrpc; +import ru.practicum.grpc.stats.action.UserActionMessage; import ru.practicum.handler.ActionsHandlers; -import ru.practicum.mapper.UserActionMapper; -import ru.practicum.ewm.stats.avro.UserActionAvro; @Slf4j @GrpcService @RequiredArgsConstructor -public class UserActionController extends UserActionControllerGrpc.UserActionControllerImplBase { +public class CollectorController extends UserActionControllerGrpc.UserActionControllerImplBase { private final ActionsHandlers actionHandler; @Override - public void collectUserAction(UserActionProto request, StreamObserver responseObserver) { + public void collectUserAction(UserActionMessage.UserActionRequest request, StreamObserver responseObserver) { try { actionHandler.handle(request); - responseObserver.onNext(Empty.newBuilder().build()); + responseObserver.onNext(UserActionMessage.UserActionResponse.newBuilder().getDefaultInstanceForType()); responseObserver.onCompleted(); } catch (IllegalArgumentException e) { log.error("IllegalArgumentException collectUserAction: {}", e.getMessage(), e); @@ -37,4 +35,6 @@ public void collectUserAction(UserActionProto request, StreamObserver res ); } } + + } \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto b/stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto deleted file mode 100644 index c2895c8..0000000 --- a/stats/serialization/proto-schemas/src/main/protobuf/controller/recommendations_controller.proto +++ /dev/null @@ -1,17 +0,0 @@ -syntax = "proto3"; - -package ru.practicum.ewm.stats.proto; - -import "messages/recommendations_messages.proto"; -import "google/protobuf/empty.proto"; - -service RecommendationsController { - rpc GetRecommendationsForUser(UserPredictionsRequestProto) - returns (stream RecommendedEventProto); - - rpc GetSimilarEvents(SimilarEventsRequestProto) - returns (stream RecommendedEventProto); - - rpc GetInteractionsCount(InteractionsCountRequestProto) - returns (stream RecommendedEventProto); -} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto b/stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto deleted file mode 100644 index 0108be8..0000000 --- a/stats/serialization/proto-schemas/src/main/protobuf/controller/user_action_controller.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -package ru.practicum.ewm.stats.proto; - -import "google/protobuf/empty.proto"; -import "messages/user_action.proto"; - -service UserActionController { - rpc CollectUserAction(UserActionProto) returns (google.protobuf.Empty); -} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto similarity index 80% rename from stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto rename to stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto index f4c7216..b5c0e1a 100644 --- a/stats/serialization/proto-schemas/src/main/protobuf/messages/recommendations_messages.proto +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto @@ -1,8 +1,7 @@ syntax = "proto3"; +package stats.message; -package ru.practicum.ewm.stats.proto; - -import "google/protobuf/timestamp.proto"; +option java_package = "ru.practicum.grpc.stats.recommendation"; message UserPredictionsRequestProto { int64 user_id = 1; diff --git a/stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/message/user_action_message.proto similarity index 61% rename from stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto rename to stats/serialization/proto-schemas/src/main/protobuf/stats/message/user_action_message.proto index 2f91b96..f35e7ea 100644 --- a/stats/serialization/proto-schemas/src/main/protobuf/messages/user_action.proto +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/message/user_action_message.proto @@ -1,21 +1,23 @@ syntax = "proto3"; +package stats.service.collector; -package ru.practicum.ewm.stats.proto; +option java_package = "ru.practicum.grpc.stats.action"; import "google/protobuf/timestamp.proto"; -option java_multiple_files = true; -option java_package = "ru.practicum.ewm.stats.proto"; - enum ActionTypeProto { ACTION_VIEW = 0; ACTION_REGISTER = 1; ACTION_LIKE = 2; } -message UserActionProto { +message UserActionRequest { int64 user_id = 1; int64 event_id = 2; ActionTypeProto action_type = 3; google.protobuf.Timestamp timestamp = 4; +} + +message UserActionResponse { + bool success = 1; } \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto new file mode 100644 index 0000000..f41b177 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package stats.service.dashboard; + +option java_package = "ru.practicum.grpc.stats.recommendation"; + +import "stats/message/recommendation_message.proto"; + +service RecommendationsController { + rpc GetRecommendationsForUser(message.UserPredictionsRequestProto) returns (stream message.RecommendedEventProto); + rpc GetSimilarEvents(message.SimilarEventsRequestProto) returns (stream message.RecommendedEventProto); + rpc GetInteractionsCount(message.InteractionsCountRequestProto) returns (stream stats.message.RecommendedEventProto); +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto new file mode 100644 index 0000000..22dd3a9 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package stats.service.collector; + +option java_package = "ru.practicum.grpc.stats.action"; + +import "stats/message/user_action_message.proto"; +import "google/protobuf/empty.proto"; + +service UserActionController { + rpc CollectUserAction(UserActionRequest) returns (UserActionResponse); +} \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java b/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java index 0916c5d..a21a23d 100644 --- a/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java +++ b/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java @@ -3,8 +3,8 @@ import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.client.inject.GrpcClient; import org.springframework.stereotype.Service; -import ru.practicum.ewm.stats.proto.RecommendationsControllerGrpc; -import ru.practicum.ewm.stats.proto.RecommendationsMessages; +import ru.practicum.grpc.stats.recommendation.RecommendationMessage; +import ru.practicum.grpc.stats.recommendation.RecommendationsControllerGrpc; import java.util.Iterator; import java.util.Spliterator; @@ -19,17 +19,17 @@ public class AnalyzerClient { @GrpcClient("analyzer") private RecommendationsControllerGrpc.RecommendationsControllerBlockingStub analyzerStub; - public Stream getSimilarEvents( + public Stream getSimilarEvents( long eventId, long userId, int maxResults) { try { log.info("Fetching similar events: eventId={}, userId={}, maxResults={}", eventId, userId, maxResults); - RecommendationsMessages.SimilarEventsRequestProto requestProto = - RecommendationsMessages.SimilarEventsRequestProto.newBuilder() + RecommendationMessage.SimilarEventsRequestProto requestProto = + RecommendationMessage.SimilarEventsRequestProto.newBuilder() .setEventId(eventId) .setUserId(userId) .setMaxResults(maxResults) .build(); - Iterator iterator = analyzerStub.getSimilarEvents(requestProto); + Iterator iterator = analyzerStub.getSimilarEvents(requestProto); return toStream(iterator); } catch (Exception e) { log.error("Error occurred while fetching similar events: eventId={}, userId={}, maxResults={}", @@ -38,15 +38,15 @@ public Stream getSimilarEvents( } } - public Stream getRecommendationsForUser(long userId, int maxResults) { + public Stream getRecommendationsForUser(long userId, int maxResults) { try { log.info("Fetching recommendations for user : userId={}, maxResults={}", userId, maxResults); - RecommendationsMessages.UserPredictionsRequestProto requestProto = - RecommendationsMessages.UserPredictionsRequestProto.newBuilder() + RecommendationMessage.UserPredictionsRequestProto requestProto = + RecommendationMessage.UserPredictionsRequestProto.newBuilder() .setUserId(userId) .setMaxResults(maxResults) .build(); - Iterator iterator = analyzerStub.getRecommendationsForUser(requestProto); + Iterator iterator = analyzerStub.getRecommendationsForUser(requestProto); return toStream(iterator); } catch (Exception e) { log.error("Error occurred while fetching recommendations for user : userId={}, maxResults={}", userId, maxResults); @@ -54,14 +54,14 @@ public Stream getRecommendationsF } } - public Stream getInteractionsCount(Iterable eventIds) { + public Stream getInteractionsCount(Iterable eventIds) { try { log.info("Fetching interactions count for events"); - RecommendationsMessages.InteractionsCountRequestProto.Builder builder = - RecommendationsMessages.InteractionsCountRequestProto.newBuilder(); + RecommendationMessage.InteractionsCountRequestProto.Builder builder = + RecommendationMessage.InteractionsCountRequestProto.newBuilder(); eventIds.forEach(builder::addEventId); - RecommendationsMessages.InteractionsCountRequestProto requestProto = builder.build(); - Iterator iterator = analyzerStub.getInteractionsCount(requestProto); + RecommendationMessage.InteractionsCountRequestProto requestProto = builder.build(); + Iterator iterator = analyzerStub.getInteractionsCount(requestProto); return toStream(iterator); } catch (Exception e) { log.error("Error occurred while fetching interactions count", e); @@ -69,7 +69,7 @@ public Stream getInteractionsCoun } } - private Stream toStream(Iterator iterator) { + private Stream toStream(Iterator iterator) { return StreamSupport.stream( Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false diff --git a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java index 166f30c..4c66fed 100644 --- a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java +++ b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java @@ -1,12 +1,10 @@ package ru.practicum; -import com.google.protobuf.Empty; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.client.inject.GrpcClient; import org.springframework.stereotype.Service; -import ru.practicum.ewm.stats.proto.ActionTypeProto; -import ru.practicum.ewm.stats.proto.UserActionControllerGrpc; -import ru.practicum.ewm.stats.proto.UserActionProto; +import ru.practicum.grpc.stats.action.UserActionControllerGrpc; +import ru.practicum.grpc.stats.action.UserActionMessage; import java.time.Instant; @@ -17,13 +15,13 @@ public class CollectorClient { @GrpcClient("collector") private UserActionControllerGrpc.UserActionControllerBlockingStub collectorStub; - public void sendUserAction(long userId, long eventId, ActionTypeProto actionTypeProto) { + public void sendUserAction(long userId, long eventId, UserActionMessage.ActionTypeProto actionTypeProto) { try { log.info("Sending user action: userId={}, eventId={}, actionType={}", userId, eventId, actionTypeProto); long secondes = Instant.now().getEpochSecond(); int nanos = Instant.now().getNano(); - UserActionProto userActionProto = UserActionProto.newBuilder() + UserActionMessage.UserActionRequest userActionProto = UserActionMessage.UserActionRequest.newBuilder() .setUserId(userId) .setEventId(eventId) .setActionType(actionTypeProto) @@ -33,7 +31,7 @@ public void sendUserAction(long userId, long eventId, ActionTypeProto actionType .setNanos(nanos) ) .build(); - Empty response = collectorStub.collectUserAction(userActionProto); + UserActionMessage.UserActionResponse response = collectorStub.collectUserAction(userActionProto); log.info("sendUserAction -> Collector answered"); } catch (Exception e) { log.error("Ошибка при отправке действия пользователя: userId={}, eventId={}, actionType={}", @@ -42,14 +40,14 @@ public void sendUserAction(long userId, long eventId, ActionTypeProto actionType } public void sendEventView(long userId, long eventId) { - sendUserAction(userId, eventId, ActionTypeProto.ACTION_VIEW); + sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_VIEW); } public void sendEventLike(long userId, long eventId) { - sendUserAction(userId, eventId, ActionTypeProto.ACTION_LIKE); + sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_LIKE); } public void sendEventRegistration(long userId, long eventId) { - sendUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER); + sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_REGISTER); } } From 63537a73f654b2467b705ea6bf3cf65b0f43e32e Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Fri, 9 May 2025 18:49:40 +0300 Subject: [PATCH 12/26] feat: refactored stats and released recommendations feature --- .../ru/practicum/model/EventSimilarity.java | 7 +++++-- .../java/ru/practicum/model/UserAction.java | 15 ++++++--------- .../analyzer/src/main/resources/application.yml | 6 ++++++ stats/analyzer/src/main/resources/schema.sql | 17 +++++++++++++++++ .../practicum/service/CollectorController.java | 1 - 5 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 stats/analyzer/src/main/resources/schema.sql diff --git a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java index b9dfdab..82de489 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java @@ -12,15 +12,18 @@ @NoArgsConstructor @Builder @Entity -@Table(name = "events_similarity") +@Table(name = "similarities") @FieldDefaults(level = AccessLevel.PRIVATE) public class EventSimilarity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) Long id; - + @Column(name = "event_id_a") Long eventA; + @Column(name = "event_id_b") Long eventB; + @Column(name = "score") Double score; + @Column(name = "timestamp") Instant timestamp; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java index 686c4dc..c4de9e7 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java @@ -1,11 +1,7 @@ package ru.practicum.model; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Table; +import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -20,16 +16,17 @@ @NoArgsConstructor @Builder @Entity -@Table(name = "user_actions") +@Table(name = "actions") public class UserAction { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - + @Column(name = "user_id") private Long userId; + @Column(name = "event_id") private Long eventId; - + @Column(name = "score") private Double score; - + @Column(name = "lastInteraction") private Instant lastInteraction; } \ No newline at end of file diff --git a/stats/analyzer/src/main/resources/application.yml b/stats/analyzer/src/main/resources/application.yml index ecaf492..81a4d6e 100644 --- a/stats/analyzer/src/main/resources/application.yml +++ b/stats/analyzer/src/main/resources/application.yml @@ -13,6 +13,12 @@ spring: retry: useRandomPolicy: true max-interval: 6000 + flyway: + flyway: + database: postgresql + ignore-future-migrations: true + postgresql: + transactional-lock: false eureka: client: serviceUrl: diff --git a/stats/analyzer/src/main/resources/schema.sql b/stats/analyzer/src/main/resources/schema.sql new file mode 100644 index 0000000..3129fa5 --- /dev/null +++ b/stats/analyzer/src/main/resources/schema.sql @@ -0,0 +1,17 @@ +CREATE TABLE similarities ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + event_id_a BIGINT, + event_id_b BIGINT, + score DOUBLE PRECISION, + timestamp TIMESTAMP with time zone, + CONSTRAINT pk_similarities PRIMARY KEY (id) +); + +CREATE TABLE actions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + user_id BIGINT, + event_id BIGINT, + score DOUBLE PRECISION, + last_interaction TIMESTAMP with time zone, + CONSTRAINT pk_actions PRIMARY KEY (id) +); \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/CollectorController.java b/stats/collector/src/main/java/ru/practicum/service/CollectorController.java index a84b8f0..b947f9b 100644 --- a/stats/collector/src/main/java/ru/practicum/service/CollectorController.java +++ b/stats/collector/src/main/java/ru/practicum/service/CollectorController.java @@ -1,6 +1,5 @@ package ru.practicum.service; -import com.google.protobuf.Empty; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; From 63d9a341b2befde1fb30b3a53f6d050cd5eb72e7 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 02:45:56 +0300 Subject: [PATCH 13/26] feat: refactored stats and released recommendations feature --- .../controller/PublicEventController.java | 29 +-- .../practicum/mapper/event/EventMapper.java | 4 + .../practicum/service/event/EventService.java | 10 +- .../service/event/EventServiceImpl.java | 55 +++-- .../ru/practicum/dto/event/EventShortDto.java | 2 +- .../dto/event/RecommendedEventDto.java | 15 ++ .../practicum/service/RequestServiceImpl.java | 1 - .../src/main/resources/application.yml | 11 +- .../config/stats/aggregator/application.yml | 38 ++-- .../config/stats/analyzer/application.yml | 103 +++++---- .../config/stats/collector/application.yml | 30 ++- stats/aggregator/pom.xml | 27 +-- .../ru/practicum/AggregatorApplication.java | 9 +- .../java/ru/practicum/config/AppConfig.java | 102 ++++----- .../java/ru/practicum/config/KafkaConfig.java | 60 ++--- .../config/KafkaConfigProperties.java | 26 +++ .../config/SimilarityEventProducer.java | 11 - .../SimilarityEventProducerConfiguration.java | 94 ++++---- .../practicum/service/AggregationStarter.java | 76 +++---- .../practicum/service/AggregatorService.java | 9 +- .../service/AggregatorServiceImpl.java | 211 +++++++++++------- .../src/main/resources/application.yml | 16 +- stats/analyzer/pom.xml | 69 +++--- .../ru/practicum/AnalyzerApplication.java | 22 +- ...rSettings.java => ConsumerProperties.java} | 16 +- .../java/ru/practicum/config/KafkaConfig.java | 54 +++++ .../config/KafkaConfigProperties.java | 17 ++ .../java/ru/practicum/config/KafkaTopics.java | 16 -- .../controller/RecommendationController.java | 72 +++--- .../kafka/ConfigKafkaProperties.java | 61 ----- .../mapper/EventSimilarityMapper.java | 10 - .../main/java/ru/practicum/mapper/Mapper.java | 43 ++++ .../java/ru/practicum/model/ActionType.java | 16 ++ .../ru/practicum/model/EventSimilarity.java | 45 ++-- .../ru/practicum/model/RecommendedEvent.java | 15 +- .../java/ru/practicum/model/UserAction.java | 58 +++-- .../processor/EventSimilarityProcessor.java | 70 +++--- .../processor/UserActionEventProcessor.java | 68 +++--- .../repository/EventSimilarityRepository.java | 21 +- .../repository/UserActionRepository.java | 16 +- .../service/RecommendationService.java | 168 +------------- .../service/RecommendationServiceImpl.java | 163 ++++++++++++++ .../service/event/EventSimilarityService.java | 16 +- .../event/EventSimilarityServiceImpl.java | 78 +++---- .../service/user/UserActionService.java | 14 +- .../service/user/UserActionServiceImpl.java | 112 +++++----- .../src/main/resources/application.yml | 22 +- stats/analyzer/src/main/resources/schema.sql | 30 +-- stats/collector/pom.xml | 64 +++--- .../ru/practicum/CollectorApplication.java | 2 + .../java/ru/practicum/config/KafkaConfig.java | 33 +++ .../config/KafkaConfigProperties.java | 16 ++ .../practicum/config/KafkaProducerConfig.java | 38 ---- .../ru/practicum/config/KafkaProperties.java | 29 --- .../practicum/config/UserActionProducer.java | 11 - .../UserActionProducerConfiguration.java | 70 ------ .../ru/practicum/handler/ActionsHandlers.java | 8 - .../practicum/handler/UserActionHandler.java | 60 ----- .../ru/practicum/mapper/UserActionMapper.java | 47 ++-- .../java/ru/practicum/model/ActionType.java | 7 + .../java/ru/practicum/model/UserAction.java | 22 ++ .../ru/practicum/service/ActionService.java | 9 + .../practicum/service/ActionServiceImpl.java | 62 +++++ .../service/CollectorController.java | 37 +-- .../service/KafkaMessageProducer.java | 32 --- .../ru/practicum/service/MessageProducer.java | 7 - .../src/main/resources/application.yml | 18 +- .../stats/avro}/EventSimilarityProtocol.avdl | 0 .../ewm/stats/avro/UserActionProtocol.avdl} | 0 .../message/recommendation_message.proto | 24 -- .../messages/recommendation_request.proto | 26 +++ .../user_action.proto} | 19 +- .../service/recommendations_controller.proto | 18 ++ .../service/recommendations_service.proto | 12 - .../service/user_action_controller.proto | 12 + .../stats/service/user_action_service.proto | 11 - stats/stats-client/pom.xml | 38 ++-- .../java/ru/practicum/AnalyzerClient.java | 78 ------- .../java/ru/practicum/CollectorClient.java | 53 ----- .../main/java/ru/practicum/StatClient.java | 24 -- .../java/ru/practicum/StatServiceClient.java | 34 --- .../ru/practicum/stats/client/StatClient.java | 19 ++ .../stats/client/StatClientImpl.java | 102 +++++++++ .../src/main/resources/application.yml | 13 -- 84 files changed, 1676 insertions(+), 1610 deletions(-) create mode 100644 core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java create mode 100644 stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java delete mode 100644 stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java rename stats/analyzer/src/main/java/ru/practicum/config/{KafkaConsumerSettings.java => ConsumerProperties.java} (50%) create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/mapper/Mapper.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/model/ActionType.java create mode 100644 stats/analyzer/src/main/java/ru/practicum/service/RecommendationServiceImpl.java create mode 100644 stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java create mode 100644 stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java delete mode 100644 stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java delete mode 100644 stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java delete mode 100644 stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java delete mode 100644 stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java delete mode 100644 stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java delete mode 100644 stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java create mode 100644 stats/collector/src/main/java/ru/practicum/model/ActionType.java create mode 100644 stats/collector/src/main/java/ru/practicum/model/UserAction.java create mode 100644 stats/collector/src/main/java/ru/practicum/service/ActionService.java create mode 100644 stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java delete mode 100644 stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java delete mode 100644 stats/collector/src/main/java/ru/practicum/service/MessageProducer.java rename stats/serialization/avro-schemas/src/main/avro/{ => ru/practicum/ewm/stats/avro}/EventSimilarityProtocol.avdl (100%) rename stats/serialization/avro-schemas/src/main/avro/{UserActionAvro.avdl => ru/practicum/ewm/stats/avro/UserActionProtocol.avdl} (100%) delete mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/messages/recommendation_request.proto rename stats/serialization/proto-schemas/src/main/protobuf/stats/{message/user_action_message.proto => messages/user_action.proto} (56%) create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_controller.proto delete mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto create mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_controller.proto delete mode 100644 stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto delete mode 100644 stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java delete mode 100644 stats/stats-client/src/main/java/ru/practicum/CollectorClient.java delete mode 100644 stats/stats-client/src/main/java/ru/practicum/StatClient.java delete mode 100644 stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java create mode 100644 stats/stats-client/src/main/java/ru/practicum/stats/client/StatClient.java create mode 100644 stats/stats-client/src/main/java/ru/practicum/stats/client/StatClientImpl.java delete mode 100644 stats/stats-client/src/main/resources/application.yml diff --git a/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java b/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java index 3076047..1f3ae0d 100644 --- a/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java +++ b/core/event-service/src/main/java/ru/practicum/controller/PublicEventController.java @@ -5,21 +5,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.web.bind.annotation.*; -import ru.practicum.AnalyzerClient; -import ru.practicum.CollectorClient; import ru.practicum.dto.event.EventFullDto; -import ru.practicum.dto.event.EventRecommendationDto; import ru.practicum.dto.event.EventShortDto; +import ru.practicum.dto.event.RecommendedEventDto; import ru.practicum.exception.IncorrectValueException; -import ru.practicum.grpc.stats.action.UserActionMessage; -import ru.practicum.grpc.stats.recommendation.RecommendationMessage; import ru.practicum.service.event.EventSearchParams; import ru.practicum.service.event.EventService; import ru.practicum.service.event.PublicSearchParams; +import ru.practicum.stats.client.StatClient; import java.time.LocalDateTime; -import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; import static ru.practicum.constant.Constant.PATTERN_DATE; @@ -32,10 +29,7 @@ @Slf4j public class PublicEventController { - private final CollectorClient collectorClient; - - private final AnalyzerClient analyzerClient; - + private final StatClient statClient; private final EventService eventService; private static final String X_EWM_USER_ID_HEADER = "X-EWM-USER-ID"; @@ -104,23 +98,14 @@ public EventFullDto getEventById(@PathVariable("event-id") Long eventId, @Reques } @GetMapping("/recommendations") - public List getRecommendations(@RequestHeader(X_EWM_USER_ID_HEADER) long userId, - @RequestParam(defaultValue = "10") int maxResults) { - var recommendationStream = analyzerClient.getRecommendationsForUser(userId, maxResults); - var recommendationList = recommendationStream.toList(); - - List result = new ArrayList<>(); - for (RecommendationMessage.RecommendedEventProto requestProto : recommendationList) { - result.add(new EventRecommendationDto(requestProto.getEventId(), requestProto.getScore())); - } - return result; + public Stream getRecommendations(@RequestHeader(X_EWM_USER_ID_HEADER) long userId, + @RequestParam(defaultValue = "10") int maxResults) { + return eventService.getRecommendations(userId, maxResults); } @PutMapping("/{event-id}/like") public void likeEvent(@PathVariable("event-id") Long eventId, @RequestHeader(X_EWM_USER_ID_HEADER) long userId) { eventService.addLike(userId, eventId); - - collectorClient.sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_LIKE); } } diff --git a/core/event-service/src/main/java/ru/practicum/mapper/event/EventMapper.java b/core/event-service/src/main/java/ru/practicum/mapper/event/EventMapper.java index a8f096e..f59ed4a 100644 --- a/core/event-service/src/main/java/ru/practicum/mapper/event/EventMapper.java +++ b/core/event-service/src/main/java/ru/practicum/mapper/event/EventMapper.java @@ -3,6 +3,8 @@ import org.mapstruct.Mapper; import ru.practicum.dto.event.EventFullDto; import ru.practicum.dto.event.EventShortDto; +import ru.practicum.dto.event.RecommendedEventDto; +import ru.practicum.grpc.stat.request.RecommendedEventProto; import ru.practicum.mapper.location.LocationMapper; import ru.practicum.model.Event; @@ -45,4 +47,6 @@ default String formatDateTime(LocalDateTime dateTime) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); return dateTime.format(formatter); } + + RecommendedEventDto map(RecommendedEventProto proto); } diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventService.java b/core/event-service/src/main/java/ru/practicum/service/event/EventService.java index d03b017..80cd92c 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventService.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventService.java @@ -1,14 +1,12 @@ package ru.practicum.service.event; import org.springframework.transaction.annotation.Transactional; -import ru.practicum.dto.event.EventFullDto; -import ru.practicum.dto.event.EventShortDto; -import ru.practicum.dto.event.NewEventDto; -import ru.practicum.dto.event.UpdateEventAdminRequest; +import ru.practicum.dto.event.*; import ru.practicum.enums.EventState; import java.time.LocalDateTime; import java.util.List; +import java.util.stream.Stream; /** * The interface Event service. @@ -96,9 +94,11 @@ List getEvents(String text, List categories, Boolean paid, @Transactional(readOnly = true) List getAllByPublic(EventSearchParams searchParams, Boolean onlyAvailable, String sort, String clientIp); - EventShortDto addLike(long userId, long eventId); + void addLike(long userId, long eventId); void deleteLike(long userId, long eventId); + + Stream getRecommendations(Long userId, int limit); } diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index 253e3f4..753da6d 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -9,19 +9,19 @@ import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import ru.practicum.AnalyzerClient; -import ru.practicum.CollectorClient; import ru.practicum.client.RequestServiceClient; import ru.practicum.client.UserServiceClient; import ru.practicum.dto.category.CategoryDto; import ru.practicum.dto.event.*; +import ru.practicum.dto.user.UserDto; +import ru.practicum.dto.user.UserShortDto; import ru.practicum.enums.AdminStateAction; import ru.practicum.enums.EventState; import ru.practicum.enums.RequestStatus; import ru.practicum.exception.ConflictException; import ru.practicum.exception.NotFoundException; import ru.practicum.exception.ValidationException; -import ru.practicum.grpc.stats.action.UserActionMessage; +import ru.practicum.grpc.stat.action.ActionTypeProto; import ru.practicum.mapper.event.EventMapper; import ru.practicum.mapper.event.UtilEventClass; import ru.practicum.mapper.location.LocationMapper; @@ -29,11 +29,14 @@ import ru.practicum.model.Location; import ru.practicum.repository.*; import ru.practicum.service.category.CategoryService; +import ru.practicum.stats.client.StatClient; +import java.time.Instant; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import static ru.practicum.constant.Constant.PATTERN_DATE; import static ru.practicum.model.QEvent.event; @@ -46,8 +49,6 @@ @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class EventServiceImpl implements EventService { - private final CollectorClient collectorClient; - private final UserServiceClient userServiceClient; private final LocationMapper locationMapper; @@ -61,7 +62,7 @@ public class EventServiceImpl implements EventService { LocationRepository locationRepository; SearchEventRepository searchEventRepository; CategoryRepository categoryRepository; - AnalyzerClient analyzerClient; + StatClient statClient; @Autowired public EventServiceImpl(EventRepository eventRepository, @@ -70,8 +71,7 @@ public EventServiceImpl(EventRepository eventRepository, LocationRepository locationRepository, SearchEventRepository searchEventRepository, CategoryRepository categoryRepository, LocationMapper locationMapper, - UserServiceClient userServiceClient, AnalyzerClient analyzerClient, - CollectorClient collectorClient) { + UserServiceClient userServiceClient, StatClient statClient) { this.eventRepository = eventRepository; this.requestServiceClient = requestServiceClient; this.eventMapper = eventMapper; @@ -80,17 +80,19 @@ public EventServiceImpl(EventRepository eventRepository, this.locationRepository = locationRepository; this.searchEventRepository = searchEventRepository; this.categoryRepository = categoryRepository; - this.analyzerClient = analyzerClient; + this.statClient = statClient; this.locationMapper = locationMapper; this.userServiceClient = userServiceClient; - this.collectorClient = collectorClient; } @Override @Transactional(readOnly = true) public List getEventsForUser(Long userId, Integer from, Integer size) { List events = eventRepository.findByInitiatorId(userId, PageRequest.of(from, size)); - + List eventsDto = events.stream() + .map(eventMapper::toEventShortDto) + .toList(); + populateWithStats(eventsDto); return events.stream() .map(eventMapper::toEventShortDto) .collect(Collectors.toList()); @@ -205,6 +207,8 @@ public EventFullDto updateEventByAdmin(UpdateEventAdminRequest request, Long eve } } + EventShortDto eventShortDto = eventMapper.toEventShortDto(event); + populateWithStats(List.of(eventShortDto)); return utilEventClass.toEventFullDto(eventRepository.save(event)); } @@ -292,14 +296,16 @@ public EventFullDto getEventById(Long eventId, long userId) { eventRepository.save(event); - collectorClient.sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_VIEW); - + log.info("starting statClient.registerUserAction"); + statClient.registerUserAction(event.getId(), userId, ActionTypeProto.ACTION_VIEW, Instant.now()); // Подсчет подтвержденных запросов long confirmedRequests = requestServiceClient.countByStatusAndEventId(RequestStatus.CONFIRMED, eventId); // Создание DTO EventFullDto eventFullDto = utilEventClass.toEventFullDto(event); eventFullDto.setConfirmedRequests(confirmedRequests); + EventShortDto eventShortDto = eventMapper.toEventShortDto(event); + populateWithStats(List.of(eventShortDto)); return eventFullDto; @@ -503,7 +509,7 @@ public List getAllByPublic(EventSearchParams searchParams, Boolea } @Override - public EventShortDto addLike(long userId, long eventId) { + public void addLike(long userId, long eventId) { Event event = eventRepository.findById(eventId).orElseThrow( () -> new NotFoundException("Event with id = " + eventId, " not found") ); @@ -512,7 +518,7 @@ public EventShortDto addLike(long userId, long eventId) { } eventRepository.addLike(userId, eventId); event.setLikes(eventRepository.countLikesByEventId(eventId)); - return eventMapper.toEventShortDto(event); + statClient.registerUserAction(eventId, userId, ActionTypeProto.ACTION_LIKE, Instant.now()); } @Override @@ -528,4 +534,23 @@ public void deleteLike(long userId, long eventId) { + " by user: " + userId + " not exists"); } } + + @Override + public Stream getRecommendations(Long userId, int limit) { + return statClient.getRecommendationsFor(userId, limit) + .map(eventMapper::map); + } + + private void populateWithStats(List eventsDto) { + if (eventsDto.isEmpty()) return; + + List eventIds = eventsDto.stream() + .map(EventShortDto::getId).toList(); + Map ratedEvents = statClient.getInteractionsCount(eventIds) + .map(eventMapper::map) + .collect(Collectors.toMap(RecommendedEventDto::getEventId, RecommendedEventDto::getScore)); + log.info("ratedEvents are: {}", ratedEvents); + eventsDto.forEach(event -> Optional.ofNullable(ratedEvents.get(event.getId())) + .ifPresent(event::setRating)); + } } diff --git a/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java index aa7eb2f..2ef8628 100644 --- a/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java +++ b/core/interaction-api/src/main/java/ru/practicum/dto/event/EventShortDto.java @@ -24,7 +24,7 @@ public class EventShortDto { UserShortDto initiator; Boolean paid; String title; - Long rating; + double rating; diff --git a/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java b/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java new file mode 100644 index 0000000..ec9868f --- /dev/null +++ b/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java @@ -0,0 +1,15 @@ +package ru.practicum.dto.event; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor +public class RecommendedEventDto { + private Long eventId; + private double score; +} \ No newline at end of file diff --git a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java index 995df48..51f1b65 100644 --- a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java +++ b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java @@ -4,7 +4,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import ru.practicum.CollectorClient; import ru.practicum.client.EventServiceClient; import ru.practicum.client.UserServiceClient; import ru.practicum.dto.request.EventRequestStatusUpdateRequest; diff --git a/infra/config-server/src/main/resources/application.yml b/infra/config-server/src/main/resources/application.yml index bf89bc7..e9c702d 100644 --- a/infra/config-server/src/main/resources/application.yml +++ b/infra/config-server/src/main/resources/application.yml @@ -10,17 +10,18 @@ spring: searchLocations: - classpath:config/core/{application} - classpath:config/infra/{application} - - classpath:config/stats/{application} + - classpath:config/stat/{application} + discovery: + enabled: true + eureka: client: - register-with-eureka: true - fetch-registry: true serviceUrl: defaultZone: http://localhost:8761/eureka/ instance: - prefer-ip-address: true + preferIpAddress: true hostname: localhost instance-id: "${spring.application.name}:${random.value}" - lease-renewal-interval-in-seconds: 10 + leaseRenewalIntervalInSeconds: 10 server: port: 0 \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml index 52a5a9d..cd8fbb0 100644 --- a/infra/config-server/src/main/resources/config/stats/aggregator/application.yml +++ b/infra/config-server/src/main/resources/config/stats/aggregator/application.yml @@ -1,22 +1,20 @@ server: - port: 0 + port: 8889 -spring: - kafka: - producer: - bootstrap-servers: localhost:9092 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.serializer.AvroSerializer - consumer: - bootstrap-servers: localhost:9092 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.deserializer.UserActionDeserializer - client-id: action-consumer - group-id: group-practicum - max-poll-records: 100 - fetch-max-bytes: 3072000 - max-partition-fetch-bytes: 307200 - consume-attempts-timeout-ms: 1000 - topics: - action-topic: stats.user-actions.v1 - similarity-topic: stats.events-similarity.v1 \ No newline at end of file +kafka: + bootstrapServers: localhost:9092 + producerClientIdConfig: aggregator-producer + producerKeySerializer: org.apache.kafka.common.serialization.LongSerializer + producerValueSerializer: ru.practicum.serializer.AvroSerializer + consumerGroupId: aggregator-group + consumerClientIdConfig: aggregator-consumer + consumerKeyDeserializer: org.apache.kafka.common.serialization.LongDeserializer + consumerValueDeserializer: ru.practicum.deserializer.UserActionDeserializer + consumerEnableAutoCommit: "false" + userActionTopic: stats.user-actions.v1 + eventsSimilarityTopic: stats.events-similarity.v1 + +logging: + level: + ru.practicum: debug + root: info \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml index d06f86c..cd9c0c9 100644 --- a/infra/config-server/src/main/resources/config/stats/analyzer/application.yml +++ b/infra/config-server/src/main/resources/config/stats/analyzer/application.yml @@ -1,56 +1,63 @@ +server: + port: 9090 + +kafka: + bootstrapServers: localhost:9092 + userActionTopic: stats.user-actions.v1 + eventsSimilarityTopic: stats.events-similarity.v1 + + userActionConsumer: + groupId: analyzer-group + clientId: analyzer-consumer + keyDeserializer: org.apache.kafka.common.serialization.LongDeserializer + valueDeserializer: ru.practicum.deserializer.UserActionDeserializer + enableAutoCommit: "false" + maxPollRecords: 500 + maxPollIntervalMs: 300000 + sessionTimeoutMs: 10000 + + eventSimilarityConsumer: + groupId: event-similarity + clientId: event-similarity-client + keyDeserializer: org.apache.kafka.common.serialization.LongDeserializer + valueDeserializer: ru.practicum.deserializer.EventSimilarityDeserializer + enableAutoCommit: "false" + maxPollRecords: 500 + maxPollIntervalMs: 300000 + sessionTimeoutMs: 10000 + spring: - datasource: - url: jdbc:postgresql://localhost:5432/ewm-stats-analyzer-db - username: root - password: root - sql: - init: - mode: always - output: - ansi: - enabled: ALWAYS jpa: - hibernate: - ddl-auto: none - show-sql: false + hibernate.ddl-auto: none + show-sql: true properties: hibernate: format_sql: true - kafka: - consumer-user-actions: - bootstrap-servers: localhost:9092 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.yandex.practicum.deserializer.UserActionDeserializer - client-id: user-consumer - group-id: group-user - max-poll-records: 100 - fetch-max-bytes: 3072000 - max-partition-fetch-bytes: 307200 - consume-attempts-timeout-ms: 1000 - consumer-events-similarity: - bootstrap-servers: localhost:9092 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: ru.practicum.deserializer.EventSimilarityDeserializer - client-id: similarity-consumer - group-id: group-similarity - max-poll-records: 100 - fetch-max-bytes: 3072000 - max-partition-fetch-bytes: 307200 - consume-attempts-timeout-ms: 1000 - producer: - bootstrap-servers: localhost:9092 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.serializer.AvroSerializer - topics: - user-action-topic: stats.user-actions.v1 - events-similarity-topic: stats.events-similarity.v1 + sql.init.mode: always -grpc: - analyzer: - address: 'discovery:///analyzer' - enableKeepAlive: true - keepAliveWithoutCalls: true - negotiationType: plaintext logging: + file: + name: .from_the_beginning/analyzer_report.txt + max-size: 10MB + max-history: 1 level: - io.grpc: DEBUG \ No newline at end of file + ru.practicum.repository: TRACE + ru.practicum.service: DEBUG + org.hibernate.SQL: DEBUG + org.hibernate.type.descriptor.sql.BasicBinder: TRACE + +logging.level: + org.springframework.orm.jpa: INFO + org.springframework.transaction: INFO + ru.practicum: DEBUG +--- + +spring: + config: + activate: + on-profile: dev + datasource: + driver-class-name: org.h2.Driver + url: jdbc:h2:mem:analyzer + username: stats + password: stats \ No newline at end of file diff --git a/infra/config-server/src/main/resources/config/stats/collector/application.yml b/infra/config-server/src/main/resources/config/stats/collector/application.yml index fa9f262..d98ad61 100644 --- a/infra/config-server/src/main/resources/config/stats/collector/application.yml +++ b/infra/config-server/src/main/resources/config/stats/collector/application.yml @@ -1,20 +1,18 @@ -server: - port: 0 +logging: + level: + ru.yandex.practicum: debug + root: info -spring: - kafka: - producer: - bootstrap-servers: localhost:9092 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: ru.practicum.serializer.UserActionsAvroSerializer - topics: - actions-topic: stats.user-actions.v1 grpc: server: port: 0 - client: - collector: - address: 'discovery:///collector' - enableKeepAlive: true - keepAliveWithoutCalls: true - negotiationType: plaintext \ No newline at end of file + +server: + port: 8888 + +kafka: + userActionTopic: stats.user-actions.v1 + bootstrapServers: localhost:9092 + clientIdConfig: collector-client + producerKeySerializer: org.apache.kafka.common.serialization.LongSerializer + producerValueSerializer: ru.practicum.serializer.UserActionsAvroSerializer \ No newline at end of file diff --git a/stats/aggregator/pom.xml b/stats/aggregator/pom.xml index d34ad44..aec55c0 100644 --- a/stats/aggregator/pom.xml +++ b/stats/aggregator/pom.xml @@ -18,27 +18,19 @@ - - org.springframework.boot - spring-boot-starter-web - - ru.practicum avro-schemas 0.0.1-SNAPSHOT - - org.projectlombok - lombok - org.springframework.boot spring-boot-starter - org.apache.kafka - kafka-clients + org.projectlombok + lombok + true org.springframework.cloud @@ -48,10 +40,6 @@ org.springframework.retry spring-retry - - org.springframework.boot - spring-boot-starter-validation - org.springframework.cloud spring-cloud-starter-netflix-eureka-client @@ -61,11 +49,20 @@ spring-boot-starter-actuator + org.springframework.boot spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + diff --git a/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java b/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java index ae72ee5..87a9504 100644 --- a/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java +++ b/stats/aggregator/src/main/java/ru/practicum/AggregatorApplication.java @@ -2,12 +2,15 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.context.ConfigurableApplicationContext; +import ru.practicum.service.AggregationStarter; @SpringBootApplication -@ConfigurationPropertiesScan public class AggregatorApplication { public static void main(String[] args) { - SpringApplication.run(AggregatorApplication.class, args); + ConfigurableApplicationContext context = SpringApplication.run(AggregatorApplication.class, args); + + AggregationStarter aggregator = context.getBean(AggregationStarter.class); + aggregator.start(); } } diff --git a/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java b/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java index 31fae33..6745fdc 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java @@ -1,51 +1,51 @@ -package ru.practicum.config; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.springframework.boot.context.properties.ConfigurationProperties; - -import java.time.Duration; - -@Getter -@Setter -@ToString -@ConfigurationProperties("spring.kafka") -public class AppConfig { - ProducerSettings producer; - ConsumerSettings consumer; - TopicsSettings topics; - - @Setter - @Getter - @ToString - - public static class ProducerSettings { - private String bootstrapServers; - private String keySerializer; - private String valueSerializer; - } - - @Setter - @Getter - @ToString - public static class ConsumerSettings { - private String bootstrapServers; - private String keyDeserializer; - private String valueDeserializer; - private String clientId; - private String groupId; - private String maxPollRecords; - private String fetchMaxBytes; - private String maxPartitionFetchBytes; - private Duration consumeAttemptsTimeoutMs; - } - - @ToString - @Getter - @Setter - public static class TopicsSettings { - private String actionTopic; - private String similarityTopic; - } -} \ No newline at end of file +//package ru.practicum.config; +// +//import lombok.Getter; +//import lombok.Setter; +//import lombok.ToString; +//import org.springframework.boot.context.properties.ConfigurationProperties; +// +//import java.time.Duration; +// +//@Getter +//@Setter +//@ToString +//@ConfigurationProperties("spring.kafka") +//public class AppConfig { +// ProducerSettings producer; +// ConsumerSettings consumer; +// TopicsSettings topics; +// +// @Setter +// @Gettera +// @ToString +// +// public static class ProducerSettings { +// private String bootstrapServers; +// private String keySerializer; +// private String valueSerializer; +// } +// +// @Setter +// @Getter +// @ToString +// public static class ConsumerSettings { +// private String bootstrapServers; +// private String keyDeserializer; +// private String valueDeserializer; +// private String clientId; +// private String groupId; +// private String maxPollRecords; +// private String fetchMaxBytes; +// private String maxPartitionFetchBytes; +// private Duration consumeAttemptsTimeoutMs; +// } +// +// @ToString +// @Getter +// @Setter +// public static class TopicsSettings { +// private String actionTopic; +// private String similarityTopic; +// } +//} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java index e15e007..7ad4423 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfig.java @@ -1,45 +1,51 @@ package ru.practicum.config; -import lombok.Data; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.specific.SpecificRecordBase; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; -import org.springframework.stereotype.Component; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import ru.practicum.ewm.stats.avro.UserActionAvro; import java.util.Properties; @Slf4j -@Component -@Data - +@Getter +@Configuration +@EnableConfigurationProperties({KafkaConfigProperties.class}) public class KafkaConfig { - private final AppConfig appConfig; + private final KafkaConfigProperties kafkaProperties; - public KafkaConfig(AppConfig appConfig) { - this.appConfig = appConfig; + public KafkaConfig(KafkaConfigProperties properties) { + this.kafkaProperties = properties; } - public Properties getConsumerProperties() { + @Bean + public Producer producer() { Properties properties = new Properties(); - properties.put(ConsumerConfig.GROUP_ID_CONFIG, appConfig.getConsumer().getGroupId()); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, appConfig.getConsumer().getBootstrapServers()); - properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, appConfig.getConsumer().getKeyDeserializer()); - properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, appConfig.getConsumer().getValueDeserializer()); - properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, appConfig.getConsumer().getMaxPollRecords()); - properties.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, appConfig.getConsumer().getFetchMaxBytes()); - properties.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, appConfig.getConsumer().getMaxPartitionFetchBytes()); - return properties; + properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, kafkaProperties.getProducerClientIdConfig()); + properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducerKeySerializer()); + properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducerValueSerializer()); + log.info("properties for producer are: {}", properties); + return new KafkaProducer<>(properties); } - public Properties getProducerProperties() { - Properties config = new Properties(); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - appConfig.getProducer().getBootstrapServers()); - config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - appConfig.getProducer().getKeySerializer()); - config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - appConfig.getProducer().getValueSerializer()); - log.info("Kafka producer config is ready = {}", config); - return config; + @Bean + public KafkaConsumer consumer() { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProperties.getConsumerGroupId()); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, kafkaProperties.getConsumerClientIdConfig()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, kafkaProperties.getConsumerKeyDeserializer()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, kafkaProperties.getConsumerValueDeserializer()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaProperties.getConsumerEnableAutoCommit()); + return new KafkaConsumer<>(props); } } \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java new file mode 100644 index 0000000..bcce754 --- /dev/null +++ b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java @@ -0,0 +1,26 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Getter +@Setter +@ConfigurationProperties(prefix = "kafka") +public class KafkaConfigProperties { + private String bootstrapServers; + + private String producerClientIdConfig; + private String producerKeySerializer; + private String producerValueSerializer; + + private String consumerGroupId; + private String consumerClientIdConfig; + private String consumerKeyDeserializer; + private String consumerValueDeserializer; + private long consumerAttemptTimeout; + private String consumerEnableAutoCommit; + + private String userActionTopic; + private String eventsSimilarityTopic; +} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java deleted file mode 100644 index 77ba538..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducer.java +++ /dev/null @@ -1,11 +0,0 @@ -package ru.practicum.config; - -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.kafka.clients.producer.Producer; - -public interface SimilarityEventProducer { - - Producer getProducer(); - - void stop(); -} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java index 320e425..ceb759b 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java @@ -1,47 +1,47 @@ -package ru.practicum.config; - -import jakarta.annotation.PreDestroy; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; - -import java.util.Properties; - -@Slf4j -@RequiredArgsConstructor -@Component -public class SimilarityEventProducerConfiguration { - private final KafkaConfig kafkaConfig; - - @Bean - SimilarityEventProducer getClient() { - return new SimilarityEventProducer() { - private Producer producer; - - @Override - public Producer getProducer() { - if (producer == null) { - initProducer(); - } - return producer; - } - - private void initProducer() { - Properties config = kafkaConfig.getProducerProperties(); - producer = new KafkaProducer<>(config); - } - - @PreDestroy - @Override - public void stop() { - if (producer != null) { - producer.close(); - } - } - }; - } -} \ No newline at end of file +//package ru.practicum.config; +// +//import jakarta.annotation.PreDestroy; +//import lombok.RequiredArgsConstructor; +//import lombok.extern.slf4j.Slf4j; +//import org.apache.avro.specific.SpecificRecordBase; +//import org.apache.kafka.clients.producer.KafkaProducer; +//import org.apache.kafka.clients.producer.Producer; +//import org.springframework.context.annotation.Bean; +//import org.springframework.stereotype.Component; +// +//import java.util.Properties; +// +//@Slf4j +//@RequiredArgsConstructor +//@Component +//public class SimilarityEventProducerConfiguration { +// private final KafkaConfig kafkaConfig; +// +// @Bean +// SimilarityEventProducer getClient() { +// return new SimilarityEventProducer() { +// private Producer producer; +// +// @Override +// public Producer getProducer() { +// if (producer == null) { +// initProducer(); +// } +// return producer; +// } +// +// private void initProducer() { +// Properties config = kafkaConfig.getProducerProperties(); +// producer = new KafkaProducer<>(config); +// } +// +// @PreDestroy +// @Override +// public void stop() { +// if (producer != null) { +// producer.close(); +// } +// } +// }; +// } +//} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java index b98c70d..132f44c 100644 --- a/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregationStarter.java @@ -1,103 +1,87 @@ package ru.practicum.service; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.consumer.*; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.WakeupException; import org.springframework.stereotype.Component; -import ru.practicum.config.AppConfig; import ru.practicum.config.KafkaConfig; -import ru.practicum.config.SimilarityEventProducer; import ru.practicum.ewm.stats.avro.EventSimilarityAvro; import ru.practicum.ewm.stats.avro.UserActionAvro; +import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @Slf4j @Component - +@RequiredArgsConstructor public class AggregationStarter { - private static final Map currentOffsets = new HashMap<>(); private final AggregatorService aggregatorService; - private final SimilarityEventProducer similarityEventProducer; + private final Consumer consumer; private final KafkaConfig kafkaConfig; - private final AppConfig appConfig; + private final Map currentOffsets = new HashMap<>(); + - public AggregationStarter(AggregatorService aggregatorService, SimilarityEventProducer eventProducer, KafkaConfig kafkaConfig, AppConfig appConfig) { - this.aggregatorService = aggregatorService; - this.similarityEventProducer = eventProducer; - this.kafkaConfig = kafkaConfig; - this.appConfig = appConfig; - } - private static void manageOffsets(ConsumerRecord record, int count, - KafkaConsumer consumer) { + private void manageOffsets(ConsumerRecord consumerRecord, int count, Consumer consumer) { currentOffsets.put( - new TopicPartition(record.topic(), record.partition()), - new OffsetAndMetadata(record.offset() + 1) + new TopicPartition(consumerRecord.topic(), consumerRecord.partition()), + new OffsetAndMetadata(consumerRecord.offset() + 1) ); if (count % 10 == 0) { consumer.commitAsync(currentOffsets, (offsets, exception) -> { if (exception != null) { - log.warn("Error occurred while pinning offsets : {}", offsets, exception); + log.warn("Ошибка во время фиксации оффсетов: {}", offsets, exception); } }); } } public void start() { - - KafkaConsumer consumer = new KafkaConsumer<>(kafkaConfig.getConsumerProperties()); Runtime.getRuntime().addShutdownHook(new Thread(consumer::wakeup)); try { - consumer.subscribe(List.of(appConfig.getTopics().getActionTopic())); + consumer.subscribe(List.of(kafkaConfig.getKafkaProperties().getUserActionTopic())); while (true) { - ConsumerRecords records = consumer.poll(appConfig.getConsumer().getConsumeAttemptsTimeoutMs()); - + ConsumerRecords records = consumer + .poll(Duration.ofMillis(kafkaConfig.getKafkaProperties().getConsumerAttemptTimeout())); int count = 0; - for (ConsumerRecord record : records) { + for (ConsumerRecord record : records) { + log.info("UserActionAvro got from consumer: {}", record); handleRecord(record); manageOffsets(record, count, consumer); count++; } - consumer.commitAsync(); - } - } catch (WakeupException | InterruptedException ignores) { + + } catch (WakeupException ignores) { + } catch (Exception e) { - log.error("Error occurred while reading data", e); + log.error("Ошибка во время обработки событий от датчиков", e); } finally { try { consumer.commitSync(currentOffsets); + } finally { - log.info("Closing consumer"); + log.info("Закрываем консьюмер"); consumer.close(); - + log.info("Отправляем все сообщения из буфера продюсера"); + aggregatorService.flush(); + log.info("Закрываем продюсер"); + aggregatorService.close(); } } } - private void handleRecord(ConsumerRecord record) throws InterruptedException { - - log.info("топик = {}, партиция = {}, смещение = {}, значение: {}\n", - record.topic(), record.partition(), record.offset(), record.value()); - List result = aggregatorService.getSimilarities(record.value()); - log.info("Сервис aggregatorService.getSimilarities отработал= {}", result); - if (!result.isEmpty()) { - log.info("Отправляем результаты расчета: {}", result); - for (EventSimilarityAvro event : result) { - similarityEventProducer.getProducer().send(new ProducerRecord<>(appConfig.getTopics().getSimilarityTopic(), - event)); - } + private void handleRecord(ConsumerRecord consumerRecord) throws InterruptedException { + List eventSimilarityList = aggregatorService.updateSimilarity(consumerRecord.value()); + for (EventSimilarityAvro eventSimilarity : eventSimilarityList) { + aggregatorService.collectEventSimilarity(eventSimilarity); } } } \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java index 221dfba..1e9fe29 100644 --- a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java @@ -6,5 +6,12 @@ import java.util.List; public interface AggregatorService { - List getSimilarities(UserActionAvro actionAvro); + List updateSimilarity(UserActionAvro userAction); + + void collectEventSimilarity(EventSimilarityAvro eventSimilarityAvro); + + default void close() { + } + + void flush();; } \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java index 9f9adad..1672f0a 100644 --- a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorServiceImpl.java @@ -2,10 +2,16 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; import org.springframework.stereotype.Service; +import ru.practicum.config.KafkaConfig; import ru.practicum.ewm.stats.avro.EventSimilarityAvro; import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.ewm.stats.avro.ActionTypeAvro; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -16,114 +22,163 @@ @AllArgsConstructor public class AggregatorServiceImpl implements AggregatorService { - private final Map> eventUserWeightMap = new HashMap<>(); - private final Map eventSum = new HashMap<>(); - private final Map> eventMinSum = new HashMap<>(); + private final Producer producer; + private final KafkaConfig kafkaConfig; + + private final Map> eventUserWeights = new HashMap<>(); + private final Map eventTotalWeights = new HashMap<>(); + private final Map> pairMinWeights = new HashMap<>(); @Override - public List getSimilarities(UserActionAvro actionAvro) { - log.info("Service AggregatorServiceImpl.getSimilarities"); - long eventId = actionAvro.getEventId(); - double newScore = getActionScore(actionAvro); - - double currentWeight = 0.0; - if (eventUserWeightMap.containsKey(eventId)) { - currentWeight = eventUserWeightMap.get(eventId).get(eventId); - } else { - currentWeight = newScore; - eventUserWeightMap.put(eventId, new HashMap<>()); - } - if (currentWeight >= newScore) { - log.info("Old weight equals or greater than new one, returning void"); - return List.of(); - } + public List updateSimilarity(UserActionAvro userAction) { + log.info("Processing action for user {} and event {}", + userAction.getUserId(), userAction.getEventId()); + + List results = new ArrayList<>(); + Long eventId = userAction.getEventId(); + Long userId = userAction.getUserId(); + double newWeight = getWeightByActionType(userAction.getActionType()); - double newEventScoreSum; + log.debug("Received weight: {} for event: {}, user: {}", newWeight, eventId, userId); - newEventScoreSum = eventSum.getOrDefault(eventId, 0.0) - currentWeight + newScore; - eventSum.put(eventId, newEventScoreSum); + eventUserWeights.putIfAbsent(eventId, new HashMap<>()); + double currentWeight = eventUserWeights.get(eventId).getOrDefault(userId, 0.0); + log.debug("Current weight: {} for event: {}, user: {}", currentWeight, eventId, userId); + if (newWeight <= currentWeight) { + log.debug("Weight not increased, skipping processing"); + return results; + } + + eventUserWeights.get(eventId).put(userId, newWeight); + log.debug("Updated user weight to: {} for event: {}, user: {}", newWeight, eventId, userId); - Map eventsToRecalculate = getLongDoubleMap(actionAvro, eventId); + double deltaWeight = newWeight - currentWeight; + double newTotalWeight = eventTotalWeights.merge(eventId, deltaWeight, Double::sum); + log.debug("Updated total weight for event {}: {}", eventId, newTotalWeight); - List similarities = new ArrayList<>(); + for (Map.Entry> entry : eventUserWeights.entrySet()) { + Long otherEventId = entry.getKey(); - for (Map.Entry event2 : eventsToRecalculate.entrySet()) { - double minSum = getMinScore(eventId, event2.getKey()); - double deltaMin = Math.min(newScore, event2.getValue()) - Math.min(currentWeight, event2.getValue()); - if (deltaMin != 0) { - minSum += deltaMin; - putMinWeights(eventId, event2.getKey(), minSum); + if (otherEventId.equals(eventId)) { + log.debug("Skipping same event: {}", eventId); + continue; } - double event2Sum = eventSum.get(event2.getKey()); - float score = (float) (minSum / Math.sqrt(newEventScoreSum) / Math.sqrt(event2Sum)); - - similarities.add( - EventSimilarityAvro.newBuilder() - .setEventA(Math.min(eventId, event2.getKey())) - .setEventB(Math.max(eventId, event2.getKey())) - .setTimestamp(actionAvro.getTimestamp()) - .setScore(score) - .build() - ); - } - log.info("New weight {}", similarities); + if (entry.getValue().containsKey(userId)) { + double otherWeight = entry.getValue().get(userId); + log.debug("Found interaction with event: {}, weight: {}", otherEventId, otherWeight); - return similarities; - } + long firstEvent = Math.min(eventId, otherEventId); + long secondEvent = Math.max(eventId, otherEventId); + log.debug("Processing pair: {} and {}", firstEvent, secondEvent); - private Map getLongDoubleMap(UserActionAvro action, long eventId) { - Map eventsToRecalculate = new HashMap<>(); + double oldMin = Math.min(currentWeight, otherWeight); + double newMin = Math.min(newWeight, otherWeight); + double deltaMin = newMin - oldMin; + log.debug("Min weights - old: {}, new: {}, delta: {}", oldMin, newMin, deltaMin); - for (Map.Entry> entry : eventUserWeightMap.entrySet()) { - Long currentEventId = entry.getKey(); - Map userWeights = entry.getValue(); + Map secondLevelMap = pairMinWeights.computeIfAbsent(firstEvent, k -> new HashMap<>()); + double currentSum = secondLevelMap.getOrDefault(secondEvent, 0.0); + double updatedSum = currentSum + deltaMin; + secondLevelMap.put(secondEvent, updatedSum); - if (!currentEventId.equals(eventId) && userWeights.containsKey(action.getUserId())) { - Double weight = userWeights.get(action.getUserId()); - eventsToRecalculate.put(currentEventId, weight); - } - } - return eventsToRecalculate; - } + log.debug("Updated min weights sum for pair ({}, {}): was {}, now {}", + firstEvent, secondEvent, currentSum, updatedSum); + + double sumA = eventTotalWeights.get(firstEvent); + double sumB = eventTotalWeights.get(secondEvent); + log.debug("Total weights - sumA: {}, sumB: {}", sumA, sumB); - private void putMinWeights(long eventA, long eventB, double sum) { - long first = Math.min(eventA, eventB); - long second = Math.max(eventA, eventB); + double score = calculateCosineSimilarity(sumA, sumB, updatedSum); + log.info("Calculated similarity score for events {} and {}: {}", + firstEvent, secondEvent, score); - if (!eventMinSum.containsKey(first)) { - eventMinSum.put(first, new HashMap<>()); + if (score > 0) { + EventSimilarityAvro similarity = createSimilarityAvro(firstEvent, secondEvent, score); + results.add(similarity); + log.debug("Created similarity record: {}", similarity); + } + } } - Map innerMap = eventMinSum.get(first); - innerMap.put(second, sum); + return results; } - private double getMinScore(long eventA, long eventB) { - long first = Math.min(eventA, eventB); - long second = Math.max(eventA, eventB); + private double calculateCosineSimilarity(double sumA, double sumB, double sumMin) { + if (sumA <= 0 || sumB <= 0 || sumMin <= 0) { + log.debug("Invalid input for similarity calculation - sumA: {}, sumB: {}, sumMin: {}", + sumA, sumB, sumMin); + return 0; + } - Double value; - Map innerMap; + double sqrtA = Math.sqrt(sumA); + double sqrtB = Math.sqrt(sumB); + double denominator = sqrtA * sqrtB; - if (!eventMinSum.containsKey(first)) { - innerMap = new HashMap<>(); - eventMinSum.put(first, innerMap); - } else { - innerMap = eventMinSum.get(first); + if (denominator == 0) { + log.debug("Denominator is zero - sumA: {}, sumB: {}", sumA, sumB); + return 0; } - value = innerMap.getOrDefault(second, 0.0); + double score = sumMin / denominator; + double roundedScore = Math.round(score * 100000.0) / 100000.0; + return roundedScore; + } - return value; + private EventSimilarityAvro createSimilarityAvro(long eventA, long eventB, double score) { + return EventSimilarityAvro.newBuilder() + .setEventA(eventA) + .setEventB(eventB) + .setScore((float) score) + .setTimestamp(Instant.now()) + .build(); } - private double getActionScore(UserActionAvro action) { - return switch (action.getActionType()) { + private double getWeightByActionType(ActionTypeAvro actionType) { + return switch (actionType) { case VIEW -> 0.4; case REGISTER -> 0.8; case LIKE -> 1.0; }; } + + @Override + public void collectEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { + try { + ProducerRecord record = new ProducerRecord<>( + kafkaConfig.getKafkaProperties().getEventsSimilarityTopic(), + eventSimilarityAvro.getEventA(), + eventSimilarityAvro); + producer.send(record); + } catch (Exception e) { + log.error("Error sending to Kafka: {}", e.getMessage()); + } + } + + @Override + public void flush() { + if (producer != null) { + producer.flush(); + } + } + + @Override + public void close() { + try { + if (producer != null) { + producer.flush(); + } + } finally { + if (producer != null) { + producer.close(); + } + } + } + + public void resetState() { + eventUserWeights.clear(); + eventTotalWeights.clear(); + pairMinWeights.clear(); + } } \ No newline at end of file diff --git a/stats/aggregator/src/main/resources/application.yml b/stats/aggregator/src/main/resources/application.yml index 3e252c1..f0fb364 100644 --- a/stats/aggregator/src/main/resources/application.yml +++ b/stats/aggregator/src/main/resources/application.yml @@ -2,18 +2,20 @@ spring: application: name: aggregator config: - import: "configserver:" + import: 'configserver:' cloud: config: discovery: + service-id: config-server enabled: true - serviceId: config-server - enabled: true fail-fast: true retry: - useRandomPolicy: true - max-interval: 6000 + use-random-policy: true + max-interval: 10000 eureka: + instance: + prefer-ip-address: true client: - serviceUrl: - defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ \ No newline at end of file + service-url: + defaultZone: http://localhost:8761/eureka + register-with-eureka: true \ No newline at end of file diff --git a/stats/analyzer/pom.xml b/stats/analyzer/pom.xml index 4ef8a15..10747be 100644 --- a/stats/analyzer/pom.xml +++ b/stats/analyzer/pom.xml @@ -19,39 +19,46 @@ - org.springframework.boot - spring-boot-starter-data-jpa + ru.practicum + avro-schemas + 0.0.1-SNAPSHOT + + + ru.practicum + proto-schemas + 0.0.1-SNAPSHOT org.springframework.boot - spring-boot-starter-validation + spring-boot-starter - org.projectlombok - lombok + org.springframework.boot + spring-boot-starter-data-jpa org.springframework.boot - spring-boot-starter + spring-boot-starter-validation - org.apache.kafka - kafka-clients + org.postgresql + postgresql + runtime - ru.practicum - avro-schemas - 0.0.1-SNAPSHOT + com.h2database + h2 + runtime - ru.practicum - proto-schemas - 0.0.1-SNAPSHOT + org.projectlombok + lombok + true - - org.postgresql - postgresql + net.devh + grpc-server-spring-boot-starter + ${grpc-spring-boot-starter.version} org.springframework.cloud @@ -65,35 +72,25 @@ org.springframework.cloud spring-cloud-starter-netflix-eureka-client - - org.mapstruct - mapstruct - 1.5.5.Final - - - org.mapstruct - mapstruct-processor - 1.5.5.Final - provided - - - net.devh - grpc-server-spring-boot-starter - org.springframework.boot spring-boot-starter-actuator - - io.grpc - grpc-stub - + org.springframework.boot spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + diff --git a/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java b/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java index 8544ab8..71939da 100644 --- a/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java +++ b/stats/analyzer/src/main/java/ru/practicum/AnalyzerApplication.java @@ -1,13 +1,27 @@ - package ru.practicum; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.context.ConfigurableApplicationContext; +import ru.practicum.processor.EventSimilarityProcessor; +import ru.practicum.processor.UserActionEventProcessor; +@EnableDiscoveryClient @SpringBootApplication public class AnalyzerApplication { public static void main(String[] args) { - SpringApplication.run(AnalyzerApplication.class, args); - } + ConfigurableApplicationContext context = SpringApplication.run(AnalyzerApplication.class, args); + + final UserActionEventProcessor userActionProcessor = + context.getBean(UserActionEventProcessor.class); + final EventSimilarityProcessor eventSimilarityProcessor = + context.getBean(EventSimilarityProcessor.class); -} + Thread hubEventsThread = new Thread(userActionProcessor); + hubEventsThread.setName("UserActionHandlerThread"); + hubEventsThread.start(); + + eventSimilarityProcessor.run(); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java b/stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java similarity index 50% rename from stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java rename to stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java index eacf240..08f387b 100644 --- a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConsumerSettings.java +++ b/stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java @@ -1,19 +1,17 @@ package ru.practicum.config; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -@Slf4j @Getter @Setter -public class KafkaConsumerSettings { - private String bootstrapServers; +@NoArgsConstructor +public class ConsumerProperties { + private String groupId; + private String clientId; private String keyDeserializer; private String valueDeserializer; - private String clientId; - private String groupId; - private String maxPollRecords; - private int fetchMaxBytes; - private int maxPartitionFetchBytes; + private long attemptTimeout; + private String enableAutoCommit; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java new file mode 100644 index 0000000..61f9085 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfig.java @@ -0,0 +1,54 @@ +package ru.practicum.config; + +import lombok.Getter; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.Properties; + +@Getter +@Configuration +@EnableConfigurationProperties({KafkaConfigProperties.class}) +public class KafkaConfig { + private final KafkaConfigProperties kafkaProperties; + + public KafkaConfig(KafkaConfigProperties properties) { + this.kafkaProperties = properties; + } + + @Bean + public KafkaConsumer getEventSimilarityConsumer() { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProperties.getEventSimilarityConsumer().getGroupId()); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, kafkaProperties.getEventSimilarityConsumer().getClientId()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + kafkaProperties.getEventSimilarityConsumer().getKeyDeserializer()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + kafkaProperties.getEventSimilarityConsumer().getValueDeserializer()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, + kafkaProperties.getEventSimilarityConsumer().getEnableAutoCommit()); + + return new KafkaConsumer<>(props); + } + + @Bean + public KafkaConsumer getUserActionConsumer() { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaProperties.getUserActionConsumer().getGroupId()); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, kafkaProperties.getUserActionConsumer().getClientId()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + kafkaProperties.getUserActionConsumer().getKeyDeserializer()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + kafkaProperties.getUserActionConsumer().getValueDeserializer()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, + kafkaProperties.getUserActionConsumer().getEnableAutoCommit()); + return new KafkaConsumer<>(props); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java new file mode 100644 index 0000000..b1c2a50 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java @@ -0,0 +1,17 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Getter +@Setter +@ConfigurationProperties(prefix = "kafka") +public class KafkaConfigProperties { + private String bootstrapServers; + private ConsumerProperties userActionConsumer; + private ConsumerProperties eventSimilarityConsumer; + + private String userActionTopic; + private String eventsSimilarityTopic; +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java deleted file mode 100644 index d20995e..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/config/KafkaTopics.java +++ /dev/null @@ -1,16 +0,0 @@ -package ru.practicum.config; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Component -@Getter -@Setter -@ConfigurationProperties(prefix = "spring.kafka.topics") -public class KafkaTopics { - private String userActionsTopic; - private String eventsSimilarityTopic; - -} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java index 814cfef..1867f7a 100644 --- a/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java +++ b/stats/analyzer/src/main/java/ru/practicum/controller/RecommendationController.java @@ -1,14 +1,18 @@ package ru.practicum.controller; -import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; -import ru.practicum.grpc.stats.recommendation.RecommendationMessage; -import ru.practicum.grpc.stats.recommendation.RecommendationsControllerGrpc; +import ru.practicum.grpc.stat.dashboard.RecommendationsControllerGrpc; +import ru.practicum.grpc.stat.request.InteractionsCountRequestProto; +import ru.practicum.grpc.stat.request.RecommendedEventProto; +import ru.practicum.grpc.stat.request.SimilarEventsRequestProto; +import ru.practicum.grpc.stat.request.UserPredictionsRequestProto; import ru.practicum.service.RecommendationService; +import java.util.List; + @GrpcService @Slf4j @RequiredArgsConstructor @@ -16,52 +20,56 @@ public class RecommendationController extends RecommendationsControllerGrpc.Reco private final RecommendationService recommendationService; @Override - public void getSimilarEvents(RecommendationMessage.SimilarEventsRequestProto eventsRequestProto, - StreamObserver responseObserver) { + public void getRecommendationsForUser(UserPredictionsRequestProto request, + StreamObserver responseObserver) { + log.info("Получен запрос на рекомендации для пользователя: {}", request); try { - recommendationService.getSimilarEvents(eventsRequestProto) - .forEach(responseObserver::onNext); + List recommendedEvents = recommendationService.generateRecommendationsForUser(request); + recommendedEvents.forEach(responseObserver::onNext); responseObserver.onCompleted(); + log.info("Успешно сформированы рекомендации для пользователя"); } catch (Exception e) { - log.error("Unexpected error occurred in getSimilarEvents: {}", e.getMessage(), e); - responseObserver.onError( - new RuntimeException("Error while trying to complete getSimilarEvents") - ); + log.error("Ошибка при формировании рекомендаций для пользователя: {}", request, e); + responseObserver.onError(io.grpc.Status.INTERNAL + .withDescription("Ошибка сервера при получении рекомендаций: " + e.getMessage()) + .withCause(e) + .asRuntimeException()); } } @Override - public void getRecommendationsForUser(RecommendationMessage.UserPredictionsRequestProto request, - StreamObserver responseObserver) { + public void getSimilarEvents(SimilarEventsRequestProto request, + StreamObserver responseObserver) { + log.info("Получен запрос на поиск похожих событий: {}", request); try { - recommendationService.getRecommendationsForUser(request) - .forEach(responseObserver::onNext); + List similarEvents = recommendationService.getSimilarEvents(request); + similarEvents.forEach(responseObserver::onNext); responseObserver.onCompleted(); + log.info("Успешно найдены похожие события"); } catch (Exception e) { - log.error("Unexpected error occurred in getRecommendationsForUser: {}", e.getMessage(), e); - responseObserver.onError( - new RuntimeException("Error while trying to complete getRecommendationsForUser") - ); + log.error("Ошибка при поиске похожих событий: {}", request, e); + responseObserver.onError(io.grpc.Status.INTERNAL + .withDescription("Ошибка сервера при поиске похожих событий: " + e.getMessage()) + .withCause(e) + .asRuntimeException()); } } @Override - public void getInteractionsCount(RecommendationMessage.InteractionsCountRequestProto request, - StreamObserver responseObserver) { + public void getInteractionsCount(InteractionsCountRequestProto request, + StreamObserver responseObserver) { + log.info("Получен запрос на получение количества взаимодействий: {}", request); try { - log.info("Received request for getting number of activities about event. Stage 1"); - recommendationService.getInteractionsCount(request) - .forEach(responseObserver::onNext); - log.info("Received request for getting number of activities about event. Stage 2"); + List interactions = recommendationService.getInteractionsCount(request); + interactions.forEach(responseObserver::onNext); responseObserver.onCompleted(); - } catch (StatusRuntimeException e) { - log.error("Unexpected error occurred StatusRuntimeException in getSimilarEvents: {}", e.getMessage(), e); - + log.info("Успешно получено количество взаимодействий"); } catch (Exception e) { - log.error("Unexpected error occurred in getSimilarEvents: {}", e.getMessage(), e); - responseObserver.onError( - new RuntimeException("Error while trying to complete GetSimilarEvents") - ); + log.error("Ошибка при получении количества взаимодействий: {}", request, e); + responseObserver.onError(io.grpc.Status.INTERNAL + .withDescription("Ошибка сервера при получении количества взаимодействий: " + e.getMessage()) + .withCause(e) + .asRuntimeException()); } } } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java b/stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java deleted file mode 100644 index e7f5876..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/kafka/ConfigKafkaProperties.java +++ /dev/null @@ -1,61 +0,0 @@ -package ru.practicum.kafka; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import ru.practicum.config.KafkaConsumerSettings; - -import java.util.Properties; - -@Getter -@Configuration -public class ConfigKafkaProperties { - KafkaConsumerSettings kafkaSettings; - - @Bean(name = "user-actions") - @Qualifier("user-actions") - @ConfigurationProperties(prefix = "spring.kafka.consumer-user-actions") - protected KafkaConsumerSettings kafkaSnapshotKafkaConfig() { - return new KafkaConsumerSettings(); - } - - @Bean(name = "events-similarity") - @Qualifier("events-similarity") - @ConfigurationProperties(prefix = "spring.kafka.consumer-events-similarity") - protected KafkaConsumerSettings kafkaHubKafkaConfig() { - return new KafkaConsumerSettings(); - } - - public Properties getSnapshotProperties() { - kafkaSettings = kafkaSnapshotKafkaConfig(); - return getProperties(kafkaSettings); - } - - public Properties getHubProperties() { - kafkaSettings = kafkaHubKafkaConfig(); - return getProperties(kafkaSettings); - } - - private Properties getProperties(KafkaConsumerSettings kafkaSettings) { - Properties properties = new Properties(); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.CLIENT_ID_CONFIG, - kafkaSettings.getClientId()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG, - kafkaSettings.getGroupId()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - kafkaSettings.getBootstrapServers()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - kafkaSettings.getKeyDeserializer()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - kafkaSettings.getValueDeserializer()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.MAX_POLL_RECORDS_CONFIG, - kafkaSettings.getMaxPollRecords()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.FETCH_MAX_BYTES_CONFIG, - kafkaSettings.getFetchMaxBytes()); - properties.put(org.apache.kafka.clients.consumer.ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, - kafkaSettings.getMaxPartitionFetchBytes()); - return properties; - } -} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java b/stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java deleted file mode 100644 index 37be0c7..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/mapper/EventSimilarityMapper.java +++ /dev/null @@ -1,10 +0,0 @@ -package ru.practicum.mapper; - -import org.mapstruct.Mapper; -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -import ru.practicum.model.EventSimilarity; - -@Mapper(componentModel = "spring") -public interface EventSimilarityMapper { - EventSimilarity map(EventSimilarityAvro avro); -} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/mapper/Mapper.java b/stats/analyzer/src/main/java/ru/practicum/mapper/Mapper.java new file mode 100644 index 0000000..d5827b4 --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/mapper/Mapper.java @@ -0,0 +1,43 @@ +package ru.practicum.mapper; + +import ru.practicum.ewm.stats.avro.ActionTypeAvro; +import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.grpc.stat.request.RecommendedEventProto; +import ru.practicum.model.ActionType; +import ru.practicum.model.EventSimilarity; +import ru.practicum.model.RecommendedEvent; +import ru.practicum.model.UserAction; + +public class Mapper { + + public static UserAction mapToUserAction(UserActionAvro userActionAvro) { + return UserAction.builder() + .userId(userActionAvro.getUserId()) + .eventId(userActionAvro.getEventId()) + .actionType(toActionType(userActionAvro.getActionType())) + .created(userActionAvro.getTimestamp()) + .weight(toActionType(userActionAvro.getActionType()).getWeight()) + .build(); + } + + public static ActionType toActionType(ActionTypeAvro actionTypeAvro) { + return ActionType.valueOf(actionTypeAvro.name()); + } + + public static EventSimilarity mapToEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { + return EventSimilarity.builder() + .aeventId(eventSimilarityAvro.getEventA()) + .beventId(eventSimilarityAvro.getEventB()) + .score(eventSimilarityAvro.getScore()) + .build(); + + } + + public static RecommendedEventProto mapToRecommendedEventProto(RecommendedEvent recommendedEvent) { + return RecommendedEventProto.newBuilder() + .setEventId(recommendedEvent.getEventId()) + .setScore(recommendedEvent.getScore()) + .build(); + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/ActionType.java b/stats/analyzer/src/main/java/ru/practicum/model/ActionType.java new file mode 100644 index 0000000..83a1a2b --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/model/ActionType.java @@ -0,0 +1,16 @@ +package ru.practicum.model; + +import lombok.Getter; + +@Getter +public enum ActionType { + VIEW(0.4), + REGISTER(0.8), + LIKE(1.0); + + final double weight; + + ActionType(double weight) { + this.weight = weight; + } +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java index 82de489..1d6025e 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/EventSimilarity.java @@ -3,27 +3,46 @@ import jakarta.persistence.*; import lombok.*; import lombok.experimental.FieldDefaults; +import org.hibernate.proxy.HibernateProxy; -import java.time.Instant; +import java.util.Objects; +@Entity +@Table(name = "event_similarity") @Getter @Setter +@Builder(toBuilder = true) +@FieldDefaults(level = AccessLevel.PRIVATE) @AllArgsConstructor @NoArgsConstructor -@Builder -@Entity -@Table(name = "similarities") -@FieldDefaults(level = AccessLevel.PRIVATE) public class EventSimilarity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) Long id; - @Column(name = "event_id_a") - Long eventA; - @Column(name = "event_id_b") - Long eventB; - @Column(name = "score") - Double score; - @Column(name = "timestamp") - Instant timestamp; + + Long aeventId; + + Long beventId; + + double score; + + @Override + public final boolean equals(Object o) { + if (this == o) return true; + if (o == null) return false; + Class oEffectiveClass = o instanceof HibernateProxy ? + ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass() : o.getClass(); + Class thisEffectiveClass = this instanceof HibernateProxy ? + ((HibernateProxy) this).getHibernateLazyInitializer().getPersistentClass() : this.getClass(); + if (thisEffectiveClass != oEffectiveClass) return false; + EventSimilarity eventSimilarity = (EventSimilarity) o; + return getId() != null && Objects.equals(getId(), eventSimilarity.getId()); + } + + @Override + public final int hashCode() { + return this instanceof HibernateProxy + ? ((HibernateProxy) this).getHibernateLazyInitializer().getPersistentClass().hashCode() + : getClass().hashCode(); + } } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java b/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java index 47cd704..1b01c73 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/RecommendedEvent.java @@ -1,7 +1,14 @@ package ru.practicum.model; -public record RecommendedEvent( - long eventId, - double score -) { +import lombok.*; +import lombok.experimental.FieldDefaults; + +@Builder +@Getter +@Setter +@ToString +@FieldDefaults(level = AccessLevel.PRIVATE) +public class RecommendedEvent { + Long eventId; + double score; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java index c4de9e7..073a6e3 100644 --- a/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java +++ b/stats/analyzer/src/main/java/ru/practicum/model/UserAction.java @@ -1,32 +1,54 @@ package ru.practicum.model; - import jakarta.persistence.*; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.*; +import lombok.experimental.FieldDefaults; +import org.hibernate.proxy.HibernateProxy; import java.time.Instant; +import java.util.Objects; +@Entity +@Table(name = "user_action") @Getter @Setter +@Builder(toBuilder = true) +@FieldDefaults(level = AccessLevel.PRIVATE) @AllArgsConstructor @NoArgsConstructor -@Builder -@Entity -@Table(name = "actions") public class UserAction { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - @Column(name = "user_id") - private Long userId; - @Column(name = "event_id") - private Long eventId; - @Column(name = "score") - private Double score; - @Column(name = "lastInteraction") - private Instant lastInteraction; + Long id; + + Long userId; + + Long eventId; + + @Enumerated(EnumType.STRING) + ActionType actionType; + + Instant created; + + double weight; + + @Override + public final boolean equals(Object o) { + if (this == o) return true; + if (o == null) return false; + Class oEffectiveClass = o instanceof HibernateProxy ? + ((HibernateProxy) o).getHibernateLazyInitializer().getPersistentClass() : o.getClass(); + Class thisEffectiveClass = this instanceof HibernateProxy ? + ((HibernateProxy) this).getHibernateLazyInitializer().getPersistentClass() : this.getClass(); + if (thisEffectiveClass != oEffectiveClass) return false; + UserAction userAction = (UserAction) o; + return getId() != null && Objects.equals(getId(), userAction.getId()); + } + + @Override + public final int hashCode() { + return this instanceof HibernateProxy + ? ((HibernateProxy) this).getHibernateLazyInitializer().getPersistentClass().hashCode() + : getClass().hashCode(); + } } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java b/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java index 6e27c7b..27cbe95 100644 --- a/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java +++ b/stats/analyzer/src/main/java/ru/practicum/processor/EventSimilarityProcessor.java @@ -2,74 +2,83 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.WakeupException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import ru.practicum.config.KafkaConfig; import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -import ru.practicum.config.KafkaTopics; -import ru.practicum.kafka.ConfigKafkaProperties; -import ru.practicum.service.event.EventSimilarityService; +import ru.practicum.mapper.Mapper; +import ru.practicum.model.EventSimilarity; +import ru.practicum.repository.EventSimilarityRepository; import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; @Slf4j @Component @RequiredArgsConstructor public class EventSimilarityProcessor implements Runnable { - @Value(value = "${spring.kafka.consumer-events-similarity.consume-attempts-timeout-ms}") - private Duration consumeAttemptTimeout; - private final Map currentOffsets = new HashMap<>();// снимок состояния - private final ConfigKafkaProperties configClass; - private final EventSimilarityService eventSimilarityService; - private final KafkaTopics kafkaTopics; + + private final Consumer consumer; + private final KafkaConfig kafkaConfig; + private final Map currentOffsets = new HashMap<>(); + private final EventSimilarityRepository eventSimilarityRepository; @Override public void run() { - Properties config = configClass.getSnapshotProperties(); - KafkaConsumer consumer = new KafkaConsumer<>(config); Runtime.getRuntime().addShutdownHook(new Thread(consumer::wakeup)); - try { - consumer.subscribe(List.of(kafkaTopics.getEventsSimilarityTopic())); + consumer.subscribe(List.of(kafkaConfig.getKafkaProperties().getEventsSimilarityTopic())); while (true) { - ConsumerRecords records = consumer.poll(consumeAttemptTimeout); + ConsumerRecords records = consumer + .poll(Duration.ofMillis(kafkaConfig.getKafkaProperties() + .getEventSimilarityConsumer().getAttemptTimeout())); int count = 0; - for (ConsumerRecord record : records) { + for (ConsumerRecord record : records) { handleRecord(record); manageOffsets(record, count, consumer); count++; } - consumer.commitAsync(); } - } catch (WakeupException | InterruptedException ignores) { + + } catch (WakeupException ignores) { } catch (Exception e) { - log.error("Error while reading", e); + log.error("Ошибка во время обработки события похожести ", e); } finally { try { consumer.commitSync(currentOffsets); + } finally { - log.info("Closing consumer"); + log.info("Закрываем консьюмер"); consumer.close(); - } } } - private void manageOffsets(ConsumerRecord record, int count, - KafkaConsumer consumer) { + private void handleRecord(ConsumerRecord consumerRecord) throws InterruptedException { + log.info("handleRecord {}", consumerRecord); + EventSimilarity eventSimilarity = Mapper.mapToEventSimilarity(consumerRecord.value()); + + eventSimilarityRepository.findByAeventIdAndBeventId( + eventSimilarity.getAeventId(), + eventSimilarity.getBeventId()).ifPresent(oldEventSimilarity -> + eventSimilarity.setId(oldEventSimilarity.getId())); + eventSimilarityRepository.save(eventSimilarity); + } + + private void manageOffsets(ConsumerRecord consumerRecord, + int count, + Consumer consumer) { currentOffsets.put( - new TopicPartition(record.topic(), record.partition()), - new OffsetAndMetadata(record.offset() + 1) + new TopicPartition(consumerRecord.topic(), consumerRecord.partition()), + new OffsetAndMetadata(consumerRecord.offset() + 1) ); if (count % 10 == 0) { @@ -80,11 +89,4 @@ private void manageOffsets(ConsumerRecord record, i }); } } - - private void handleRecord(ConsumerRecord record) throws InterruptedException { - - log.info("топик = {}, партиция = {}, смещение = {}, значение: {}\n", - record.topic(), record.partition(), record.offset(), record.value()); - eventSimilarityService.handleEventSimilarity(record.value()); - } } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java b/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java index 54ec9ad..45e4bf8 100644 --- a/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java +++ b/stats/analyzer/src/main/java/ru/practicum/processor/UserActionEventProcessor.java @@ -2,88 +2,84 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.WakeupException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import ru.practicum.config.KafkaConfig; import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.config.KafkaTopics; -import ru.practicum.kafka.ConfigKafkaProperties; -import ru.practicum.service.user.UserActionService; +import ru.practicum.service.RecommendationService; import java.time.Duration; +import java.util.HashMap; import java.util.List; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; - +import java.util.Map; @Slf4j @Component @RequiredArgsConstructor public class UserActionEventProcessor implements Runnable { - @Value(value = "${spring.kafka.consumer-user-actions.consume-attempts-timeout-ms}") - private Duration consumeAttemptTimeout; - private final ConcurrentHashMap currentOffsets = new ConcurrentHashMap<>();// снимок состояния - private final ConfigKafkaProperties consumerConfig; - private final UserActionService userActionService; - private final KafkaTopics kafkaTopics; + private final Consumer consumer; + private final KafkaConfig kafkaConfig; + private final Map currentOffsets = new HashMap<>(); + private final RecommendationService recommendationService; @Override public void run() { - Properties config = consumerConfig.getHubProperties(); - KafkaConsumer consumer = new KafkaConsumer<>(config); Runtime.getRuntime().addShutdownHook(new Thread(consumer::wakeup)); - try { - consumer.subscribe(List.of(kafkaTopics.getUserActionsTopic())); + consumer.subscribe(List.of(kafkaConfig.getKafkaProperties().getUserActionTopic())); while (true) { - ConsumerRecords records = consumer.poll(consumeAttemptTimeout); + ConsumerRecords records = consumer + .poll(Duration.ofMillis(kafkaConfig.getKafkaProperties() + .getUserActionConsumer().getAttemptTimeout())); int count = 0; - for (ConsumerRecord record : records) { + for (ConsumerRecord record : records) { handleRecord(record); manageOffsets(record, count, consumer); count++; } - consumer.commitAsync(); } - } catch (WakeupException | InterruptedException ignores) { + + } catch (WakeupException ignores) { + } catch (Exception e) { - log.error("Error occurred while reading from hubs", e); + log.error("Ошибка во время обработки события хаба ", e); } finally { + try { consumer.commitSync(currentOffsets); + } finally { - log.info("Closing consumers"); + log.info("Закрываем консьюмер"); consumer.close(); } } } - private void manageOffsets(ConsumerRecord record, int count, - KafkaConsumer consumer) { + private void handleRecord(ConsumerRecord consumerRecord) throws InterruptedException { + log.info("handleRecord {}", consumerRecord); + recommendationService.saveUserAction(consumerRecord.value()); + } + + private void manageOffsets(ConsumerRecord consumerRecord, + int count, + Consumer consumer) { currentOffsets.put( - new TopicPartition(record.topic(), record.partition()), - new OffsetAndMetadata(record.offset() + 1) + new TopicPartition(consumerRecord.topic(), consumerRecord.partition()), + new OffsetAndMetadata(consumerRecord.offset() + 1) ); if (count % 10 == 0) { consumer.commitAsync(currentOffsets, (offsets, exception) -> { if (exception != null) { - log.warn("Error with offset fixation: {}", offsets, exception); + log.warn("Ошибка во время фиксации оффсетов: {}", offsets, exception); } }); } } - - private void handleRecord(ConsumerRecord record) throws InterruptedException { - - log.info("topic = {}, partition = {}, changing = {}, value: {}\n", - record.topic(), record.partition(), record.offset(), record.value()); - userActionService.handleUserAction(record.value()); - } } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java b/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java index 1cbb5ba..f9e1b8c 100644 --- a/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java +++ b/stats/analyzer/src/main/java/ru/practicum/repository/EventSimilarityRepository.java @@ -1,16 +1,29 @@ package ru.practicum.repository; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; import ru.practicum.model.EventSimilarity; import java.util.List; import java.util.Optional; +@Repository public interface EventSimilarityRepository extends JpaRepository { - Optional findEventSimilaritiesByEventAAndEventB(long eventA, long eventB); + Optional findByAeventIdAndBeventId(Long aEventId, Long bEventId); - List findAllByEventAOrEventB(long eventA, long eventB); + @Query("select es from EventSimilarity es where es.aeventId = :id or es.beventId = :id") + List findAllByEvent(@Param("id") Long eventId); - List findAllByEventAInOrEventBIn(List eventIdsA, List eventIdsB); -} + @Query("select es from EventSimilarity es " + + " where (es.aeventId = :id and es.beventId in :ids) or " + + " (es.beventId = :id and es.aeventId in :ids) " + + " order by es.score desc" + + " limit :limit") + List findAllByEventAndEventIdInLimitedTo( + @Param("id") Long eventId, + @Param("ids") List eventIds, + @Param("limit") Long limit); +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java b/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java index fac174f..c6aa963 100644 --- a/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java +++ b/stats/analyzer/src/main/java/ru/practicum/repository/UserActionRepository.java @@ -1,18 +1,26 @@ package ru.practicum.repository; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; +import ru.practicum.model.RecommendedEvent; import ru.practicum.model.UserAction; import java.util.List; import java.util.Optional; +@Repository public interface UserActionRepository extends JpaRepository { Optional findByUserIdAndEventId(Long userId, Long eventId); - List findByUserId(Long userId); + List findAllByUserId(Long userId); - List findByEventId(Long eventId); + @Query("SELECT new ru.practicum.model.RecommendedEvent(ua.eventId, sum(ua.weight)) " + + "FROM UserAction ua WHERE ua.eventId in :ids GROUP BY ua.eventId") + List getSumWeightForEvents(@Param("ids") List ids); - List findByEventIdIsIn(List eventIds); -} + @Query("SELECT ua FROM UserAction ua WHERE ua.userId = :id ORDER BY ua.created DESC LIMIT :limit") + List findByUserIdOrderByCreatedDescLimitedTo(@Param("id") Long userId, @Param("limit") long limit); +} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java index 3129bac..eaf4300 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationService.java @@ -1,166 +1,20 @@ package ru.practicum.service; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import ru.practicum.grpc.stats.recommendation.RecommendationMessage; -import ru.practicum.model.EventSimilarity; -import ru.practicum.model.UserAction; -import ru.practicum.repository.EventSimilarityRepository; -import ru.practicum.repository.UserActionRepository; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.grpc.stat.request.InteractionsCountRequestProto; +import ru.practicum.grpc.stat.request.RecommendedEventProto; +import ru.practicum.grpc.stat.request.SimilarEventsRequestProto; +import ru.practicum.grpc.stat.request.UserPredictionsRequestProto; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.List; -@Slf4j -@Service -@RequiredArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class RecommendationService { +public interface RecommendationService { - private final UserActionRepository userActionRepository; - private final EventSimilarityRepository eventSimilarityRepository; + List generateRecommendationsForUser(UserPredictionsRequestProto request); - public List getRecommendationsForUser(RecommendationMessage.UserPredictionsRequestProto requestProto) { - log.info("Recommendations for user: {}", requestProto.getUserId()); - long userId = requestProto.getUserId(); - int maxResult = requestProto.getMaxResults(); + List getSimilarEvents(SimilarEventsRequestProto request); - List userActionList = new ArrayList<>(userActionRepository.findByUserId(userId)); - if (userActionList.isEmpty()) { - return Collections.emptyList(); - } - - userActionList.sort((a, b) -> b.getLastInteraction().compareTo(a.getLastInteraction())); - - int n = 10; - List userActionListLimited = userActionList.stream().limit(n).toList(); - - Set interactedByUserLimitedEvents = userActionListLimited - .stream() - .map(UserAction::getEventId) - .collect(Collectors.toSet()); - - - Set interactedByUserAllEvents = userActionList - .stream() - .map(UserAction::getEventId) - .collect(Collectors.toSet()); - - List eventsToRequest = interactedByUserLimitedEvents.stream().toList(); - List eventSimilarityList = eventSimilarityRepository - .findAllByEventAInOrEventBIn(eventsToRequest, eventsToRequest); - - Set uniqueEvents = new HashSet<>(eventSimilarityList); - - List sortedList = uniqueEvents.stream() - .sorted(Comparator.comparing(EventSimilarity::getScore).reversed()) - .toList(); - - List unwatchedSotedLimitedEventsList = eventSimilarityList.stream() - .collect(Collectors.flatMapping( - es -> Stream.of(es.getEventA(), es.getEventB()), - Collectors.toList() - )).stream() - .filter(interactedByUserAllEvents::contains) - .limit(n) - .toList(); - - - Map weightedScoreForUnwatchedEvent = new HashMap<>(); - - for (EventSimilarity eventSimilarity : sortedList) { - long eventId = eventsToRequest.contains(eventSimilarity.getEventA()) - ? eventSimilarity.getEventA() : eventSimilarity.getEventB(); - - if (!weightedScoreForUnwatchedEvent.containsKey(eventId)) { - double score = eventSimilarity.getScore() * userActionList.stream() - .filter(event -> event.getEventId() == eventId) - .findFirst().map(UserAction::getScore).orElse(1.0); - - weightedScoreForUnwatchedEvent.put(eventId, score); - } else { - double score = eventSimilarity.getScore() * userActionList.stream() - .filter(event -> event.getEventId() == eventId) - .findFirst().map(UserAction::getScore).orElse(1.0); - double oldScore = weightedScoreForUnwatchedEvent.get(eventId); - weightedScoreForUnwatchedEvent.put(eventId, score + oldScore); - } - } - - List eventProtoList = new ArrayList<>(); - for (Long eventId : weightedScoreForUnwatchedEvent.keySet()) { - - RecommendationMessage.RecommendedEventProto.newBuilder() - .setEventId(eventId) - .setScore(weightedScoreForUnwatchedEvent.get(eventId) / weightedScoreForUnwatchedEvent.size()); - } - - return eventProtoList; - } - - public List getSimilarEvents(RecommendationMessage.SimilarEventsRequestProto eventsRequestProto) { - long eventId = eventsRequestProto.getEventId(); - List similarEventitsList = eventSimilarityRepository.findAllByEventAOrEventB(eventId, eventId); - Set watchedByUserEvents = userActionRepository.findByUserId(eventsRequestProto.getUserId()) - .stream() - .map(UserAction::getEventId) - .collect(Collectors.toSet()); - List finalEventList = new ArrayList<>(similarEventitsList); - for (EventSimilarity eventSimilarity : similarEventitsList) { - if (watchedByUserEvents.contains(eventSimilarity.getEventA()) && watchedByUserEvents.contains(eventSimilarity.getEventB())) { - finalEventList.remove(eventSimilarity); - } - } - List recommendedEventList = new ArrayList<>(); - for (EventSimilarity eventSimilarity : finalEventList) { - RecommendationMessage.RecommendedEventProto eventProto; - if (eventsRequestProto.getEventId() != eventSimilarity.getEventA()) { - eventProto = RecommendationMessage.RecommendedEventProto.newBuilder() - .setEventId(eventSimilarity.getEventA()) - .setScore(eventSimilarity.getScore()) - .build(); - } else { - eventProto = RecommendationMessage.RecommendedEventProto.newBuilder() - .setEventId(eventSimilarity.getEventB()) - .setScore(eventSimilarity.getScore()) - .build(); - } - recommendedEventList.add(eventProto); - } - - return recommendedEventList.stream().sorted(Comparator.comparingDouble(RecommendationMessage.RecommendedEventProto::getScore) - .reversed()).limit(eventsRequestProto.getMaxResults()).toList(); - } - - public List getInteractionsCount(RecommendationMessage.InteractionsCountRequestProto request) { - log.info("Method getInteractionsCount began its work"); - List userActionList = userActionRepository.findByEventIdIsIn(request.getEventIdList()); - Map recommendedEventMap = new HashMap<>(); - List recommendedEventList = new ArrayList<>(); - - for (UserAction userAction : userActionList) { - if (!recommendedEventMap.containsKey(userAction.getEventId())) { - - recommendedEventMap.put(userAction.getEventId(), userAction.getScore()); - } else { - - Double newScore = recommendedEventMap.get(userAction.getEventId()) + userAction.getScore(); - recommendedEventMap.put(userAction.getEventId(), newScore); - } - } - for (long eventId : recommendedEventMap.keySet()) { - RecommendationMessage.RecommendedEventProto eventProto = RecommendationMessage.RecommendedEventProto.newBuilder() - .setEventId(eventId) - .setScore(recommendedEventMap.get(eventId)) - .build(); - recommendedEventList.add(eventProto); - } - log.info("Method getInteractionsCount ended its work"); - return recommendedEventList; - } + List getInteractionsCount(InteractionsCountRequestProto request); + void saveUserAction(UserActionAvro userActionAvro); } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/RecommendationServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationServiceImpl.java new file mode 100644 index 0000000..cf1e02b --- /dev/null +++ b/stats/analyzer/src/main/java/ru/practicum/service/RecommendationServiceImpl.java @@ -0,0 +1,163 @@ +package ru.practicum.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import ru.practicum.ewm.stats.avro.UserActionAvro; +import ru.practicum.grpc.stat.request.InteractionsCountRequestProto; +import ru.practicum.grpc.stat.request.RecommendedEventProto; +import ru.practicum.grpc.stat.request.SimilarEventsRequestProto; +import ru.practicum.grpc.stat.request.UserPredictionsRequestProto; +import ru.practicum.model.EventSimilarity; +import ru.practicum.model.RecommendedEvent; +import ru.practicum.model.UserAction; +import ru.practicum.repository.EventSimilarityRepository; +import ru.practicum.repository.UserActionRepository; +import ru.practicum.mapper.Mapper; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +@Service +@Slf4j +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class RecommendationServiceImpl implements RecommendationService { + private static final long EVENT_COUNT_PREDICTION = 5; + private final EventSimilarityRepository eventSimilarityRepository; + private final UserActionRepository userActionRepository; + + @Override + public List generateRecommendationsForUser(UserPredictionsRequestProto request) { + List lastUserEvents = userActionRepository.findByUserIdOrderByCreatedDescLimitedTo( + request.getUserId(), request.getMaxResults() + ); + + if (lastUserEvents.isEmpty()) { + return emptyList(); + } + + List recommendedEvents = new ArrayList<>(); + lastUserEvents.forEach(event -> recommendedEvents.addAll( + getSimilarEvents(request.getUserId(), event.getEventId(), request.getMaxResults()) + .stream() + .sorted(Comparator.comparingDouble(EventSimilarity::getScore).reversed()) + .limit(request.getMaxResults()) + .map(similarEvent -> genRecommendedEventFrom(similarEvent, event.getEventId())) + .toList())); + + List limitRecommendedEvents = recommendedEvents.stream() + .sorted(Comparator.comparingDouble(RecommendedEvent::getScore).reversed()) + .limit(request.getMaxResults()) + .toList(); + log.info("RecommendedEvents: {}", recommendedEvents); + limitRecommendedEvents.forEach( + event -> event.setScore(getPrediction(event.getEventId(), request.getUserId())) + ); + return limitRecommendedEvents.stream() + .map(Mapper::mapToRecommendedEventProto) + .toList(); + } + + @Override + public List getSimilarEvents(SimilarEventsRequestProto request) { + return getSimilarEvents(request.getUserId(), request.getEventId(), request.getMaxResults()).stream() + .map(event -> genRecommendedEventProtoFrom(event, request.getEventId())) + .toList(); + } + + @Override + public List getInteractionsCount(InteractionsCountRequestProto request) { + return userActionRepository.getSumWeightForEvents(request.getEventIdList()) + .stream() + .map(Mapper::mapToRecommendedEventProto) + .toList(); + } + + @Override + @Transactional + public void saveUserAction(UserActionAvro userActionAvro) { + UserAction userAction = Mapper.mapToUserAction(userActionAvro); + log.info("Saving UserAction: userId={}, eventId={}, type={}, weight={}", + userAction.getUserId(), userAction.getEventId(), userAction.getActionType(), userAction.getWeight()); + + Optional oldUserAction = userActionRepository.findByUserIdAndEventId( + userAction.getUserId(), userAction.getEventId() + ); + + if (oldUserAction.isPresent()) { + log.info("Updating existing UserAction: oldWeight={}", oldUserAction.get().getWeight()); + userAction.setId(oldUserAction.get().getId()); + if (userAction.getWeight() < oldUserAction.get().getWeight()) { + userAction.setWeight(oldUserAction.get().getWeight()); + } + } + + UserAction savedAction = userActionRepository.save(userAction); + log.info("Saved UserAction: id={}, weight={}", savedAction.getId(), savedAction.getWeight()); + } + + private RecommendedEvent genRecommendedEventFrom(EventSimilarity eventSimilarity, Long eventId) { + Long recommendedEventId = Objects.equals(eventSimilarity.getAeventId(), eventId) ? + eventSimilarity.getBeventId() : eventSimilarity.getAeventId(); + + return RecommendedEvent.builder() + .eventId(recommendedEventId) + .score(eventSimilarity.getScore()) + .build(); + } + + private RecommendedEventProto genRecommendedEventProtoFrom(EventSimilarity eventSimilarity, Long eventId) { + Long recommendedEventId = Objects.equals(eventSimilarity.getAeventId(), eventId) ? + eventSimilarity.getBeventId() : eventSimilarity.getAeventId(); + + return RecommendedEventProto.newBuilder() + .setEventId(recommendedEventId) + .setScore(eventSimilarity.getScore()) + .build(); + } + + private List getSimilarEvents(Long userId, Long eventId, Long limit) { + List events = eventSimilarityRepository.findAllByEvent(eventId); + List actions = userActionRepository.findAllByUserId(userId).stream() + .map(UserAction::getEventId).toList(); + + List result = events.stream() + .filter(event -> !(actions.contains(event.getAeventId()) && actions.contains(event.getBeventId()))) + .sorted(Comparator.comparingDouble(EventSimilarity::getScore).reversed()) + .limit(limit) + .toList(); + + log.info("similar events are {}", result); + return result; + } + + private double getPrediction(Long eventId, Long userId) { + double prediction = 0.0; + + Map ratedEvents = userActionRepository.findAllByUserId(userId).stream() + .collect(Collectors.toMap(UserAction::getEventId, UserAction::getWeight)); + List similarEvents = eventSimilarityRepository.findAllByEventAndEventIdInLimitedTo( + eventId, ratedEvents.keySet().stream().toList(), EVENT_COUNT_PREDICTION) + .stream() + .map(eventSimilarity -> genRecommendedEventFrom(eventSimilarity, eventId)) + .toList(); + + double weightedSum = 0.0; + double similaritySum = 0.0; + + for (RecommendedEvent event : similarEvents) { + weightedSum += event.getScore() * ratedEvents.get(event.getEventId()); + similaritySum += event.getScore(); + } + + if (similaritySum != 0) { + prediction = weightedSum / similaritySum; + } + + return prediction; + } +} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java index cf89782..1a03c0b 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java @@ -1,8 +1,8 @@ -package ru.practicum.service.event; - -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; - -public interface EventSimilarityService { - - void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro); -} +//package ru.practicum.service.event; +// +//import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +// +//public interface EventSimilarityService { +// +// void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro); +//} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java index 2a043ff..c7aaef3 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java @@ -1,39 +1,39 @@ -package ru.practicum.service; - -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -import ru.practicum.mapper.EventSimilarityMapper; -import ru.practicum.model.EventSimilarity; -import ru.practicum.repository.EventSimilarityRepository; -import ru.practicum.service.event.EventSimilarityService; - -import java.util.Optional; - -@Service -@Slf4j -@AllArgsConstructor -public class EventSimilarityServiceImpl implements EventSimilarityService { - - private final EventSimilarityRepository eventSimilarityRepository; - private final EventSimilarityMapper eventSimilarityMapper; - - - @Override - public void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { - log.info("сервис EventSimilarityServiceImpl начал обработку eventSimilarityAvro {}", eventSimilarityAvro); - EventSimilarity eventSimilarity = eventSimilarityMapper.map(eventSimilarityAvro); - Optional eventSimilarityOptional = eventSimilarityRepository - .findEventSimilaritiesByEventAAndEventB(eventSimilarityAvro.getEventA(), eventSimilarityAvro.getEventB()); - - if (eventSimilarityOptional.isPresent()) { - eventSimilarityOptional.get().setScore(eventSimilarity.getScore()); - eventSimilarityOptional.get().setTimestamp(eventSimilarity.getTimestamp()); - eventSimilarityRepository.save(eventSimilarityOptional.get()); - - } else { - eventSimilarityRepository.save(eventSimilarity); - } - } -} \ No newline at end of file +//package ru.practicum.service; +// +//import lombok.AllArgsConstructor; +//import lombok.extern.slf4j.Slf4j; +//import org.springframework.stereotype.Service; +//import ru.practicum.ewm.stats.avro.EventSimilarityAvro; +//import ru.practicum.mapper.EventSimilarityMapper; +//import ru.practicum.model.EventSimilarity; +//import ru.practicum.repository.EventSimilarityRepository; +//import ru.practicum.service.event.EventSimilarityService; +// +//import java.util.Optional; +// +//@Service +//@Slf4j +//@AllArgsConstructor +//public class EventSimilarityServiceImpl implements EventSimilarityService { +// +// private final EventSimilarityRepository eventSimilarityRepository; +// private final EventSimilarityMapper eventSimilarityMapper; +// +// +// @Override +// public void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { +// log.info("сервис EventSimilarityServiceImpl начал обработку eventSimilarityAvro {}", eventSimilarityAvro); +// EventSimilarity eventSimilarity = eventSimilarityMapper.map(eventSimilarityAvro); +// Optional eventSimilarityOptional = eventSimilarityRepository +// .findEventSimilaritiesByEventAAndEventB(eventSimilarityAvro.getEventA(), eventSimilarityAvro.getEventB()); +// +// if (eventSimilarityOptional.isPresent()) { +// eventSimilarityOptional.get().setScore(eventSimilarity.getScore()); +// eventSimilarityOptional.get().setTimestamp(eventSimilarity.getTimestamp()); +// eventSimilarityRepository.save(eventSimilarityOptional.get()); +// +// } else { +// eventSimilarityRepository.save(eventSimilarity); +// } +// } +//} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java index b462ac1..15cbb47 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java @@ -1,7 +1,7 @@ -package ru.practicum.service.user; - -import ru.practicum.ewm.stats.avro.UserActionAvro; - -public interface UserActionService { - void handleUserAction(UserActionAvro userActionAvro); -} +//package ru.practicum.service.user; +// +//import ru.practicum.ewm.stats.avro.UserActionAvro; +// +//public interface UserActionService { +// void handleUserAction(UserActionAvro userActionAvro); +//} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java index d885aaf..7c788d6 100644 --- a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java +++ b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java @@ -1,56 +1,56 @@ -package ru.practicum.service.user; - -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import ru.practicum.ewm.stats.avro.ActionTypeAvro; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.model.UserAction; -import ru.practicum.repository.UserActionRepository; - -import java.util.Optional; - - -@Service -@Slf4j -@AllArgsConstructor -public class - -UserActionServiceImpl implements UserActionService { - private final UserActionRepository userActionRepository; - - @Override - public void handleUserAction(UserActionAvro userActionAvro) { - - Optional userActionOptional = userActionRepository.findByUserIdAndEventId(userActionAvro.getUserId(), - userActionAvro.getEventId()); - if (userActionOptional.isPresent()) { - if (userActionOptional.get().getScore() <= calcInteractionScore(userActionAvro.getActionType())) { - UserAction userAction = UserAction.builder() - .id(userActionOptional.get().getId()) - .userId(userActionAvro.getUserId()) - .lastInteraction(userActionAvro.getTimestamp()) - .eventId(userActionAvro.getEventId()) - .score(calcInteractionScore(userActionAvro.getActionType())) - .build(); - userActionRepository.save(userAction); - } - } else { - UserAction userAction = UserAction.builder() - .userId(userActionAvro.getUserId()) - .lastInteraction(userActionAvro.getTimestamp()) - .eventId(userActionAvro.getEventId()) - .score(calcInteractionScore(userActionAvro.getActionType())) - .build(); - userActionRepository.save(userAction); - } - } - - private double calcInteractionScore(ActionTypeAvro type) { - return switch (type) { - case VIEW -> 0.4; - case REGISTER -> 0.8; - case LIKE -> 1.0; - }; - } -} \ No newline at end of file +//package ru.practicum.service.user; +// +//import lombok.AllArgsConstructor; +//import lombok.extern.slf4j.Slf4j; +//import org.springframework.stereotype.Service; +//import ru.practicum.ewm.stats.avro.ActionTypeAvro; +//import ru.practicum.ewm.stats.avro.UserActionAvro; +//import ru.practicum.model.UserAction; +//import ru.practicum.repository.UserActionRepository; +// +//import java.util.Optional; +// +// +//@Service +//@Slf4j +//@AllArgsConstructor +//public class +// +//UserActionServiceImpl implements UserActionService { +// private final UserActionRepository userActionRepository; +// +// @Override +// public void handleUserAction(UserActionAvro userActionAvro) { +// +// Optional userActionOptional = userActionRepository.findByUserIdAndEventId(userActionAvro.getUserId(), +// userActionAvro.getEventId()); +// if (userActionOptional.isPresent()) { +// if (userActionOptional.get().getScore() <= calcInteractionScore(userActionAvro.getActionType())) { +// UserAction userAction = UserAction.builder() +// .id(userActionOptional.get().getId()) +// .userId(userActionAvro.getUserId()) +// .lastInteraction(userActionAvro.getTimestamp()) +// .eventId(userActionAvro.getEventId()) +// .score(calcInteractionScore(userActionAvro.getActionType())) +// .build(); +// userActionRepository.save(userAction); +// } +// } else { +// UserAction userAction = UserAction.builder() +// .userId(userActionAvro.getUserId()) +// .lastInteraction(userActionAvro.getTimestamp()) +// .eventId(userActionAvro.getEventId()) +// .score(calcInteractionScore(userActionAvro.getActionType())) +// .build(); +// userActionRepository.save(userAction); +// } +// } +// +// private double calcInteractionScore(ActionTypeAvro type) { +// return switch (type) { +// case VIEW -> 0.4; +// case REGISTER -> 0.8; +// case LIKE -> 1.0; +// }; +// } +//} \ No newline at end of file diff --git a/stats/analyzer/src/main/resources/application.yml b/stats/analyzer/src/main/resources/application.yml index 81a4d6e..a419635 100644 --- a/stats/analyzer/src/main/resources/application.yml +++ b/stats/analyzer/src/main/resources/application.yml @@ -2,24 +2,20 @@ spring: application: name: analyzer config: - import: "configserver:" + import: 'configserver:' cloud: config: discovery: + service-id: config-server enabled: true - serviceId: config-server - enabled: true fail-fast: true retry: - useRandomPolicy: true - max-interval: 6000 - flyway: - flyway: - database: postgresql - ignore-future-migrations: true - postgresql: - transactional-lock: false + use-random-policy: true + max-interval: 10000 eureka: + instance: + prefer-ip-address: true client: - serviceUrl: - defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ \ No newline at end of file + service-url: + defaultZone: http://localhost:8761/eureka + register-with-eureka: true \ No newline at end of file diff --git a/stats/analyzer/src/main/resources/schema.sql b/stats/analyzer/src/main/resources/schema.sql index 3129fa5..2abff13 100644 --- a/stats/analyzer/src/main/resources/schema.sql +++ b/stats/analyzer/src/main/resources/schema.sql @@ -1,17 +1,19 @@ -CREATE TABLE similarities ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, - event_id_a BIGINT, - event_id_b BIGINT, - score DOUBLE PRECISION, - timestamp TIMESTAMP with time zone, - CONSTRAINT pk_similarities PRIMARY KEY (id) +create TABLE IF NOT EXISTS user_action ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY, + user_id BIGINT NOT NULL, + event_id BIGINT NOT NULL, + action_type VARCHAR(20) NOT NULL, + created TIMESTAMP NOT NULL, + weight DOUBLE PRECISION, + CONSTRAINT pk_user_action PRIMARY KEY (id), + CONSTRAINT unique_user_action_user_id_event_id UNIQUE (user_id, event_id) ); -CREATE TABLE actions ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, - user_id BIGINT, - event_id BIGINT, - score DOUBLE PRECISION, - last_interaction TIMESTAMP with time zone, - CONSTRAINT pk_actions PRIMARY KEY (id) +create TABLE IF NOT EXISTS event_similarity ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY, + aevent_id BIGINT NOT NULL, + bevent_id BIGINT NOT NULL, + score DOUBLE PRECISION, + CONSTRAINT pk_event_similarity PRIMARY KEY (id), + CONSTRAINT unique_event_similarity_aevent_id_bevent_id UNIQUE (aevent_id, bevent_id) ); \ No newline at end of file diff --git a/stats/collector/pom.xml b/stats/collector/pom.xml index d251941..3a9779d 100644 --- a/stats/collector/pom.xml +++ b/stats/collector/pom.xml @@ -20,65 +20,50 @@ - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-actuator + ru.practicum + avro-schemas + 0.0.1-SNAPSHOT - - net.devh - grpc-server-spring-boot-starter + ru.practicum + proto-schemas + 0.0.1-SNAPSHOT - org.springframework.boot - spring-boot-starter-validation + spring-boot-starter - - org.springframework.boot - spring-boot-starter-aop + net.devh + grpc-server-spring-boot-starter + ${grpc-spring-boot-starter.version} - org.projectlombok lombok - provided + true - - org.springframework.cloud - spring-cloud-starter-config + org.springframework.boot + spring-boot-starter-validation - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client + spring-cloud-starter-config - - org.springframework.kafka - spring-kafka + org.springframework.retry + spring-retry - - ru.practicum - avro-schemas - ${avro-schemas.version} + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client - - ru.practicum - proto-schemas - ${proto-schemas.version} + org.springframework.boot + spring-boot-starter-actuator - @@ -86,9 +71,16 @@ org.springframework.boot spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + - \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/CollectorApplication.java b/stats/collector/src/main/java/ru/practicum/CollectorApplication.java index e424df4..e5177f4 100644 --- a/stats/collector/src/main/java/ru/practicum/CollectorApplication.java +++ b/stats/collector/src/main/java/ru/practicum/CollectorApplication.java @@ -2,8 +2,10 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; @SpringBootApplication +@EnableDiscoveryClient public class CollectorApplication { public static void main(String[] args) { SpringApplication.run(CollectorApplication.class, args); diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java new file mode 100644 index 0000000..b219a7b --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java @@ -0,0 +1,33 @@ +package ru.practicum.config; + +import lombok.Getter; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Properties; + +@Getter +@Configuration +@EnableConfigurationProperties({KafkaConfigProperties.class}) +public class KafkaConfig { + private final KafkaConfigProperties kafkaProperties; + + public KafkaConfig(KafkaConfigProperties properties) { + this.kafkaProperties = properties; + } + + @Bean + public Producer producer() { + Properties properties = new Properties(); + properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); + properties.put(ProducerConfig.CLIENT_ID_CONFIG, kafkaProperties.getClientIdConfig()); + properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducerKeySerializer()); + properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducerValueSerializer()); + return new KafkaProducer<>(properties); + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java b/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java new file mode 100644 index 0000000..c57c0e2 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java @@ -0,0 +1,16 @@ +package ru.practicum.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Getter +@Setter +@ConfigurationProperties(prefix = "kafka") +public class KafkaConfigProperties { + private String bootstrapServers; + private String clientIdConfig; + private String producerKeySerializer; + private String producerValueSerializer; + private String userActionTopic; +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java deleted file mode 100644 index cd28f2b..0000000 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaProducerConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -//package ru.practicum.config; -// -//import lombok.Getter; -//import lombok.RequiredArgsConstructor; -//import lombok.Setter; -//import org.apache.kafka.clients.producer.ProducerConfig; -//import org.springframework.context.annotation.Bean; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.kafka.core.DefaultKafkaProducerFactory; -//import org.springframework.kafka.core.KafkaTemplate; -//import org.springframework.kafka.core.ProducerFactory; -//import ru.practicum.ewm.stats.avro.UserActionAvro; -// -//import java.util.Map; -// -//@Getter -//@Setter -//@Configuration -//@RequiredArgsConstructor -//public class KafkaProducerConfig { -// -// private final KafkaProperties kafkaProperties; -// -// @Bean -// public ProducerFactory kafkaProducer() { -// Map props = Map.of( -// ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers(), -// ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getKeySerializer(), -// ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaProperties.getProducer().getValueSerializer() -// ); -// return new DefaultKafkaProducerFactory<>(props); -// } -// -// @Bean -// public KafkaTemplate kafkaTemplate() { -// return new KafkaTemplate<>(kafkaProducer()); -// } -//} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java b/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java deleted file mode 100644 index 2fad01a..0000000 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaProperties.java +++ /dev/null @@ -1,29 +0,0 @@ -//package ru.practicum.config; -// -//import lombok.AccessLevel; -//import lombok.Data; -//import lombok.experimental.FieldDefaults; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.context.properties.ConfigurationProperties; -//import org.springframework.stereotype.Component; -// -//@Data -//@Component -//@ConfigurationProperties("spring.kafka") -//@FieldDefaults(level = AccessLevel.PRIVATE) -//public class KafkaProperties { -// -// String bootstrapServers; -// -// Producer producer = new Producer(); -// -// @Value("${collector.kafka.topic}") -// String userActionsTopic; -// -// @Data -// @FieldDefaults(level = AccessLevel.PRIVATE) -// public static class Producer { -// String keySerializer; -// String valueSerializer; -// } -//} diff --git a/stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java b/stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java deleted file mode 100644 index 60016fc..0000000 --- a/stats/collector/src/main/java/ru/practicum/config/UserActionProducer.java +++ /dev/null @@ -1,11 +0,0 @@ -package ru.practicum.config; - -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.kafka.clients.producer.Producer; - -public interface UserActionProducer { - - Producer getProducer(); - - void stop(); -} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java b/stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java deleted file mode 100644 index 269bb62..0000000 --- a/stats/collector/src/main/java/ru/practicum/config/UserActionProducerConfiguration.java +++ /dev/null @@ -1,70 +0,0 @@ -package ru.practicum.config; - -import jakarta.annotation.PreDestroy; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.Properties; - -@Slf4j -@RequiredArgsConstructor -@Configuration -public class UserActionProducerConfiguration { - @Value("${spring.kafka.producer.bootstrap-servers}") - private String bootstrapServers; - @Value("${spring.kafka.producer.key-serializer}") - private String keySerializer; - @Value("${spring.kafka.producer.value-serializer}") - private String valueSerializer; - - @Bean - UserActionProducer getClient() { - return new UserActionProducer() { - - private Producer producer; - - @Override - public Producer getProducer() { - if (producer == null) { - log.info(" Producer пустой, начинает создавтаь новый"); - initProducer(); - } - log.info("Возвращаем готовый продьюсер = {}", producer); - return producer; - } - - private void initProducer() { - log.info("Начало инициализации продьюсера"); - Properties config = new Properties(); - log.info("BOOTSTRAP_SERVERS_CONFIG: {}",bootstrapServers); - config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - bootstrapServers); - log.info("KEY_SERIALIZER_CLASS_CONFIG: {}",keySerializer); - config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - keySerializer); - log.info("VALUE_SERIALIZER_CLASS_CONFIG: {}",valueSerializer); - config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - valueSerializer); - log.info("Подготовили конфиг для продьюсера = {}", config.toString()); - - producer = new KafkaProducer<>(config); - log.info("Закончили инициализацию продьюсера = {}", producer.toString()); - } - - @PreDestroy - @Override - public void stop() { - if (producer != null) { - producer.close(); - } - } - }; - } -} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java b/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java deleted file mode 100644 index 9e2cf78..0000000 --- a/stats/collector/src/main/java/ru/practicum/handler/ActionsHandlers.java +++ /dev/null @@ -1,8 +0,0 @@ -package ru.practicum.handler; - - -import ru.practicum.grpc.stats.action.UserActionMessage; - -public interface ActionsHandlers { - void handle(UserActionMessage.UserActionRequest userActionProto); -} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java b/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java deleted file mode 100644 index 1fb2d31..0000000 --- a/stats/collector/src/main/java/ru/practicum/handler/UserActionHandler.java +++ /dev/null @@ -1,60 +0,0 @@ -package ru.practicum.handler; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import ru.practicum.ewm.stats.avro.ActionTypeAvro; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.grpc.stats.action.UserActionMessage; -import ru.practicum.service.KafkaMessageProducer; - -import java.time.Instant; - -@Slf4j -@Component -public class UserActionHandler implements ActionsHandlers { - KafkaMessageProducer kafkaMessageProducer; - - public UserActionHandler(KafkaMessageProducer kafkaMessageProducer) { - this.kafkaMessageProducer = kafkaMessageProducer; - } - - @Override - public void handle(UserActionMessage.UserActionRequest userActionProto) { - log.info("Обработчик UserActionHandler начал работать"); - - log.info("На вход:{}", userActionProto.toString()); - UserActionAvro userActionAvro = new UserActionAvro(); - - userActionAvro.setUserId(userActionProto.getUserId()); - log.info("Установили userId={}", userActionAvro.getUserId()); - - userActionAvro.setActionType(getActionType(userActionProto.getActionType())); - log.info("Установили setActionType={}", userActionAvro.getActionType()); - - userActionAvro.setEventId(userActionProto.getEventId()); - log.info("Установили EventId={}", userActionAvro.getEventId()); - - userActionAvro.setTimestamp(Instant.ofEpochSecond(userActionProto.getTimestamp().getSeconds(), - userActionProto.getTimestamp().getNanos())); - log.info("Установили timestamp={}", userActionAvro.getTimestamp()); - - - log.info("Смапили действие пользователя в AVRO {}", userActionAvro.toString()); - kafkaMessageProducer.sendUserAction(userActionAvro); - - - } - - private ActionTypeAvro getActionType(UserActionMessage.ActionTypeProto actionTypeProto) { - if (actionTypeProto.equals(UserActionMessage.ActionTypeProto.ACTION_LIKE)) { - return ActionTypeAvro.LIKE; - } - if (actionTypeProto.equals(UserActionMessage.ActionTypeProto.ACTION_REGISTER)) { - return ActionTypeAvro.REGISTER; - } - if (actionTypeProto.equals(UserActionMessage.ActionTypeProto.ACTION_VIEW)) { - return ActionTypeAvro.VIEW; - } - return null; - } -} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java index c80a12e..6557a69 100644 --- a/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java +++ b/stats/collector/src/main/java/ru/practicum/mapper/UserActionMapper.java @@ -2,29 +2,46 @@ import ru.practicum.ewm.stats.avro.ActionTypeAvro; import ru.practicum.ewm.stats.avro.UserActionAvro; -import ru.practicum.grpc.stats.action.UserActionMessage; +import ru.practicum.grpc.stat.action.ActionTypeProto; +import ru.practicum.grpc.stat.action.UserActionProto; +import ru.practicum.model.ActionType; +import ru.practicum.model.UserAction; import java.time.Instant; public class UserActionMapper { + public static UserActionAvro toUserActionAvro(UserAction userAction) { + return UserActionAvro.newBuilder() + .setUserId(userAction.getUserId()) + .setEventId(userAction.getEventId()) + .setTimestamp(userAction.getTimestamp()) + .setActionType(toActionTypeAvro(userAction.getActionType())) + .build(); - public static UserActionAvro toAvro(UserActionMessage.UserActionRequest userActionProto) { - long timestampMillis = userActionProto.getTimestamp().getSeconds() * 1000 - + userActionProto.getTimestamp().getNanos() / 1_000_000; + } - return UserActionAvro.newBuilder() - .setUserId(userActionProto.getUserId()) - .setEventId(userActionProto.getEventId()) - .setActionType(toAvroActionType(userActionProto.getActionType())) - .setTimestamp(Instant.ofEpochSecond(timestampMillis)) + public static ActionTypeAvro toActionTypeAvro(ActionType actionType) { + return ActionTypeAvro.valueOf(actionType.name()); + } + + + public static UserAction map(UserActionProto userActionProto) { + return UserAction.builder() + .userId(userActionProto.getUserId()) + .eventId(userActionProto.getEventId()) + .actionType(toActionType(userActionProto.getActionType())) + .timestamp(Instant.ofEpochSecond(userActionProto.getTimestamp().getSeconds(), + userActionProto.getTimestamp().getNanos())) .build(); } - private static ActionTypeAvro toAvroActionType(UserActionMessage.ActionTypeProto protoType) { - return switch (protoType) { - case ACTION_REGISTER -> ActionTypeAvro.REGISTER; - case ACTION_LIKE -> ActionTypeAvro.LIKE; - default -> ActionTypeAvro.VIEW; + public static ActionType toActionType(ActionTypeProto actionTypeProto) { + return switch (actionTypeProto) { + case ACTION_VIEW -> ActionType.VIEW; + case ACTION_REGISTER -> ActionType.REGISTER; + case ACTION_LIKE -> ActionType.LIKE; + default -> null; }; } -} + +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/model/ActionType.java b/stats/collector/src/main/java/ru/practicum/model/ActionType.java new file mode 100644 index 0000000..a2d84fa --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/model/ActionType.java @@ -0,0 +1,7 @@ +package ru.practicum.model; + +public enum ActionType { + VIEW, + REGISTER, + LIKE +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/model/UserAction.java b/stats/collector/src/main/java/ru/practicum/model/UserAction.java new file mode 100644 index 0000000..993069b --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/model/UserAction.java @@ -0,0 +1,22 @@ +package ru.practicum.model; + +import jakarta.validation.constraints.NotNull; +import lombok.*; +import lombok.experimental.FieldDefaults; + +import java.time.Instant; + +@Builder +@Getter +@Setter +@ToString +@FieldDefaults(level = AccessLevel.PRIVATE) +public class UserAction { + @NotNull + Long userId; + @NotNull + Long eventId; + @NotNull + ActionType actionType; + Instant timestamp = Instant.now(); +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/ActionService.java b/stats/collector/src/main/java/ru/practicum/service/ActionService.java new file mode 100644 index 0000000..ab5b8d6 --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/service/ActionService.java @@ -0,0 +1,9 @@ +package ru.practicum.service; + + +import ru.practicum.model.UserAction; + +public interface ActionService { + + void collectUserAction(UserAction userAction); +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java b/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java new file mode 100644 index 0000000..efc082d --- /dev/null +++ b/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java @@ -0,0 +1,62 @@ +package ru.practicum.service; + +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.stereotype.Service; +import ru.practicum.mapper.UserActionMapper; +import ru.practicum.model.UserAction; +import ru.practicum.config.KafkaConfig; +import ru.practicum.ewm.stats.avro.UserActionAvro; + +import java.util.Objects; + +@Service +@RequiredArgsConstructor +@Slf4j +public class ActionServiceImpl implements ActionService { + private final Producer producer; + private final KafkaConfig kafkaConfig; + + @Override + public void collectUserAction(UserAction userAction) { + Objects.requireNonNull(userAction, "UserAction cannot be null"); + + String topic = kafkaConfig.getKafkaProperties().getUserActionTopic(); + Objects.requireNonNull(topic, "Kafka topic is not configured!"); + + log.info("Sending UserAction to Kafka. Topic: {}, UserID: {}, EventID: {}", + topic, userAction.getUserId(), userAction.getEventId()); + + UserActionAvro avroRecord = UserActionMapper.toUserActionAvro(userAction); + send(topic, userAction.getUserId(), userAction.getTimestamp().toEpochMilli(), avroRecord); + } + + private void send(String topic, Long key, Long timestamp, SpecificRecordBase specificRecordBase) { + ProducerRecord rec = new ProducerRecord<>( + topic, + null, + timestamp, + key, + specificRecordBase); + producer.send(rec, (metadata, exception) -> { + if (exception != null) { + log.error("Kafka: сообщение НЕ ОТПРАВЛЕНО, topic: {}", topic, exception); + } else { + log.info("Kafka: сообщение УСПЕШНО отправлено, topic: {}, partition: {}, offset: {}", + metadata.topic(), metadata.partition(), metadata.offset()); + } + }); + } + + @PreDestroy + private void close() { + if (producer != null) { + producer.flush(); + producer.close(); + } + } +} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/CollectorController.java b/stats/collector/src/main/java/ru/practicum/service/CollectorController.java index b947f9b..f1a8bda 100644 --- a/stats/collector/src/main/java/ru/practicum/service/CollectorController.java +++ b/stats/collector/src/main/java/ru/practicum/service/CollectorController.java @@ -1,39 +1,26 @@ package ru.practicum.service; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; +import com.google.protobuf.Empty; import io.grpc.stub.StreamObserver; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.devh.boot.grpc.server.service.GrpcService; -import ru.practicum.grpc.stats.action.UserActionControllerGrpc; -import ru.practicum.grpc.stats.action.UserActionMessage; -import ru.practicum.handler.ActionsHandlers; +import ru.practicum.mapper.UserActionMapper; +import ru.practicum.grpc.stat.action.UserActionProto; +import ru.practicum.grpc.stat.collector.UserActionControllerGrpc; -@Slf4j @GrpcService +@Slf4j @RequiredArgsConstructor public class CollectorController extends UserActionControllerGrpc.UserActionControllerImplBase { - private final ActionsHandlers actionHandler; + private final ActionService actionService; @Override - public void collectUserAction(UserActionMessage.UserActionRequest request, StreamObserver responseObserver) { - try { - actionHandler.handle(request); - responseObserver.onNext(UserActionMessage.UserActionResponse.newBuilder().getDefaultInstanceForType()); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("IllegalArgumentException collectUserAction: {}", e.getMessage(), e); - responseObserver.onError( - new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e)) - ); - } catch (Exception e) { - log.error("error collectUserAction: {}", e.getMessage(), e); - responseObserver.onError( - new StatusRuntimeException(Status.UNKNOWN.withDescription("error").withCause(e)) - ); - } - } - + public void collectUserAction(UserActionProto request, StreamObserver responseObserver) { + log.info("ActionController call collectUserAction for request = {}", request); + actionService.collectUserAction(UserActionMapper.map(request)); + responseObserver.onNext(Empty.getDefaultInstance()); + responseObserver.onCompleted(); + } } \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java b/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java deleted file mode 100644 index 3422794..0000000 --- a/stats/collector/src/main/java/ru/practicum/service/KafkaMessageProducer.java +++ /dev/null @@ -1,32 +0,0 @@ -package ru.practicum.service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.springframework.beans.factory.annotation.Value; -import ru.practicum.config.UserActionProducer; -import ru.practicum.ewm.stats.avro.UserActionAvro; -import org.springframework.stereotype.Component; - -@Component -@Slf4j -@RequiredArgsConstructor -public class KafkaMessageProducer implements MessageProducer { - - @Value("${spring.kafka.topics.actions-topic}") - private String actionsTopic; - private final UserActionProducer eventClient; - - - @Override - public void sendUserAction(UserActionAvro userAction) { - log.info("Готовим сообщение UserActionAvro к отправке: {}", getClass()); - log.info("Кафка топик = {}", actionsTopic); - log.info("eventClient = {}", eventClient.getProducer()); - eventClient.getProducer().send(new ProducerRecord<>(actionsTopic, - userAction)); - log.info("Топик: {}", actionsTopic); - log.info("Обработка UserActionAvro завершена, в KAFKA ушло: {}", userAction); - } - -} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/service/MessageProducer.java b/stats/collector/src/main/java/ru/practicum/service/MessageProducer.java deleted file mode 100644 index e259c37..0000000 --- a/stats/collector/src/main/java/ru/practicum/service/MessageProducer.java +++ /dev/null @@ -1,7 +0,0 @@ -package ru.practicum.service; - -import ru.practicum.ewm.stats.avro.UserActionAvro; - -public interface MessageProducer { - void sendUserAction(UserActionAvro userActionAvro); -} \ No newline at end of file diff --git a/stats/collector/src/main/resources/application.yml b/stats/collector/src/main/resources/application.yml index c3b5a2b..8d4e45f 100644 --- a/stats/collector/src/main/resources/application.yml +++ b/stats/collector/src/main/resources/application.yml @@ -2,18 +2,22 @@ spring: application: name: collector config: - import: "configserver:" + import: 'configserver:' cloud: config: discovery: + service-id: config-server enabled: true - serviceId: config-server - enabled: true fail-fast: true retry: - useRandomPolicy: true - max-interval: 6000 + use-random-policy: true + max-interval: 10000 eureka: + instance: + hostname: localhost + prefer-ip-address: true + ip-address: 127.0.0.1 client: - serviceUrl: - defaultZone: http://${eureka.instance.hostname:localhost}:${eureka.instance.port:8761}/eureka/ \ No newline at end of file + service-url: + defaultZone: http://localhost:8761/eureka + register-with-eureka: true \ No newline at end of file diff --git a/stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl b/stats/serialization/avro-schemas/src/main/avro/ru/practicum/ewm/stats/avro/EventSimilarityProtocol.avdl similarity index 100% rename from stats/serialization/avro-schemas/src/main/avro/EventSimilarityProtocol.avdl rename to stats/serialization/avro-schemas/src/main/avro/ru/practicum/ewm/stats/avro/EventSimilarityProtocol.avdl diff --git a/stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl b/stats/serialization/avro-schemas/src/main/avro/ru/practicum/ewm/stats/avro/UserActionProtocol.avdl similarity index 100% rename from stats/serialization/avro-schemas/src/main/avro/UserActionAvro.avdl rename to stats/serialization/avro-schemas/src/main/avro/ru/practicum/ewm/stats/avro/UserActionProtocol.avdl diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto deleted file mode 100644 index b5c0e1a..0000000 --- a/stats/serialization/proto-schemas/src/main/protobuf/stats/message/recommendation_message.proto +++ /dev/null @@ -1,24 +0,0 @@ -syntax = "proto3"; -package stats.message; - -option java_package = "ru.practicum.grpc.stats.recommendation"; - -message UserPredictionsRequestProto { - int64 user_id = 1; - int32 max_results = 2; -} - -message SimilarEventsRequestProto { - int64 event_id = 1; - int64 user_id = 2; - int32 max_results = 3; -} - -message InteractionsCountRequestProto { - repeated int64 event_id = 1; -} - -message RecommendedEventProto { - int64 event_id = 1; - double score = 2; -} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/messages/recommendation_request.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/messages/recommendation_request.proto new file mode 100644 index 0000000..f86bb52 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/messages/recommendation_request.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package stats.messages.request; + +option java_multiple_files = true; +option java_package = "ru.practicum.grpc.stat.request"; + +message UserPredictionsRequestProto{ + int64 user_id = 1; + int64 max_results = 2; +} + +message SimilarEventsRequestProto{ + int64 event_id = 1; + int64 user_id = 2; + int64 max_results = 3; +} + +message InteractionsCountRequestProto{ + repeated int64 event_id = 1; +} + +message RecommendedEventProto{ + int64 event_id = 1; + double score = 2; +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/message/user_action_message.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/messages/user_action.proto similarity index 56% rename from stats/serialization/proto-schemas/src/main/protobuf/stats/message/user_action_message.proto rename to stats/serialization/proto-schemas/src/main/protobuf/stats/messages/user_action.proto index f35e7ea..db7cdaf 100644 --- a/stats/serialization/proto-schemas/src/main/protobuf/stats/message/user_action_message.proto +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/messages/user_action.proto @@ -1,23 +1,20 @@ syntax = "proto3"; -package stats.service.collector; -option java_package = "ru.practicum.grpc.stats.action"; +package stats.messages.action; +option java_multiple_files = true; +option java_package = "ru.practicum.grpc.stat.action"; import "google/protobuf/timestamp.proto"; -enum ActionTypeProto { - ACTION_VIEW = 0; - ACTION_REGISTER = 1; - ACTION_LIKE = 2; -} - -message UserActionRequest { +message UserActionProto{ int64 user_id = 1; int64 event_id = 2; ActionTypeProto action_type = 3; google.protobuf.Timestamp timestamp = 4; } -message UserActionResponse { - bool success = 1; +enum ActionTypeProto{ + ACTION_VIEW = 0; + ACTION_REGISTER = 1; + ACTION_LIKE = 2; } \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_controller.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_controller.proto new file mode 100644 index 0000000..04abaf6 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_controller.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package stats.service.dashboard; + +option java_multiple_files = true; +option java_package = "ru.practicum.grpc.stat.dashboard"; +import "stats/messages/recommendation_request.proto"; + +service RecommendationsController{ + rpc GetRecommendationsForUser(stats.messages.request.UserPredictionsRequestProto) + returns (stream stats.messages.request.RecommendedEventProto); + + rpc GetSimilarEvents(stats.messages.request.SimilarEventsRequestProto) + returns (stream stats.messages.request.RecommendedEventProto); + + rpc GetInteractionsCount(stats.messages.request.InteractionsCountRequestProto) + returns (stream stats.messages.request.RecommendedEventProto); +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto deleted file mode 100644 index f41b177..0000000 --- a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/recommendations_service.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; -package stats.service.dashboard; - -option java_package = "ru.practicum.grpc.stats.recommendation"; - -import "stats/message/recommendation_message.proto"; - -service RecommendationsController { - rpc GetRecommendationsForUser(message.UserPredictionsRequestProto) returns (stream message.RecommendedEventProto); - rpc GetSimilarEvents(message.SimilarEventsRequestProto) returns (stream message.RecommendedEventProto); - rpc GetInteractionsCount(message.InteractionsCountRequestProto) returns (stream stats.message.RecommendedEventProto); -} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_controller.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_controller.proto new file mode 100644 index 0000000..8c81016 --- /dev/null +++ b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_controller.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package stats.service.collector; + +option java_package = "ru.practicum.grpc.stat.collector"; + +import "google/protobuf/empty.proto"; +import "stats/messages/user_action.proto"; + +service UserActionController{ + rpc CollectUserAction(stats.messages.action.UserActionProto) returns (google.protobuf.Empty); +} \ No newline at end of file diff --git a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto b/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto deleted file mode 100644 index 22dd3a9..0000000 --- a/stats/serialization/proto-schemas/src/main/protobuf/stats/service/user_action_service.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; -package stats.service.collector; - -option java_package = "ru.practicum.grpc.stats.action"; - -import "stats/message/user_action_message.proto"; -import "google/protobuf/empty.proto"; - -service UserActionController { - rpc CollectUserAction(UserActionRequest) returns (UserActionResponse); -} \ No newline at end of file diff --git a/stats/stats-client/pom.xml b/stats/stats-client/pom.xml index c53c1d0..1d28091 100644 --- a/stats/stats-client/pom.xml +++ b/stats/stats-client/pom.xml @@ -20,43 +20,45 @@ - - org.springframework.boot - spring-boot-starter-web + ru.practicum + proto-schemas + 0.0.1-SNAPSHOT + compile - - org.springframework.boot - spring-boot-starter-actuator + io.grpc + grpc-stub - net.devh grpc-client-spring-boot-starter - ${grpc.spring.boot.starter.version} + ${grpc-spring-boot-starter.version} - - io.grpc - grpc-stub + org.springframework.boot + spring-boot-starter-actuator - org.springframework.boot - spring-boot-starter-aop + spring-boot-starter-web - org.projectlombok lombok provided - - ru.practicum - proto-schemas - 0.0.1-SNAPSHOT + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + org.springframework.cloud + spring-cloud-commons + + + org.springframework.retry + spring-retry diff --git a/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java b/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java deleted file mode 100644 index a21a23d..0000000 --- a/stats/stats-client/src/main/java/ru/practicum/AnalyzerClient.java +++ /dev/null @@ -1,78 +0,0 @@ -package ru.practicum; - -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.client.inject.GrpcClient; -import org.springframework.stereotype.Service; -import ru.practicum.grpc.stats.recommendation.RecommendationMessage; -import ru.practicum.grpc.stats.recommendation.RecommendationsControllerGrpc; - -import java.util.Iterator; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -@Slf4j -@Service -public class AnalyzerClient { - - @GrpcClient("analyzer") - private RecommendationsControllerGrpc.RecommendationsControllerBlockingStub analyzerStub; - - public Stream getSimilarEvents( - long eventId, long userId, int maxResults) { - try { - log.info("Fetching similar events: eventId={}, userId={}, maxResults={}", eventId, userId, maxResults); - RecommendationMessage.SimilarEventsRequestProto requestProto = - RecommendationMessage.SimilarEventsRequestProto.newBuilder() - .setEventId(eventId) - .setUserId(userId) - .setMaxResults(maxResults) - .build(); - Iterator iterator = analyzerStub.getSimilarEvents(requestProto); - return toStream(iterator); - } catch (Exception e) { - log.error("Error occurred while fetching similar events: eventId={}, userId={}, maxResults={}", - eventId, userId, maxResults); - return Stream.empty(); - } - } - - public Stream getRecommendationsForUser(long userId, int maxResults) { - try { - log.info("Fetching recommendations for user : userId={}, maxResults={}", userId, maxResults); - RecommendationMessage.UserPredictionsRequestProto requestProto = - RecommendationMessage.UserPredictionsRequestProto.newBuilder() - .setUserId(userId) - .setMaxResults(maxResults) - .build(); - Iterator iterator = analyzerStub.getRecommendationsForUser(requestProto); - return toStream(iterator); - } catch (Exception e) { - log.error("Error occurred while fetching recommendations for user : userId={}, maxResults={}", userId, maxResults); - return Stream.empty(); - } - } - - public Stream getInteractionsCount(Iterable eventIds) { - try { - log.info("Fetching interactions count for events"); - RecommendationMessage.InteractionsCountRequestProto.Builder builder = - RecommendationMessage.InteractionsCountRequestProto.newBuilder(); - eventIds.forEach(builder::addEventId); - RecommendationMessage.InteractionsCountRequestProto requestProto = builder.build(); - Iterator iterator = analyzerStub.getInteractionsCount(requestProto); - return toStream(iterator); - } catch (Exception e) { - log.error("Error occurred while fetching interactions count", e); - return Stream.empty(); - } - } - - private Stream toStream(Iterator iterator) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), - false - ); - } -} diff --git a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java b/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java deleted file mode 100644 index 4c66fed..0000000 --- a/stats/stats-client/src/main/java/ru/practicum/CollectorClient.java +++ /dev/null @@ -1,53 +0,0 @@ -package ru.practicum; - -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.client.inject.GrpcClient; -import org.springframework.stereotype.Service; -import ru.practicum.grpc.stats.action.UserActionControllerGrpc; -import ru.practicum.grpc.stats.action.UserActionMessage; - -import java.time.Instant; - -@Slf4j -@Service -public class CollectorClient { - - @GrpcClient("collector") - private UserActionControllerGrpc.UserActionControllerBlockingStub collectorStub; - - public void sendUserAction(long userId, long eventId, UserActionMessage.ActionTypeProto actionTypeProto) { - try { - log.info("Sending user action: userId={}, eventId={}, actionType={}", userId, eventId, actionTypeProto); - long secondes = Instant.now().getEpochSecond(); - int nanos = Instant.now().getNano(); - - UserActionMessage.UserActionRequest userActionProto = UserActionMessage.UserActionRequest.newBuilder() - .setUserId(userId) - .setEventId(eventId) - .setActionType(actionTypeProto) - .setTimestamp( - com.google.protobuf.Timestamp.newBuilder() - .setSeconds(secondes) - .setNanos(nanos) - ) - .build(); - UserActionMessage.UserActionResponse response = collectorStub.collectUserAction(userActionProto); - log.info("sendUserAction -> Collector answered"); - } catch (Exception e) { - log.error("Ошибка при отправке действия пользователя: userId={}, eventId={}, actionType={}", - userId, eventId, actionTypeProto, e); - } - } - - public void sendEventView(long userId, long eventId) { - sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_VIEW); - } - - public void sendEventLike(long userId, long eventId) { - sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_LIKE); - } - - public void sendEventRegistration(long userId, long eventId) { - sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_REGISTER); - } -} diff --git a/stats/stats-client/src/main/java/ru/practicum/StatClient.java b/stats/stats-client/src/main/java/ru/practicum/StatClient.java deleted file mode 100644 index c95620a..0000000 --- a/stats/stats-client/src/main/java/ru/practicum/StatClient.java +++ /dev/null @@ -1,24 +0,0 @@ -//package ru.practicum; -// -//import org.springframework.cloud.openfeign.FeignClient; -//import org.springframework.web.bind.annotation.GetMapping; -//import org.springframework.web.bind.annotation.PostMapping; -//import org.springframework.web.bind.annotation.RequestBody; -//import org.springframework.web.bind.annotation.RequestParam; -//import ru.practicum.dto.EndpointHitDto; -//import ru.practicum.dto.ViewStatsDto; -// -//import java.util.List; -// -//@FeignClient(name = "stats-server") -//public interface StatClient { -// -// @PostMapping("/hit") -// void saveHit(@RequestBody EndpointHitDto hitDto); -// -// @GetMapping("stats") -// List getStats(@RequestParam(defaultValue = "") String start, -// @RequestParam(defaultValue = "") String end, -// @RequestParam(defaultValue = "") List uris, -// @RequestParam(defaultValue = "false") Boolean unique); -//} \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java b/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java deleted file mode 100644 index 2ced547..0000000 --- a/stats/stats-client/src/main/java/ru/practicum/StatServiceClient.java +++ /dev/null @@ -1,34 +0,0 @@ -//package ru.practicum; -// -//import lombok.RequiredArgsConstructor; -//import lombok.extern.slf4j.Slf4j; -//import org.springframework.stereotype.Component; -//import ru.practicum.dto.EndpointHitDto; -//import ru.practicum.dto.ViewStatsDto; -// -//import java.util.Collections; -//import java.util.List; -// -//@Slf4j -//@Component -//@RequiredArgsConstructor -//public class StatServiceClient { -// -// private final StatClient statClient; -// -// public void saveHit(EndpointHitDto dto) { -// statClient.saveHit(dto); -// } -// -// public List getStats(String start, -// String end, -// List uris, -// Boolean unique) { -// try { -// return statClient.getStats(start, end, uris, unique); -// } catch (Exception e) { -// log.warn("Failed to get stats: {}", e.getMessage()); -// } -// return Collections.emptyList(); -// } -//} \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/stats/client/StatClient.java b/stats/stats-client/src/main/java/ru/practicum/stats/client/StatClient.java new file mode 100644 index 0000000..a6e6c35 --- /dev/null +++ b/stats/stats-client/src/main/java/ru/practicum/stats/client/StatClient.java @@ -0,0 +1,19 @@ +package ru.practicum.stats.client; + +import ru.practicum.grpc.stat.action.ActionTypeProto; +import ru.practicum.grpc.stat.request.RecommendedEventProto; + +import java.time.Instant; +import java.util.List; +import java.util.stream.Stream; + +public interface StatClient { + + void registerUserAction(long eventId, long userId, ActionTypeProto actionTypeProto, Instant instant); + + Stream getSimilarEvents(long eventId, long userId, int maxResults); + + Stream getRecommendationsFor(long userId, int maxResults); + + Stream getInteractionsCount(List eventIds); +} \ No newline at end of file diff --git a/stats/stats-client/src/main/java/ru/practicum/stats/client/StatClientImpl.java b/stats/stats-client/src/main/java/ru/practicum/stats/client/StatClientImpl.java new file mode 100644 index 0000000..a7bf2c0 --- /dev/null +++ b/stats/stats-client/src/main/java/ru/practicum/stats/client/StatClientImpl.java @@ -0,0 +1,102 @@ +package ru.practicum.stats.client; + +import com.google.protobuf.Timestamp; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.devh.boot.grpc.client.inject.GrpcClient; +import org.springframework.stereotype.Component; +import ru.practicum.grpc.stat.action.ActionTypeProto; +import ru.practicum.grpc.stat.action.UserActionProto; +import ru.practicum.grpc.stat.collector.UserActionControllerGrpc; +import ru.practicum.grpc.stat.dashboard.RecommendationsControllerGrpc; +import ru.practicum.grpc.stat.request.InteractionsCountRequestProto; +import ru.practicum.grpc.stat.request.RecommendedEventProto; +import ru.practicum.grpc.stat.request.SimilarEventsRequestProto; +import ru.practicum.grpc.stat.request.UserPredictionsRequestProto; + +import java.time.Instant; +import java.util.Iterator; +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +@Slf4j +@Component +@RequiredArgsConstructor +public class StatClientImpl implements StatClient { + + @GrpcClient("collector") + private UserActionControllerGrpc.UserActionControllerBlockingStub userClient; + + @GrpcClient("analyzer") + private RecommendationsControllerGrpc.RecommendationsControllerBlockingStub analyzerClient; + + @Override + public void registerUserAction(long eventId, long userId, ActionTypeProto actionTypeProto, Instant instant) { + log.info("statClientImpl registerUserAction for eventId = {}, userId = {}, actionType = {}, time = {}", + eventId, userId, actionTypeProto, instant); + Timestamp timestamp = Timestamp.newBuilder() + .setNanos(instant.getNano()) + .setSeconds(instant.getEpochSecond()) + .build(); + + UserActionProto request = UserActionProto.newBuilder() + .setUserId(userId) + .setEventId(eventId) + .setActionType(actionTypeProto) + .setTimestamp(timestamp) + .build(); + + log.info("statClientImpl registerUserAction request = {}", request); + userClient.collectUserAction(request); + } + + @Override + public Stream getSimilarEvents(long eventId, long userId, int maxResults) { + log.info("statsClientImpl getSimilarEvents for eventId = {}, userId = {}, maxResults = {}", + eventId, userId, maxResults); + SimilarEventsRequestProto request = SimilarEventsRequestProto.newBuilder() + .setEventId(eventId) + .setUserId(userId) + .setMaxResults(maxResults) + .build(); + + Iterator iterator = analyzerClient.getSimilarEvents(request); + + return toStream(iterator); + } + + @Override + public Stream getRecommendationsFor(long userId, int maxResults) { + log.info("statsClientImpl getRecommendationsForUser for userId = {}, maxResults = {}", userId, maxResults); + UserPredictionsRequestProto request = UserPredictionsRequestProto.newBuilder() + .setUserId(userId) + .setMaxResults(maxResults) + .build(); + + Iterator iterator = analyzerClient.getRecommendationsForUser(request); + return toStream(iterator); + } + + @Override + public Stream getInteractionsCount(List eventIds) { + log.info("statsClientImpl getInteractionsCount for event list = {}", eventIds); + + InteractionsCountRequestProto request = InteractionsCountRequestProto.newBuilder() + .addAllEventId(eventIds) + .build(); + + Iterator iterator = analyzerClient.getInteractionsCount(request); + return toStream(iterator); + } + + private Stream toStream(Iterator iterator) { + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), + false + ); + } + +} \ No newline at end of file diff --git a/stats/stats-client/src/main/resources/application.yml b/stats/stats-client/src/main/resources/application.yml deleted file mode 100644 index cb75a5d..0000000 --- a/stats/stats-client/src/main/resources/application.yml +++ /dev/null @@ -1,13 +0,0 @@ -grpc: - client: - collector: - address: 'discovery:///collector' - enableKeepAlive: true - keepAliveWithoutCalls: true - negotiationType: plaintext - - analyzer: - address: 'discovery:///analyzer' - enableKeepAlive: true - keepAliveWithoutCalls: true - negotiationType: plaintext \ No newline at end of file From a23acde3a05ea13eb82df4248b90c22548c34aab Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 02:51:46 +0300 Subject: [PATCH 14/26] feat: refactored stats and released recommendations feature --- .../java/ru/practicum/service/RequestServiceImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java index 51f1b65..fedf19e 100644 --- a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java +++ b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java @@ -12,11 +12,13 @@ import ru.practicum.enums.RequestStatus; import ru.practicum.exception.ConflictException; import ru.practicum.exception.NotFoundException; -import ru.practicum.grpc.stats.action.UserActionMessage; +import ru.practicum.grpc.stat.action.ActionTypeProto; import ru.practicum.mapper.RequestMapper; import ru.practicum.model.Request; import ru.practicum.repository.RequestRepository; +import ru.practicum.stats.client.StatClient; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -31,7 +33,7 @@ public class RequestServiceImpl implements RequestService { private final RequestMapper requestMapper; private final UserServiceClient userServiceClient; private final RequestRepository requestRepository; - private final CollectorClient collectorClient; + private final StatClient statClient; @Override public List getRequestByUserId(Long userId) { @@ -48,7 +50,7 @@ public ParticipationRequestDto createRequest(Long userId, Long eventId) { requestToEventVerification(userId, eventId); Request request = requestMapper.formUserAndEventToRequest(userId, eventId); requestRepository.save(request); - collectorClient.sendUserAction(userId, eventId, UserActionMessage.ActionTypeProto.ACTION_REGISTER); + statClient.registerUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER, Instant.now()); return requestMapper.toParticipationRequestDto(request); } From 27788ab58780f1ef02d2b1ff314c2fef64e4472c Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 03:11:19 +0300 Subject: [PATCH 15/26] feat: refactored stats and released recommendations feature --- .../resources/config/stats/collector/application.yml | 2 +- .../java/ru/practicum/service/ActionServiceImpl.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/infra/config-server/src/main/resources/config/stats/collector/application.yml b/infra/config-server/src/main/resources/config/stats/collector/application.yml index d98ad61..f2e8b0b 100644 --- a/infra/config-server/src/main/resources/config/stats/collector/application.yml +++ b/infra/config-server/src/main/resources/config/stats/collector/application.yml @@ -1,6 +1,6 @@ logging: level: - ru.yandex.practicum: debug + ru.practicum: debug root: info grpc: diff --git a/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java b/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java index efc082d..f89b086 100644 --- a/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java +++ b/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java @@ -1,11 +1,11 @@ package ru.practicum.service; import jakarta.annotation.PreDestroy; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.avro.specific.SpecificRecordBase; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import ru.practicum.mapper.UserActionMapper; import ru.practicum.model.UserAction; @@ -15,12 +15,17 @@ import java.util.Objects; @Service -@RequiredArgsConstructor @Slf4j public class ActionServiceImpl implements ActionService { private final Producer producer; private final KafkaConfig kafkaConfig; + @Autowired + public ActionServiceImpl(Producer producer, KafkaConfig kafkaConfig) { + this.producer = producer; + this.kafkaConfig = kafkaConfig; + } + @Override public void collectUserAction(UserAction userAction) { Objects.requireNonNull(userAction, "UserAction cannot be null"); From d99d909088874f35566fb06f6e59bc0a773d56a3 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 03:16:33 +0300 Subject: [PATCH 16/26] feat: refactored stats and released recommendations feature --- .../src/main/java/ru/practicum/config/KafkaConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java index b219a7b..8755de0 100644 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java @@ -5,6 +5,7 @@ import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,6 +18,7 @@ public class KafkaConfig { private final KafkaConfigProperties kafkaProperties; + @Autowired public KafkaConfig(KafkaConfigProperties properties) { this.kafkaProperties = properties; } From 27a6687b71859b12ef04c63731f8f295388018cf Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 03:40:44 +0300 Subject: [PATCH 17/26] feat: refactored stats and released recommendations feature --- pom.xml | 2 +- .../src/main/java/ru/practicum/config/KafkaConfig.java | 2 -- .../java/ru/practicum/service/ActionServiceImpl.java | 9 ++------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 8ee149f..58eecfe 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.springframework.boot spring-boot-starter-parent - 3.3.4 + 3.3.0 diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java index 8755de0..b219a7b 100644 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java @@ -5,7 +5,6 @@ import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -18,7 +17,6 @@ public class KafkaConfig { private final KafkaConfigProperties kafkaProperties; - @Autowired public KafkaConfig(KafkaConfigProperties properties) { this.kafkaProperties = properties; } diff --git a/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java b/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java index f89b086..efc082d 100644 --- a/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java +++ b/stats/collector/src/main/java/ru/practicum/service/ActionServiceImpl.java @@ -1,11 +1,11 @@ package ru.practicum.service; import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.avro.specific.SpecificRecordBase; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import ru.practicum.mapper.UserActionMapper; import ru.practicum.model.UserAction; @@ -15,17 +15,12 @@ import java.util.Objects; @Service +@RequiredArgsConstructor @Slf4j public class ActionServiceImpl implements ActionService { private final Producer producer; private final KafkaConfig kafkaConfig; - @Autowired - public ActionServiceImpl(Producer producer, KafkaConfig kafkaConfig) { - this.producer = producer; - this.kafkaConfig = kafkaConfig; - } - @Override public void collectUserAction(UserAction userAction) { Objects.requireNonNull(userAction, "UserAction cannot be null"); From 45dfa8b6f9dba47b83f430f3a2d99f6ae4a15493 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 03:53:46 +0300 Subject: [PATCH 18/26] feat: refactored stats and released recommendations feature --- .../resources/config/stats/collector/application.yml | 10 +++++----- .../main/java/ru/practicum/config/KafkaConfig.java | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/infra/config-server/src/main/resources/config/stats/collector/application.yml b/infra/config-server/src/main/resources/config/stats/collector/application.yml index f2e8b0b..1445b4e 100644 --- a/infra/config-server/src/main/resources/config/stats/collector/application.yml +++ b/infra/config-server/src/main/resources/config/stats/collector/application.yml @@ -11,8 +11,8 @@ server: port: 8888 kafka: - userActionTopic: stats.user-actions.v1 - bootstrapServers: localhost:9092 - clientIdConfig: collector-client - producerKeySerializer: org.apache.kafka.common.serialization.LongSerializer - producerValueSerializer: ru.practicum.serializer.UserActionsAvroSerializer \ No newline at end of file + user-action-topic: stats.user-actions.v1 + bootstrap-servers: localhost:9092 + client-id-config: collector-client + producer-key-serializer: org.apache.kafka.common.serialization.LongSerializer + producer-value-serializer: ru.practicum.serializer.UserActionsAvroSerializer \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java index b219a7b..bcf4c4a 100644 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaConfig.java @@ -9,6 +9,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.Objects; import java.util.Properties; @Getter @@ -23,6 +24,17 @@ public KafkaConfig(KafkaConfigProperties properties) { @Bean public Producer producer() { + // Проверка, что свойства не null + Objects.requireNonNull(kafkaProperties.getBootstrapServers(), + "kafka.bootstrap-servers не задано в конфигурации"); + Objects.requireNonNull(kafkaProperties.getClientIdConfig(), + "kafka.client-id-config не задано в конфигурации"); + Objects.requireNonNull(kafkaProperties.getProducerKeySerializer(), + "kafka.producer-key-serializer не задано в конфигурации"); + Objects.requireNonNull(kafkaProperties.getProducerValueSerializer(), + "kafka.producer-value-serializer не задано в конфигурации"); + Objects.requireNonNull(kafkaProperties.getUserActionTopic(), + "kafka.user-action-topic не задано в конфигурации"); Properties properties = new Properties(); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProperties.getBootstrapServers()); properties.put(ProducerConfig.CLIENT_ID_CONFIG, kafkaProperties.getClientIdConfig()); From b0381b257a6c07dcdfb1909fb9b4d5fd7ff7dfd1 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 04:08:06 +0300 Subject: [PATCH 19/26] feat: refactored stats and released recommendations feature --- infra/config-server/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/config-server/src/main/resources/application.yml b/infra/config-server/src/main/resources/application.yml index e9c702d..117066e 100644 --- a/infra/config-server/src/main/resources/application.yml +++ b/infra/config-server/src/main/resources/application.yml @@ -10,7 +10,7 @@ spring: searchLocations: - classpath:config/core/{application} - classpath:config/infra/{application} - - classpath:config/stat/{application} + - classpath:config/stats/{application} discovery: enabled: true From 3866185f5e3773b4c01b93312eac7a687745b726 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 04:16:32 +0300 Subject: [PATCH 20/26] feat: refactored stats and released recommendations feature --- .../src/main/java/ru/practicum/serializer/AvroSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java b/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java index 8f4754a..118cc0d 100644 --- a/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java +++ b/stats/aggregator/src/main/java/ru/practicum/serializer/AvroSerializer.java @@ -1,4 +1,4 @@ -package ru.yandex.practicum.serializer; +package ru.practicum.serializer; import lombok.extern.slf4j.Slf4j; import org.apache.avro.io.BinaryEncoder; From 983c0598f99e3db687d7d161d8eb0981d1899f2a Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 05:23:51 +0300 Subject: [PATCH 21/26] feat: refactored stats and released recommendations feature --- .../service/event/EventServiceImpl.java | 32 ++++---- docker-compose.yml | 74 ++++++++++++------- 2 files changed, 64 insertions(+), 42 deletions(-) diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index 753da6d..eda006a 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -92,7 +92,7 @@ public List getEventsForUser(Long userId, Integer from, Integer s List eventsDto = events.stream() .map(eventMapper::toEventShortDto) .toList(); - populateWithStats(eventsDto); + //populateWithStats(eventsDto); return events.stream() .map(eventMapper::toEventShortDto) .collect(Collectors.toList()); @@ -207,8 +207,8 @@ public EventFullDto updateEventByAdmin(UpdateEventAdminRequest request, Long eve } } - EventShortDto eventShortDto = eventMapper.toEventShortDto(event); - populateWithStats(List.of(eventShortDto)); + //EventShortDto eventShortDto = eventMapper.toEventShortDto(event); + //populateWithStats(List.of(eventShortDto)); return utilEventClass.toEventFullDto(eventRepository.save(event)); } @@ -305,7 +305,7 @@ public EventFullDto getEventById(Long eventId, long userId) { EventFullDto eventFullDto = utilEventClass.toEventFullDto(event); eventFullDto.setConfirmedRequests(confirmedRequests); EventShortDto eventShortDto = eventMapper.toEventShortDto(event); - populateWithStats(List.of(eventShortDto)); + //populateWithStats(List.of(eventShortDto)); return eventFullDto; @@ -541,16 +541,16 @@ public Stream getRecommendations(Long userId, int limit) { .map(eventMapper::map); } - private void populateWithStats(List eventsDto) { - if (eventsDto.isEmpty()) return; - - List eventIds = eventsDto.stream() - .map(EventShortDto::getId).toList(); - Map ratedEvents = statClient.getInteractionsCount(eventIds) - .map(eventMapper::map) - .collect(Collectors.toMap(RecommendedEventDto::getEventId, RecommendedEventDto::getScore)); - log.info("ratedEvents are: {}", ratedEvents); - eventsDto.forEach(event -> Optional.ofNullable(ratedEvents.get(event.getId())) - .ifPresent(event::setRating)); - } +// private void populateWithStats(List eventsDto) { +// if (eventsDto.isEmpty()) return; +// +// List eventIds = eventsDto.stream() +// .map(EventShortDto::getId).toList(); +// Map ratedEvents = statClient.getInteractionsCount(eventIds) +// .map(eventMapper::map) +// .collect(Collectors.toMap(RecommendedEventDto::getEventId, RecommendedEventDto::getScore)); +// log.info("ratedEvents are: {}", ratedEvents); +// eventsDto.forEach(event -> Optional.ofNullable(ratedEvents.get(event.getId())) +// .ifPresent(event::setRating)); +// } } diff --git a/docker-compose.yml b/docker-compose.yml index 93a6d0b..d5973aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -170,102 +170,124 @@ services: container_name: kafka ports: - "9092:9092" # for client connections - - "9101:9101" # JMX + - "29092:29092" restart: unless-stopped + networks: + - ewm-net environment: KAFKA_NODE_ID: 1 KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092' KAFKA_JMX_PORT: 9101 KAFKA_JMX_HOSTNAME: localhost - KAFKA_PROCESS_ROLES: 'broker, controller' + KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092' CLUSTER_ID: 'K0EA9p0yEe6MkAAAAkKsEg' + healthcheck: + test: [ "CMD", "kafka-topics", "--bootstrap-server", "localhost:9092", "--list" ] + interval: 10s + timeout: 5s + retries: 10 kafka-init-topics: image: confluentinc/confluent-local:7.4.3 container_name: kafka-init-topics depends_on: - - kafka + kafka: + condition: service_healthy + networks: + - ewm-net command: "bash -c \ - 'kafka-topics --create --topic stats.user-actions.v1 \ - --partitions 1 --replication-factor 1 --if-not-exists \ - --bootstrap-server kafka:29092 && \ - kafka-topics --create --topic stats.events-similarity.v1 \ - --partitions 1 --replication-factor 1 --if-not-exists \ - --bootstrap-server kafka:29092'" + 'kafka-topics --create --topic stats.user-actions.v1 \ + --partitions 1 --replication-factor 1 --if-not-exists \ + --bootstrap-server kafka:29092 && \ + kafka-topics --create --topic stats.events-similarity.v1 \ + --partitions 1 --replication-factor 1 --if-not-exists \ + --bootstrap-server kafka:29092'" init: true collector: build: stats/collector - container_name: ewm-collector + container_name: collector + restart: on-failure depends_on: config-server: condition: service_healthy + kafka-init-topics: + condition: service_completed_successfully + aggregator: + condition: service_healthy networks: - ewm-net environment: - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=8085 + - SERVER_PORT=8080 healthcheck: - test: "curl --fail --silent localhost:8085/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8080/actuator/health | grep UP || exit 1" timeout: 5s - interval: 15s + interval: 25s retries: 10 aggregator: build: stats/aggregator - container_name: ewm-aggregator + container_name: aggregator + restart: on-failure depends_on: config-server: condition: service_healthy + kafka-init-topics: + condition: service_completed_successfully networks: - ewm-net environment: - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=8086 + - SERVER_PORT=8080 healthcheck: - test: "curl --fail --silent localhost:8086/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8080/actuator/health | grep UP || exit 1" timeout: 5s - interval: 15s + interval: 25s retries: 10 analyzer: build: stats/analyzer - container_name: ewm-analyzer + container_name: analyzer + restart: on-failure depends_on: analyzer-db: condition: service_healthy config-server: condition: service_healthy + kafka-init-topics: + condition: service_completed_successfully networks: - ewm-net environment: - - SPRING_DATASOURCE_URL=jdbc:postgresql://analyzer-db:5432/ewm-analyzer + - SPRING_DATASOURCE_URL=jdbc:postgresql://analyzer-db:5432/analyzer - SPRING_DATASOURCE_USERNAME=root - SPRING_DATASOURCE_PASSWORD=root - EUREKA_CLIENT_SERVICEURL_DEFAULTZONE=http://discovery-server:8761/eureka/ - - SERVER_PORT=8087 + - SERVER_PORT=8080 healthcheck: - test: "curl --fail --silent localhost:8087/actuator/health | grep UP || exit 1" + test: "curl --fail --silent localhost:8080/actuator/health | grep UP || exit 1" timeout: 5s - interval: 15s + interval: 25s retries: 10 analyzer-db: image: postgres:16.1 - container_name: postgres-ewm-analyzer-db + container_name: analyzer-db + restart: on-failure environment: - - POSTGRES_PASSWORD=root + - POSTGRES_DB=analyzer - POSTGRES_USER=root - - POSTGRES_DB=ewm-analyzer + - POSTGRES_PASSWORD=root networks: - ewm-net healthcheck: test: pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER timeout: 5s interval: 10s - retries: 15 + retries: 10 networks: From a9e8b8ea22b650bc254502e00b56f608b3d74879 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 05:56:39 +0300 Subject: [PATCH 22/26] feat: refactored stats and released recommendations feature --- .../main/java/ru/practicum/service/event/EventServiceImpl.java | 2 +- .../src/main/java/ru/practicum/service/RequestServiceImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index eda006a..fc099bf 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -304,7 +304,7 @@ public EventFullDto getEventById(Long eventId, long userId) { // Создание DTO EventFullDto eventFullDto = utilEventClass.toEventFullDto(event); eventFullDto.setConfirmedRequests(confirmedRequests); - EventShortDto eventShortDto = eventMapper.toEventShortDto(event); + //EventShortDto eventShortDto = eventMapper.toEventShortDto(event); //populateWithStats(List.of(eventShortDto)); diff --git a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java index fedf19e..24408e3 100644 --- a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java +++ b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java @@ -50,7 +50,7 @@ public ParticipationRequestDto createRequest(Long userId, Long eventId) { requestToEventVerification(userId, eventId); Request request = requestMapper.formUserAndEventToRequest(userId, eventId); requestRepository.save(request); - statClient.registerUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER, Instant.now()); + //statClient.registerUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER, Instant.now()); return requestMapper.toParticipationRequestDto(request); } From 41c4b677422d97307bf48b14bd6cbcadb317fdc8 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 06:10:20 +0300 Subject: [PATCH 23/26] feat: refactored stats and released recommendations feature --- .../main/java/ru/practicum/service/event/EventServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index fc099bf..a3e0c14 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -297,7 +297,7 @@ public EventFullDto getEventById(Long eventId, long userId) { eventRepository.save(event); log.info("starting statClient.registerUserAction"); - statClient.registerUserAction(event.getId(), userId, ActionTypeProto.ACTION_VIEW, Instant.now()); + //statClient.registerUserAction(event.getId(), userId, ActionTypeProto.ACTION_VIEW, Instant.now()); // Подсчет подтвержденных запросов long confirmedRequests = requestServiceClient.countByStatusAndEventId(RequestStatus.CONFIRMED, eventId); From 5062eec02c12aaa63fdf83020b808985d1ceda40 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 06:24:30 +0300 Subject: [PATCH 24/26] fix: code cleanup --- .../service/event/EventServiceImpl.java | 86 ------------------- .../controller/PrivateRequestController.java | 2 - .../practicum/service/RequestServiceImpl.java | 2 - .../java/ru/practicum/config/AppConfig.java | 51 ----------- .../config/KafkaConfigProperties.java | 30 +++---- .../SimilarityEventProducerConfiguration.java | 47 ---------- .../practicum/service/AggregatorService.java | 3 +- .../practicum/config/ConsumerProperties.java | 15 ++-- .../config/KafkaConfigProperties.java | 14 +-- .../service/event/EventSimilarityService.java | 8 -- .../event/EventSimilarityServiceImpl.java | 39 --------- .../service/user/UserActionService.java | 7 -- .../service/user/UserActionServiceImpl.java | 56 ------------ .../config/KafkaConfigProperties.java | 13 +-- 14 files changed, 41 insertions(+), 332 deletions(-) delete mode 100644 stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java delete mode 100644 stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java delete mode 100644 stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java diff --git a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java index a3e0c14..5f25457 100644 --- a/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java +++ b/core/event-service/src/main/java/ru/practicum/service/event/EventServiceImpl.java @@ -13,8 +13,6 @@ import ru.practicum.client.UserServiceClient; import ru.practicum.dto.category.CategoryDto; import ru.practicum.dto.event.*; -import ru.practicum.dto.user.UserDto; -import ru.practicum.dto.user.UserShortDto; import ru.practicum.enums.AdminStateAction; import ru.practicum.enums.EventState; import ru.practicum.enums.RequestStatus; @@ -89,10 +87,6 @@ public EventServiceImpl(EventRepository eventRepository, @Transactional(readOnly = true) public List getEventsForUser(Long userId, Integer from, Integer size) { List events = eventRepository.findByInitiatorId(userId, PageRequest.of(from, size)); - List eventsDto = events.stream() - .map(eventMapper::toEventShortDto) - .toList(); - //populateWithStats(eventsDto); return events.stream() .map(eventMapper::toEventShortDto) .collect(Collectors.toList()); @@ -207,8 +201,6 @@ public EventFullDto updateEventByAdmin(UpdateEventAdminRequest request, Long eve } } - //EventShortDto eventShortDto = eventMapper.toEventShortDto(event); - //populateWithStats(List.of(eventShortDto)); return utilEventClass.toEventFullDto(eventRepository.save(event)); } @@ -296,81 +288,16 @@ public EventFullDto getEventById(Long eventId, long userId) { eventRepository.save(event); - log.info("starting statClient.registerUserAction"); - //statClient.registerUserAction(event.getId(), userId, ActionTypeProto.ACTION_VIEW, Instant.now()); // Подсчет подтвержденных запросов long confirmedRequests = requestServiceClient.countByStatusAndEventId(RequestStatus.CONFIRMED, eventId); // Создание DTO EventFullDto eventFullDto = utilEventClass.toEventFullDto(event); eventFullDto.setConfirmedRequests(confirmedRequests); - //EventShortDto eventShortDto = eventMapper.toEventShortDto(event); - //populateWithStats(List.of(eventShortDto)); - return eventFullDto; } -// private void saveEventsRequestToStats(String clientIp) { -// try { -// // Создание объекта для статистики -// log.info("Создание объекта для статистики"); -// EndpointHitDto hitDto = new EndpointHitDto(); -// hitDto.setApp("ewm-main-service"); -// hitDto.setUri("/events"); -// hitDto.setIp(clientIp); -// hitDto.setTimestamp(LocalDateTime.now().format(dateTimeFormatter)); -// -// // Логируем успешный запрос -// log.info("Логируем запрос в статистику: URI={}, IP={}", hitDto.getUri(), hitDto.getIp()); -// -// // Отправка статистики -// statClient.saveHit(hitDto); -// } catch (Exception e) { -// log.error("Ошибка при сохранении статистики для URI=/events, IP=" + clientIp, e); -// } -// } - -// private void saveEventRequestToStats(Event event, String clientIp) { -// try { -// EndpointHitDto hitDto = new EndpointHitDto(); -// hitDto.setApp("ewm-main-service"); -// hitDto.setUri("/events/" + event.getId()); -// hitDto.setIp(clientIp); -// hitDto.setTimestamp(LocalDateTime.now().format(dateTimeFormatter)); -// -// statClient.saveHit(hitDto); -// } catch (Exception e) { -// log.error("Ошибка при сохранении статистики для события id=" + event.getId(), e); -// } -// } - -// private long getViewsFromStats(Event event) { -// DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); -// try { -// String uri = "/events/" + event.getId(); -// // Добавляем одну секунду к началу и завершению диапазона -// String start = event.getCreatedOn().minusSeconds(1).format(formatter); -// String end = LocalDateTime.now().plusSeconds(1).format(formatter); -// -// List stats = statClient.getStats( -// start, -// end, -// List.of(uri), -// true -// ); -// -// return stats.stream() -// .filter(stat -> stat.getUri().equals(uri)) -// .mapToLong(ViewStatsDto::getHits) -// .sum(); -// } catch (Exception e) { -// log.error("Ошибка при получении статистики просмотров для события id=" + event.getId(), e); -// return 0; -// } -// } - - private void checkDateTime(LocalDateTime rangeStart, LocalDateTime rangeEnd) { if (rangeStart != null && rangeEnd != null && rangeStart.isAfter(rangeEnd)) { throw new ValidationException("start time can't be after end time", "time range is incorrect"); @@ -540,17 +467,4 @@ public Stream getRecommendations(Long userId, int limit) { return statClient.getRecommendationsFor(userId, limit) .map(eventMapper::map); } - -// private void populateWithStats(List eventsDto) { -// if (eventsDto.isEmpty()) return; -// -// List eventIds = eventsDto.stream() -// .map(EventShortDto::getId).toList(); -// Map ratedEvents = statClient.getInteractionsCount(eventIds) -// .map(eventMapper::map) -// .collect(Collectors.toMap(RecommendedEventDto::getEventId, RecommendedEventDto::getScore)); -// log.info("ratedEvents are: {}", ratedEvents); -// eventsDto.forEach(event -> Optional.ofNullable(ratedEvents.get(event.getId())) -// .ifPresent(event::setRating)); -// } } diff --git a/core/request-service/src/main/java/ru/practicum/controller/PrivateRequestController.java b/core/request-service/src/main/java/ru/practicum/controller/PrivateRequestController.java index bd7ae7f..c1ffc4c 100644 --- a/core/request-service/src/main/java/ru/practicum/controller/PrivateRequestController.java +++ b/core/request-service/src/main/java/ru/practicum/controller/PrivateRequestController.java @@ -6,7 +6,6 @@ import org.springframework.http.HttpStatus; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; -import ru.practicum.client.EventServiceClient; import ru.practicum.dto.request.EventRequestStatusUpdateRequest; import ru.practicum.dto.request.EventRequestStatusUpdateResult; import ru.practicum.dto.request.ParticipationRequestDto; @@ -27,7 +26,6 @@ public class PrivateRequestController { private static final String USERID = "user-id"; private static final String EVENTID = "event-id"; - private final EventServiceClient eventServiceClient; private final RequestService requestService; /** diff --git a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java index 24408e3..5412e3a 100644 --- a/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java +++ b/core/request-service/src/main/java/ru/practicum/service/RequestServiceImpl.java @@ -33,7 +33,6 @@ public class RequestServiceImpl implements RequestService { private final RequestMapper requestMapper; private final UserServiceClient userServiceClient; private final RequestRepository requestRepository; - private final StatClient statClient; @Override public List getRequestByUserId(Long userId) { @@ -50,7 +49,6 @@ public ParticipationRequestDto createRequest(Long userId, Long eventId) { requestToEventVerification(userId, eventId); Request request = requestMapper.formUserAndEventToRequest(userId, eventId); requestRepository.save(request); - //statClient.registerUserAction(userId, eventId, ActionTypeProto.ACTION_REGISTER, Instant.now()); return requestMapper.toParticipationRequestDto(request); } diff --git a/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java b/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java deleted file mode 100644 index 6745fdc..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/config/AppConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -//package ru.practicum.config; -// -//import lombok.Getter; -//import lombok.Setter; -//import lombok.ToString; -//import org.springframework.boot.context.properties.ConfigurationProperties; -// -//import java.time.Duration; -// -//@Getter -//@Setter -//@ToString -//@ConfigurationProperties("spring.kafka") -//public class AppConfig { -// ProducerSettings producer; -// ConsumerSettings consumer; -// TopicsSettings topics; -// -// @Setter -// @Gettera -// @ToString -// -// public static class ProducerSettings { -// private String bootstrapServers; -// private String keySerializer; -// private String valueSerializer; -// } -// -// @Setter -// @Getter -// @ToString -// public static class ConsumerSettings { -// private String bootstrapServers; -// private String keyDeserializer; -// private String valueDeserializer; -// private String clientId; -// private String groupId; -// private String maxPollRecords; -// private String fetchMaxBytes; -// private String maxPartitionFetchBytes; -// private Duration consumeAttemptsTimeoutMs; -// } -// -// @ToString -// @Getter -// @Setter -// public static class TopicsSettings { -// private String actionTopic; -// private String similarityTopic; -// } -//} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java index bcce754..f436052 100644 --- a/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java +++ b/stats/aggregator/src/main/java/ru/practicum/config/KafkaConfigProperties.java @@ -1,26 +1,26 @@ package ru.practicum.config; +import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; +import lombok.experimental.FieldDefaults; import org.springframework.boot.context.properties.ConfigurationProperties; @Getter @Setter @ConfigurationProperties(prefix = "kafka") +@FieldDefaults(level = AccessLevel.PRIVATE) public class KafkaConfigProperties { - private String bootstrapServers; - - private String producerClientIdConfig; - private String producerKeySerializer; - private String producerValueSerializer; - - private String consumerGroupId; - private String consumerClientIdConfig; - private String consumerKeyDeserializer; - private String consumerValueDeserializer; - private long consumerAttemptTimeout; - private String consumerEnableAutoCommit; - - private String userActionTopic; - private String eventsSimilarityTopic; + String bootstrapServers; + String producerClientIdConfig; + String producerKeySerializer; + String producerValueSerializer; + String consumerGroupId; + String consumerClientIdConfig; + String consumerKeyDeserializer; + String consumerValueDeserializer; + long consumerAttemptTimeout; + String consumerEnableAutoCommit; + String userActionTopic; + String eventsSimilarityTopic; } \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java b/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java deleted file mode 100644 index ceb759b..0000000 --- a/stats/aggregator/src/main/java/ru/practicum/config/SimilarityEventProducerConfiguration.java +++ /dev/null @@ -1,47 +0,0 @@ -//package ru.practicum.config; -// -//import jakarta.annotation.PreDestroy; -//import lombok.RequiredArgsConstructor; -//import lombok.extern.slf4j.Slf4j; -//import org.apache.avro.specific.SpecificRecordBase; -//import org.apache.kafka.clients.producer.KafkaProducer; -//import org.apache.kafka.clients.producer.Producer; -//import org.springframework.context.annotation.Bean; -//import org.springframework.stereotype.Component; -// -//import java.util.Properties; -// -//@Slf4j -//@RequiredArgsConstructor -//@Component -//public class SimilarityEventProducerConfiguration { -// private final KafkaConfig kafkaConfig; -// -// @Bean -// SimilarityEventProducer getClient() { -// return new SimilarityEventProducer() { -// private Producer producer; -// -// @Override -// public Producer getProducer() { -// if (producer == null) { -// initProducer(); -// } -// return producer; -// } -// -// private void initProducer() { -// Properties config = kafkaConfig.getProducerProperties(); -// producer = new KafkaProducer<>(config); -// } -// -// @PreDestroy -// @Override -// public void stop() { -// if (producer != null) { -// producer.close(); -// } -// } -// }; -// } -//} \ No newline at end of file diff --git a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java index 1e9fe29..85e8f07 100644 --- a/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java +++ b/stats/aggregator/src/main/java/ru/practicum/service/AggregatorService.java @@ -10,8 +10,7 @@ public interface AggregatorService { void collectEventSimilarity(EventSimilarityAvro eventSimilarityAvro); - default void close() { - } + default void close() {} void flush();; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java b/stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java index 08f387b..e4cace7 100644 --- a/stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java +++ b/stats/analyzer/src/main/java/ru/practicum/config/ConsumerProperties.java @@ -1,17 +1,20 @@ package ru.practicum.config; +import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.experimental.FieldDefaults; @Getter @Setter @NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) public class ConsumerProperties { - private String groupId; - private String clientId; - private String keyDeserializer; - private String valueDeserializer; - private long attemptTimeout; - private String enableAutoCommit; + String groupId; + String clientId; + String keyDeserializer; + String valueDeserializer; + long attemptTimeout; + String enableAutoCommit; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java index b1c2a50..f04b42f 100644 --- a/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java +++ b/stats/analyzer/src/main/java/ru/practicum/config/KafkaConfigProperties.java @@ -1,17 +1,19 @@ package ru.practicum.config; +import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; +import lombok.experimental.FieldDefaults; import org.springframework.boot.context.properties.ConfigurationProperties; @Getter @Setter @ConfigurationProperties(prefix = "kafka") +@FieldDefaults(level = AccessLevel.PRIVATE) public class KafkaConfigProperties { - private String bootstrapServers; - private ConsumerProperties userActionConsumer; - private ConsumerProperties eventSimilarityConsumer; - - private String userActionTopic; - private String eventsSimilarityTopic; + String bootstrapServers; + ConsumerProperties userActionConsumer; + ConsumerProperties eventSimilarityConsumer; + String userActionTopic; + String eventsSimilarityTopic; } \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java deleted file mode 100644 index 1a03c0b..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityService.java +++ /dev/null @@ -1,8 +0,0 @@ -//package ru.practicum.service.event; -// -//import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -// -//public interface EventSimilarityService { -// -// void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro); -//} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java deleted file mode 100644 index c7aaef3..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/service/event/EventSimilarityServiceImpl.java +++ /dev/null @@ -1,39 +0,0 @@ -//package ru.practicum.service; -// -//import lombok.AllArgsConstructor; -//import lombok.extern.slf4j.Slf4j; -//import org.springframework.stereotype.Service; -//import ru.practicum.ewm.stats.avro.EventSimilarityAvro; -//import ru.practicum.mapper.EventSimilarityMapper; -//import ru.practicum.model.EventSimilarity; -//import ru.practicum.repository.EventSimilarityRepository; -//import ru.practicum.service.event.EventSimilarityService; -// -//import java.util.Optional; -// -//@Service -//@Slf4j -//@AllArgsConstructor -//public class EventSimilarityServiceImpl implements EventSimilarityService { -// -// private final EventSimilarityRepository eventSimilarityRepository; -// private final EventSimilarityMapper eventSimilarityMapper; -// -// -// @Override -// public void handleEventSimilarity(EventSimilarityAvro eventSimilarityAvro) { -// log.info("сервис EventSimilarityServiceImpl начал обработку eventSimilarityAvro {}", eventSimilarityAvro); -// EventSimilarity eventSimilarity = eventSimilarityMapper.map(eventSimilarityAvro); -// Optional eventSimilarityOptional = eventSimilarityRepository -// .findEventSimilaritiesByEventAAndEventB(eventSimilarityAvro.getEventA(), eventSimilarityAvro.getEventB()); -// -// if (eventSimilarityOptional.isPresent()) { -// eventSimilarityOptional.get().setScore(eventSimilarity.getScore()); -// eventSimilarityOptional.get().setTimestamp(eventSimilarity.getTimestamp()); -// eventSimilarityRepository.save(eventSimilarityOptional.get()); -// -// } else { -// eventSimilarityRepository.save(eventSimilarity); -// } -// } -//} \ No newline at end of file diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java deleted file mode 100644 index 15cbb47..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionService.java +++ /dev/null @@ -1,7 +0,0 @@ -//package ru.practicum.service.user; -// -//import ru.practicum.ewm.stats.avro.UserActionAvro; -// -//public interface UserActionService { -// void handleUserAction(UserActionAvro userActionAvro); -//} diff --git a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java b/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java deleted file mode 100644 index 7c788d6..0000000 --- a/stats/analyzer/src/main/java/ru/practicum/service/user/UserActionServiceImpl.java +++ /dev/null @@ -1,56 +0,0 @@ -//package ru.practicum.service.user; -// -//import lombok.AllArgsConstructor; -//import lombok.extern.slf4j.Slf4j; -//import org.springframework.stereotype.Service; -//import ru.practicum.ewm.stats.avro.ActionTypeAvro; -//import ru.practicum.ewm.stats.avro.UserActionAvro; -//import ru.practicum.model.UserAction; -//import ru.practicum.repository.UserActionRepository; -// -//import java.util.Optional; -// -// -//@Service -//@Slf4j -//@AllArgsConstructor -//public class -// -//UserActionServiceImpl implements UserActionService { -// private final UserActionRepository userActionRepository; -// -// @Override -// public void handleUserAction(UserActionAvro userActionAvro) { -// -// Optional userActionOptional = userActionRepository.findByUserIdAndEventId(userActionAvro.getUserId(), -// userActionAvro.getEventId()); -// if (userActionOptional.isPresent()) { -// if (userActionOptional.get().getScore() <= calcInteractionScore(userActionAvro.getActionType())) { -// UserAction userAction = UserAction.builder() -// .id(userActionOptional.get().getId()) -// .userId(userActionAvro.getUserId()) -// .lastInteraction(userActionAvro.getTimestamp()) -// .eventId(userActionAvro.getEventId()) -// .score(calcInteractionScore(userActionAvro.getActionType())) -// .build(); -// userActionRepository.save(userAction); -// } -// } else { -// UserAction userAction = UserAction.builder() -// .userId(userActionAvro.getUserId()) -// .lastInteraction(userActionAvro.getTimestamp()) -// .eventId(userActionAvro.getEventId()) -// .score(calcInteractionScore(userActionAvro.getActionType())) -// .build(); -// userActionRepository.save(userAction); -// } -// } -// -// private double calcInteractionScore(ActionTypeAvro type) { -// return switch (type) { -// case VIEW -> 0.4; -// case REGISTER -> 0.8; -// case LIKE -> 1.0; -// }; -// } -//} \ No newline at end of file diff --git a/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java b/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java index c57c0e2..b8611f3 100644 --- a/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java +++ b/stats/collector/src/main/java/ru/practicum/config/KafkaConfigProperties.java @@ -1,16 +1,19 @@ package ru.practicum.config; +import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; +import lombok.experimental.FieldDefaults; import org.springframework.boot.context.properties.ConfigurationProperties; @Getter @Setter @ConfigurationProperties(prefix = "kafka") +@FieldDefaults(level = AccessLevel.PRIVATE) public class KafkaConfigProperties { - private String bootstrapServers; - private String clientIdConfig; - private String producerKeySerializer; - private String producerValueSerializer; - private String userActionTopic; + String bootstrapServers; + String clientIdConfig; + String producerKeySerializer; + String producerValueSerializer; + String userActionTopic; } \ No newline at end of file From c7b91a491f87cbf92fdc9e2d68652d1101794db9 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 12:29:46 +0300 Subject: [PATCH 25/26] fix: added Readme.md --- README.md | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..42e4262 --- /dev/null +++ b/README.md @@ -0,0 +1,159 @@ +# Explore-with-me + +## Introduction + +Explore-with-me - это платформа, позволяющая пользователям создавать мероприятия, управлять ими и участвовать в них. Приложение перешло на **микросервисную архитектуру** для улучшения масштабируемости, отказоустойчивости и гибкости разработки. + +Ключевые компоненты архитектуры: +* **Config Server**: Централизованное управление конфигурациями для всех микросервисов. +* **Discovery Server (Eureka)**: Обеспечивает регистрацию и обнаружение сервисов в сети. +* **Gateway Server**: Единая точка входа для всех клиентских запросов, выполняет маршрутизацию к соответствующим микросервисам. +* **Event Service**: Отвечает за логику, связанную с мероприятиями (создание, поиск, управление). +* **User Service**: Управляет данными пользователей и их аутентификацией/авторизацией. +* **Request Service**: Обрабатывает запросы на участие в мероприятиях. +* **Stats Service**: Система сбора и анализа статистики, разделенная на: + * **Collector**: Принимает и сохраняет информацию о просмотрах и обращениях к эндпоинтам. + * **Aggregator**: Агрегирует собранные данные для последующего анализа. + * **Analyzer**: Анализирует агрегированные данные для формирования рекомендаций пользователям. + +Взаимодействие между сервисами осуществляется с помощью **Feign клиентов**. + +## Technologies Used + +- **Spring Boot**: Основа для создания микросервисов. +- **Spring Cloud**: + - **Spring Cloud Config**: Для `config-server`. + - **Spring Cloud Netflix Eureka** (или **Consul**): Для `discovery-server`. + - **Spring Cloud Gateway**: Для `gateway-server`. + - **OpenFeign**: Для декларативного REST-взаимодействия между сервисами. +- **REST**: Для API коммуникации. +- **Docker & Docker Compose**: Для контейнеризации и оркестрации окружения. +- **PostgreSQL**: В качестве основной базы данных для сервисов (где это необходимо). +- **Lombok**: Для уменьшения шаблонного кода. +- **SLF4J**: Для логирования. + +## Service Overview + +Проект состоит из следующих основных микросервисов: + +- **config-server**: Сервер конфигураций. +- **discovery-server**: Сервер регистрации и обнаружения сервисов. +- **gateway-server**: API шлюз, маршрутизирующий запросы. +- **event-service**: Сервис управления событиями. +- **user-service**: Сервис управления пользователями. +- **request-service**: Сервис управления запросами на участие. +- **collector**: Сервис сбора статистики. +- **aggregator**: Сервис агрегации статистики. +- **analyzer**: Сервис анализа статистики и формирования рекомендаций. + +## Setup and Installation + +### Prerequisites + +- Java 21+ +- Maven +- Docker +- Docker Compose + +### Installation Steps + +1. **Clone the Repository** + + ```bash + git clone https://github.com/yiqes/java-plus-graduation.git + cd java-plus-graduation + ``` + +2. **Build the Project Modules** + + ```bash + mvn clean install + ``` + +3. **Running with Docker Compose (Recommended)** + + - Убедитесь, что Docker и Docker Compose установлены. + - Перейдите в корневую директорию проекта, где находится `docker-compose.yml`. + - Запустите: + + ```bash + docker-compose up --build + ``` + Это поднимет все сконфигурированные сервисы, включая `config-server`, `discovery-server` и базы данных. + +4. **Running Without Docker (More Complex)** + + - Запустите `config-server`. + - Запустите `discovery-server`. Убедитесь, что он регистрируется в `config-server` для получения своей конфигурации. + - Запустите остальные микросервисы (`event-service`, `user-service`, `request-service`, `stats-collector`, `stats-aggregator`, `stats-analyzer`). Они должны регистрироваться в `discovery-server` и получать конфигурацию из `config-server`. + - Запустите `gateway-server`. Он будет использовать `discovery-server` для маршрутизации запросов. + - Для каждого сервиса необходимо настроить подключение к базе данных (если используется) в его `bootstrap.properties` (для подключения к config-server) и `application.properties` (или получать из config-server). + - Запуск каждого сервиса: + ```bash + # Пример для event-service + # java -jar event-service/target/event-service.jar + ``` + +## API Documentation + +Все API запросы должны направляться через **Gateway Server**, который обычно доступен по адресу `http://localhost:8080` (или порт, указанный в вашей конфигурации `gateway-server`). Gateway автоматически маршрутизирует запросы к соответствующим микросервисам. + +**Эндпоинты остались теми же, что и в монолитной версии.** + +### Admin API Endpoints (через Gateway) + +- **POST /admin/categories** + - Создает новую категорию. + - Request Body: `NewCategoryDto` + - *Маршрутизируется на `event-service`* + +- **GET /admin/compilations** + - Получает подборки событий. + - Query Params: `pinned`, `from`, `size` + - *Маршрутизируется на `event-service`* + +- **POST /admin/events** (и другие PATCH, GET эндпоинты для событий) + - Создает/обновляет/получает события (администратор). + - Request Body: `NewEventDto` / `UpdateEventAdminRequest` + - *Маршрутизируется на `event-service`* + +- **GET /admin/users** (и другие эндпоинты для управления пользователями) + - Получает пользователей. + - Query Params: `ids`, `from`, `size` + - *Маршрутизируется на `user-service`* + +### Public API Endpoints (через Gateway) + +- **GET /categories** + - Получает категории. + - Query Params: `from`, `size` + - *Маршрутизируется на `event-service`* + +- **GET /compilations** + - Получает подборки событий. + - Query Params: `pinned`, `from`, `size` + - *Маршрутизируется на `event-service`* + +- **GET /events** + - Получает события (публичный доступ). + - Query Params: `text`, `categories`, `paid`, `rangeStart`, `rangeEnd`, `onlyAvailable`, `sort`, `from`, `size` + - *Маршрутизируется на `event-service`* + +### Private API Endpoints (через Gateway) + +- **POST /users/{userId}/events** (и другие эндпоинты для событий пользователя) + - Создает событие для пользователя. + - Request Body: `NewEventDto` + - *Маршрутизируется на `event-service` (с проверкой пользователя через `user-service` или передачей `userId`)* + +- **GET /users/{userId}/requests** (и другие эндпоинты для запросов пользователя) + - Получает запросы на участие для событий пользователя. + - *Маршрутизируется на `request-service` (с проверкой пользователя)* + +## Usage Examples + +### Getting an event with provided id & user-id from header (через Gateway) + +```bash +curl -X GET "http://localhost:8080/123" \ + -H "X-Ewm-User-Id: 456" \ No newline at end of file From 0b728bdb251a41ba09253accb524402c75f6a973 Mon Sep 17 00:00:00 2001 From: yiqes <155768368+yiqes@users.noreply.github.com> Date: Sun, 11 May 2025 14:52:08 +0300 Subject: [PATCH 26/26] fix: resolved reviewer comments --- .../java/ru/practicum/dto/event/RecommendedEventDto.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java b/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java index ec9868f..5bbd5bd 100644 --- a/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java +++ b/core/interaction-api/src/main/java/ru/practicum/dto/event/RecommendedEventDto.java @@ -1,15 +1,18 @@ package ru.practicum.dto.event; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; import lombok.experimental.SuperBuilder; @Data @SuperBuilder(toBuilder = true) @AllArgsConstructor @NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) public class RecommendedEventDto { - private Long eventId; - private double score; + Long eventId; + double score; } \ No newline at end of file