add-D4Metric-2.0

README.md

@@ -1,3 +1,52 @@
 # seec-analysis-backend
 
-这是seecoder统计与可视化的后端
+这是 Seecoder 统计与可视化的后端服务，提供日志检索、日志统计以及 D4 指标聚合能力。
+
+## 本地启动
+
+```bash
+./mvnw spring-boot:run
+```
+
+Windows:
+
+```powershell
+mvnw.cmd spring-boot:run
+```
+
+## D4 指标能力
+
+- D4-1：协作任务完成率、按时完成率
+- D4-2：沟通平均响应耗时、有效协作次数、冲突解决次数与解决率
+
+指标数据来源于 `biz_name` 为以下值的日志事件：
+
+- `D4_TASK_ASSIGNED`
+- `D4_TASK_COMPLETED`
+- `D4_COLLAB_EFFECTIVE`
+- `D4_COLLAB_CONFLICT_CREATED`
+- `D4_COLLAB_CONFLICT_RESOLVED`
+- `D4_COMM_MESSAGE_SENT`
+- `D4_COMM_MESSAGE_RESPONSE`
+
+## PaaS 部署（Elastic 仅云端可发现）
+
+部署时通过环境变量配置 Elastic 连接，不要把云端地址和凭据写在代码仓库中。
+
+```properties
+ELASTICSEARCH_URIS=http://<cloud-es-service>:9200
+ELASTICSEARCH_USERNAME=<optional>
+ELASTICSEARCH_PASSWORD=<optional>
+ELASTICSEARCH_CONNECTION_TIMEOUT=5s
+ELASTICSEARCH_SOCKET_TIMEOUT=10s
+```
+
+应用默认使用：
+
+- `spring.elasticsearch.uris=${ELASTICSEARCH_URIS:http://localhost:9200}`
+- `spring.elasticsearch.username=${ELASTICSEARCH_USERNAME:}`
+- `spring.elasticsearch.password=${ELASTICSEARCH_PASSWORD:}`
+
+## 文档
+
+完整接口文档见 `README-API.md`。

src/main/java/com/demo/seecanalysisbackend/model/LogEntry.java

@@ -7,6 +7,7 @@ import lombok.Builder;
 import org.springframework.data.annotation.Id;
 import org.springframework.data.elasticsearch.annotations.Document;
 import org.springframework.data.elasticsearch.annotations.Field;
+import org.springframework.data.elasticsearch.annotations.DateFormat;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
 import java.time.LocalDateTime;
@@ -16,13 +17,13 @@ import java.util.Map;
 @NoArgsConstructor
 @AllArgsConstructor
 @Builder
-@Document(indexName = "logs")
+@Document(indexName = "logs", createIndex = false)
 public class LogEntry {
     
     @Id
     private String id;
     
-    @Field(type = FieldType.Text, name = "timestamp")
+    @Field(type = FieldType.Date, name = "timestamp", format = DateFormat.date_time)
     private LocalDateTime timestamp;
     
     @Field(type = FieldType.Text, name = "raw_message")
@@ -36,6 +37,9 @@ public class LogEntry {
     
     @Field(type = FieldType.Keyword, name = "level")
     private String level;
+
+    @Field(type = FieldType.Keyword, name = "application")
+    private String application;
     
     @Field(type = FieldType.Keyword, name = "class_name")
     private String className;

src/main/java/com/demo/seecanalysisbackend/service/D4MetricsService.java

@@ -8,10 +8,11 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
+import org.springframework.data.domain.Sort;
 import org.springframework.stereotype.Service;
 
+import java.time.Duration;
 import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -27,6 +28,8 @@ public class D4MetricsService {
         "D4_COLLAB_CONFLICT_CREATED", "D4_COLLAB_CONFLICT_RESOLVED",
         "D4_COMM_MESSAGE_SENT", "D4_COMM_MESSAGE_RESPONSE"
     );
+    private static final int D4_QUERY_PAGE_SIZE = 1000;
+    private static final long DEFAULT_D4_SLA_THRESHOLD_MS = 24L * 60L * 60L * 1000L;
     
     public D4Metrics calculateD4Metrics(LocalDateTime startTime, LocalDateTime endTime) {
         log.info("Calculating D4 metrics from {} to {}", startTime, endTime);
@@ -43,14 +46,13 @@ public class D4MetricsService {
     }
     
     private D4Metrics.D1Metrics calculateD1Metrics(LocalDateTime startTime, LocalDateTime endTime) {
-        Pageable pageable = PageRequest.of(0, 10000);
-        Page<LogEntry> allLogs = logEntryRepository.findByTimestampBetween(startTime, endTime, pageable);
+        List<LogEntry> allLogs = loadAllD4Logs(startTime, endTime);
         
-        List<LogEntry> taskAssignedLogs = allLogs.getContent().stream()
+        List<LogEntry> taskAssignedLogs = allLogs.stream()
             .filter(log -> "D4_TASK_ASSIGNED".equals(log.getBizName()))
             .collect(Collectors.toList());
         
-        List<LogEntry> taskCompletedLogs = allLogs.getContent().stream()
+        List<LogEntry> taskCompletedLogs = allLogs.stream()
             .filter(log -> "D4_TASK_COMPLETED".equals(log.getBizName()))
             .collect(Collectors.toList());
         
@@ -82,33 +84,34 @@ public class D4MetricsService {
     }
     
     private D4Metrics.D2Metrics calculateD2Metrics(LocalDateTime startTime, LocalDateTime endTime) {
-        Pageable pageable = PageRequest.of(0, 10000);
-        Page<LogEntry> allLogs = logEntryRepository.findByTimestampBetween(startTime, endTime, pageable);
+        List<LogEntry> allLogs = loadAllD4Logs(startTime, endTime);
         
-        List<LogEntry> messageResponseLogs = allLogs.getContent().stream()
+        List<LogEntry> messageResponseLogs = allLogs.stream()
             .filter(log -> "D4_COMM_MESSAGE_RESPONSE".equals(log.getBizName()))
             .collect(Collectors.toList());
         
-        List<LogEntry> collabEffectiveLogs = allLogs.getContent().stream()
+        List<LogEntry> collabEffectiveLogs = allLogs.stream()
             .filter(log -> "D4_COLLAB_EFFECTIVE".equals(log.getBizName()))
             .collect(Collectors.toList());
         
-        List<LogEntry> conflictCreatedLogs = allLogs.getContent().stream()
+        List<LogEntry> conflictCreatedLogs = allLogs.stream()
             .filter(log -> "D4_COLLAB_CONFLICT_CREATED".equals(log.getBizName()))
             .collect(Collectors.toList());
         
-        List<LogEntry> conflictResolvedLogs = allLogs.getContent().stream()
+        List<LogEntry> conflictResolvedLogs = allLogs.stream()
             .filter(log -> "D4_COLLAB_CONFLICT_RESOLVED".equals(log.getBizName()))
             .collect(Collectors.toList());
         
         double avgResponseTimeMs = calculateAvgResponseTime(messageResponseLogs);
         long totalEffectiveCollaborations = collabEffectiveLogs.size();
         long totalConflictResolutions = conflictResolvedLogs.size();
-        double conflictResolutionRate = conflictCreatedLogs.size() > 0 ? 
-            (double) totalConflictResolutions / conflictCreatedLogs.size() : 0.0;
+        long totalCreatedConflicts = countDistinctConflicts(conflictCreatedLogs);
+        double conflictResolutionRate = totalCreatedConflicts > 0 ?
+            (double) totalConflictResolutions / totalCreatedConflicts : 0.0;
         
         Map<String, Long> collaborationsBySource = calculateCollaborationsBySource(collabEffectiveLogs);
         Map<String, Long> conflictsByType = calculateConflictsByType(conflictCreatedLogs);
+        Map<String, LocalDateTime> conflictCreatedTimeByKey = buildConflictCreatedTimeIndex(conflictCreatedLogs);
         
         List<D4Metrics.ResponseTimeDetail> recentResponseTimes = messageResponseLogs.stream()
             .limit(10)
@@ -117,7 +120,7 @@ public class D4MetricsService {
         
         List<D4Metrics.ConflictResolutionDetail> recentResolutions = conflictResolvedLogs.stream()
             .limit(10)
-            .map(this::convertToConflictResolutionDetail)
+            .map(log -> convertToConflictResolutionDetail(log, conflictCreatedTimeByKey))
             .collect(Collectors.toList());
         
         return D4Metrics.D2Metrics.builder()
@@ -131,6 +134,25 @@ public class D4MetricsService {
             .recentResolutions(recentResolutions)
             .build();
     }
+
+    private List<LogEntry> loadAllD4Logs(LocalDateTime startTime, LocalDateTime endTime) {
+        List<LogEntry> allLogs = new ArrayList<>();
+        List<String> d4Events = new ArrayList<>(D4_EVENTS);
+        int page = 0;
+
+        while (true) {
+            Pageable pageable = PageRequest.of(page, D4_QUERY_PAGE_SIZE, Sort.by(Sort.Direction.DESC, "timestamp"));
+            Page<LogEntry> pageResult = logEntryRepository.findByTimestampBetweenAndBizNameIn(startTime, endTime, d4Events, pageable);
+            allLogs.addAll(pageResult.getContent());
+
+            if (!pageResult.hasNext()) {
+                break;
+            }
+            page++;
+        }
+
+        return allLogs;
+    }
     
     private Map<String, Double> calculateCompletionRateByTaskType(List<LogEntry> assignedLogs, List<LogEntry> completedLogs) {
         Map<String, Long> assignedByType = assignedLogs.stream()
@@ -188,21 +210,18 @@ public class D4MetricsService {
     
     private Map<String, Double> calculateCompletionRateByUser(List<LogEntry> assignedLogs, List<LogEntry> completedLogs) {
         Map<String, Long> assignedByUser = assignedLogs.stream()
-            .filter(log -> log.getContext() != null && log.getContext().containsKey("creatorId"))
+            .map(this::extractAssignedUserId)
+            .filter(Objects::nonNull)
             .collect(Collectors.groupingBy(
-                log -> String.valueOf(log.getContext().get("creatorId")),
+                userId -> userId,
                 Collectors.counting()
             ));
         
         Map<String, Long> completedByUser = completedLogs.stream()
-            .filter(log -> log.getContext() != null && 
-                (log.getContext().containsKey("reviewerId") || log.getContext().containsKey("operatorId")))
+            .map(this::extractExecutorUserId)
+            .filter(Objects::nonNull)
             .collect(Collectors.groupingBy(
-                log -> {
-                    String userId = (String) log.getContext().getOrDefault("reviewerId", 
-                        log.getContext().get("operatorId"));
-                    return String.valueOf(userId);
-                },
+                userId -> userId,
                 Collectors.counting()
             ));
         
@@ -219,14 +238,10 @@ public class D4MetricsService {
     }
     
     private double calculateOnTimeCompletionRate(List<LogEntry> completedLogs) {
-        final long SLA_THRESHOLD_MS = 24 * 60 * 60 * 1000; // 24小时
-        
         long onTimeCount = completedLogs.stream()
-            .filter(log -> log.getContext() != null && log.getContext().containsKey("taskDurationMs"))
-            .mapToLong(log -> {
-                Long duration = (Long) log.getContext().get("taskDurationMs");
-                return duration != null && duration <= SLA_THRESHOLD_MS ? 1L : 0L;
-            })
+            .map(log -> getContextLong(log, "taskDurationMs"))
+            .filter(Objects::nonNull)
+            .mapToLong(duration -> duration <= DEFAULT_D4_SLA_THRESHOLD_MS ? 1L : 0L)
             .sum();
         
         return completedLogs.size() > 0 ? (double) onTimeCount / completedLogs.size() : 0.0;
@@ -234,11 +249,9 @@ public class D4MetricsService {
     
     private double calculateAvgResponseTime(List<LogEntry> messageResponseLogs) {
         return messageResponseLogs.stream()
-            .filter(log -> log.getContext() != null && log.getContext().containsKey("responseMs"))
-            .mapToLong(log -> {
-                Long responseMs = (Long) log.getContext().get("responseMs");
-                return responseMs != null ? responseMs : 0L;
-            })
+            .map(log -> getContextLong(log, "responseMs"))
+            .filter(Objects::nonNull)
+            .mapToLong(Long::longValue)
             .average()
             .orElse(0.0);
     }
@@ -256,43 +269,139 @@ public class D4MetricsService {
         return conflictLogs.stream()
             .filter(log -> log.getContext() != null && log.getContext().containsKey("conflictType"))
             .collect(Collectors.groupingBy(
-                log -> (String) log.getContext().get("conflictType"),
+                log -> String.valueOf(log.getContext().get("conflictType")),
                 Collectors.counting()
             ));
     }
     
     private D4Metrics.TaskCompletionDetail convertToTaskCompletionDetail(LogEntry log) {
+        Long durationMs = getContextLong(log, "taskDurationMs");
         return D4Metrics.TaskCompletionDetail.builder()
-            .taskId(log.getContext() != null ? String.valueOf(log.getContext().get("taskId")) : null)
-            .taskType(log.getContext() != null ? (String) log.getContext().get("taskType") : null)
-            .projectId(log.getContext() != null ? String.valueOf(log.getContext().get("projectId")) : null)
-            .userId(log.getContext() != null ? String.valueOf(log.getContext().getOrDefault("reviewerId", 
-                log.getContext().get("operatorId"))) : null)
+            .taskId(getContextString(log, "taskId"))
+            .taskType(getContextString(log, "taskType"))
+            .projectId(getContextString(log, "projectId"))
+            .userId(extractExecutorUserId(log))
             .completedTime(log.getTimestamp())
-            .durationMs(log.getContext() != null ? (Long) log.getContext().get("taskDurationMs") : null)
-            .onTime(log.getContext() != null && log.getContext().containsKey("taskDurationMs") ? 
-                ((Long) log.getContext().get("taskDurationMs")) <= 24 * 60 * 60 * 1000 : null)
+            .durationMs(durationMs)
+            .onTime(durationMs != null ? durationMs <= DEFAULT_D4_SLA_THRESHOLD_MS : null)
             .build();
     }
     
     private D4Metrics.ResponseTimeDetail convertToResponseTimeDetail(LogEntry log) {
         return D4Metrics.ResponseTimeDetail.builder()
-            .messageId(log.getContext() != null ? String.valueOf(log.getContext().get("messageId")) : null)
-            .creatorId(log.getContext() != null ? String.valueOf(log.getContext().get("creatorId")) : null)
-            .receiverId(log.getContext() != null ? String.valueOf(log.getContext().get("receiverId")) : null)
+            .messageId(getContextString(log, "messageId"))
+            .creatorId(getContextString(log, "creatorId"))
+            .receiverId(getContextString(log, "receiverId"))
             .responseTime(log.getTimestamp())
-            .responseMs(log.getContext() != null ? (Long) log.getContext().get("responseMs") : null)
+            .responseMs(getContextLong(log, "responseMs"))
             .build();
     }
     
-    private D4Metrics.ConflictResolutionDetail convertToConflictResolutionDetail(LogEntry log) {
+    private D4Metrics.ConflictResolutionDetail convertToConflictResolutionDetail(
+            LogEntry log,
+            Map<String, LocalDateTime> conflictCreatedTimeByKey) {
+        String conflictKey = buildConflictKey(log);
+        LocalDateTime createdTime = conflictCreatedTimeByKey.get(conflictKey);
+        Long resolutionTimeMs = null;
+        if (createdTime != null && log.getTimestamp() != null && !log.getTimestamp().isBefore(createdTime)) {
+            resolutionTimeMs = Duration.between(createdTime, log.getTimestamp()).toMillis();
+        }
+
         return D4Metrics.ConflictResolutionDetail.builder()
-            .conflictId(log.getContext() != null ? String.valueOf(log.getContext().get("taskId")) : null)
-            .conflictType(log.getContext() != null ? (String) log.getContext().get("conflictType") : null)
-            .projectId(log.getContext() != null ? String.valueOf(log.getContext().get("projectId")) : null)
-            .resolverId(log.getContext() != null ? String.valueOf(log.getContext().get("resolverId")) : null)
+            .conflictId(getContextString(log, "taskId"))
+            .conflictType(getContextString(log, "conflictType"))
+            .projectId(getContextString(log, "projectId"))
+            .resolverId(getContextString(log, "resolverId"))
             .resolvedTime(log.getTimestamp())
-            .resolutionTimeMs(null)
+            .resolutionTimeMs(resolutionTimeMs)
             .build();
     }
+
+    private long countDistinctConflicts(List<LogEntry> conflictCreatedLogs) {
+        return conflictCreatedLogs.stream()
+            .map(this::buildConflictKey)
+            .filter(Objects::nonNull)
+            .distinct()
+            .count();
+    }
+
+    private Map<String, LocalDateTime> buildConflictCreatedTimeIndex(List<LogEntry> conflictCreatedLogs) {
+        Map<String, LocalDateTime> result = new HashMap<>();
+        for (LogEntry log : conflictCreatedLogs) {
+            String key = buildConflictKey(log);
+            if (key == null || log.getTimestamp() == null) {
+                continue;
+            }
+            result.merge(key, log.getTimestamp(), (existing, current) -> current.isBefore(existing) ? current : existing);
+        }
+        return result;
+    }
+
+    private String buildConflictKey(LogEntry log) {
+        String taskId = getContextString(log, "taskId");
+        String projectId = getContextString(log, "projectId");
+        String conflictType = getContextString(log, "conflictType");
+        if (taskId == null && projectId == null && conflictType == null) {
+            return null;
+        }
+        return String.join("|",
+            conflictType == null ? "" : conflictType,
+            projectId == null ? "" : projectId,
+            taskId == null ? "" : taskId);
+    }
+
+    private String extractAssignedUserId(LogEntry log) {
+        String reviewerId = getContextString(log, "reviewerId");
+        if (reviewerId != null) {
+            return reviewerId;
+        }
+        String operatorId = getContextString(log, "operatorId");
+        if (operatorId != null) {
+            return operatorId;
+        }
+        return getContextString(log, "creatorId");
+    }
+
+    private String extractExecutorUserId(LogEntry log) {
+        String reviewerId = getContextString(log, "reviewerId");
+        if (reviewerId != null) {
+            return reviewerId;
+        }
+        return getContextString(log, "operatorId");
+    }
+
+    private String getContextString(LogEntry logEntry, String key) {
+        if (logEntry == null || logEntry.getContext() == null) {
+            return null;
+        }
+        Object value = logEntry.getContext().get(key);
+        return value != null ? String.valueOf(value) : null;
+    }
+
+    private Long getContextLong(LogEntry logEntry, String key) {
+        if (logEntry == null || logEntry.getContext() == null) {
+            return null;
+        }
+        Object value = logEntry.getContext().get(key);
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof Number number) {
+            return number.longValue();
+        }
+        String text = String.valueOf(value).trim();
+        if (text.isEmpty()) {
+            return null;
+        }
+        try {
+            return Long.parseLong(text);
+        } catch (NumberFormatException ignored) {
+            try {
+                return (long) Double.parseDouble(text);
+            } catch (NumberFormatException ex) {
+                this.log.debug("Cannot parse context key {} value {} to long", key, text);
+                return null;
+            }
+        }
+    }
 }

src/main/resources/application.yml

@@ -3,11 +3,11 @@ spring:
     name: seec-analysis-backend
 
   elasticsearch:
-    uris: http://environment-27-45.seec.svc.cluster.local:9200
-    username: "elastic"
-    password: "NJU67es"
-    connection-timeout: 5s
-    socket-timeout: 10s
+    uris: ${ELASTICSEARCH_URIS:http://localhost:9200}
+    username: ${ELASTICSEARCH_USERNAME:}
+    password: ${ELASTICSEARCH_PASSWORD:}
+    connection-timeout: ${ELASTICSEARCH_CONNECTION_TIMEOUT:5s}
+    socket-timeout: ${ELASTICSEARCH_SOCKET_TIMEOUT:10s}
 
   jackson:
     datatype: