feat:coder sonar 相关数据抽取

seec-data-server/src/main/java/cn/seecoder/data/server/controller/EvaluateController.java

@@ -41,7 +41,6 @@ public class EvaluateController {
     @RequestMapping("/eval/analysis/{examId}")
     public Result<Map<String, Map<String,Double>>> evalAnalysisByExamId(@PathVariable("examId") Integer examId) {
         Map<String, Map<String, Double>> res = evaluateService.analyzeEvalTest(examId);
-        ErrorCodeEnum serviceErrorC0001 = ErrorCodeEnum.SERVICE_ERROR_C0001;
         if(res == null) throw new GlobalException(ErrorCodeEnum.SERVICE_ERROR_C0001);
         return Result.buildSuccess(evaluateService.analyzeEvalTest(examId));
     }

seec-data-server/src/main/java/cn/seecoder/data/server/domain/sonar/SonarCoderDao.java

@@ -0,0 +1,24 @@
+package cn.seecoder.data.server.domain.sonar;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.persistence.*;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Entity
+@Table(name = "coder")
+public class SonarCoderDao {
+
+    @Column(name = "user_id" , nullable = false)
+    private Integer userId;
+
+    @Id
+    @Column(name = "sonar_name" , nullable = false)
+    private String sonarName;
+}

seec-data-server/src/main/java/cn/seecoder/data/server/domain/sonar/SonarDevcloudDao.java

@@ -0,0 +1,4 @@
+package cn.seecoder.data.server.domain.sonar;
+
+public class SonarDevcloudDao {
+}

seec-data-server/src/main/java/cn/seecoder/data/server/repository/SonarCoderRepository.java

@@ -0,0 +1,9 @@
+package cn.seecoder.data.server.repository;
+
+import cn.seecoder.data.server.domain.sonar.SonarCoderDao;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface SonarCoderRepository extends JpaRepository<SonarCoderDao,String> {
+}

seec-data-server/src/main/java/cn/seecoder/data/server/schedule/CoderSchedule.java

@@ -7,12 +7,14 @@ import cn.seecoder.data.server.repository.CoderRepository;
 import cn.seecoder.data.server.service.ExtractionLogService;
 import cn.seecoder.data.server.utils.aspects.dataExtraction.DataExtractor;
 import cn.seecoder.data.server.utils.enums.ErrorCodeEnum;
+import cn.seecoder.data.server.utils.enums.TimeType;
 import cn.seecoder.data.server.utils.exception.GlobalException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.client.RestTemplate;
@@ -38,15 +40,15 @@ public class CoderSchedule {
     @Autowired
     private RestTemplate restTemplate;
 
-    private final String source1 = "coder1";
+    private final String source = "coder";
 
 //    @Scheduled(cron = "00 00 00 * * ?")
-    @DataExtractor(source1)
+    @DataExtractor(source)
     @Transactional(rollbackFor = Exception.class)
     public void coderDataExtraction() throws Exception {
-        LocalDateTime start = extractionLogService.getExtractionStartTime(source1,1);
-        LocalDateTime middle = extractionLogService.getExtractionStartTime(source1);
-        LocalDateTime end = extractionLogService.getExtractionEndTime(source1);
+        LocalDateTime start = extractionLogService.getExtractionStartTime(source,1, TimeType.DAY);
+        LocalDateTime middle = extractionLogService.getExtractionStartTime(source);
+        LocalDateTime end = extractionLogService.getExtractionEndTime(source);
 
         Map<String,String> param1 = new HashMap<>();
         param1.put("startTime",start.toString());
@@ -59,11 +61,11 @@ public class CoderSchedule {
         System.out.println(param2);
 
 
-        String coderApi1 = "/xxx?startTime={startTime}&endTime={endTime}";
+        String coderApi1 = "/api/statistic/getAnalysis?startAt={startTime}&endAt={endTime}";
         String url = coderBaseUrl + coderApi1;
         ParameterizedTypeReference<Response<List<CoderDto>>> responseType = new ParameterizedTypeReference<Response<List<CoderDto>>>() {};
 
-        //再pull一次上次pull的时间段的数据，将一些遗漏的数据存入数据库
+        //上次抽取的时刻可能有些状态为RUNNING数据没抽取到，现在重复抽取上次抽取时刻的当天数据，把未抽取到的数据取到
         List<CoderDto> source = getCoderApi1Data(url,param1,responseType);
         source.forEach(data -> {
             CoderDao coderDao = coderRepository.findCoderDaoByRecordIdAndSubmitAt(data.getRecordId(),data.getSubmitAt());

seec-data-server/src/main/java/cn/seecoder/data/server/schedule/SonarSchedule.java

@@ -0,0 +1,92 @@
+package cn.seecoder.data.server.schedule;
+
+import cn.seecoder.data.server.domain.sonar.SonarCoderDao;
+import cn.seecoder.data.server.dto.coder.CoderDto;
+import cn.seecoder.data.server.dto.coder.CoderSonarDto;
+import cn.seecoder.data.server.dto.coder.Response;
+import cn.seecoder.data.server.repository.SonarCoderRepository;
+import cn.seecoder.data.server.service.ExtractionLogService;
+import cn.seecoder.data.server.utils.aspects.dataExtraction.DataExtractor;
+import cn.seecoder.data.server.utils.enums.ErrorCodeEnum;
+import cn.seecoder.data.server.utils.enums.TimeType;
+import cn.seecoder.data.server.utils.exception.GlobalException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.client.RestTemplate;
+
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SonarSchedule {
+
+    private final String source = "coderSonar";
+
+    @Value("${remoteUrls.coder}")
+    private String coderBaseUrl;
+
+    @Autowired
+    private ExtractionLogService extractionLogService;
+
+    @Autowired
+    private SonarCoderRepository sonarCoderRepository;
+
+    @Autowired
+    private RestTemplate restTemplate;
+
+    //    @Scheduled(cron = "00 00 00 * * ?")
+    @DataExtractor(source)
+    @Transactional(rollbackFor = Exception.class)
+    public void coderSoanrDataExtraction() throws Exception {
+        LocalDateTime start = extractionLogService.getExtractionStartTime(source,1, TimeType.DAY);
+        LocalDateTime end = extractionLogService.getExtractionEndTime(source);
+
+        Map<String,String> param1 = buildParam(start, end);
+
+        String sonarCoderApi = "/api/statistic/getSonar?startAt={startTime}&endAt={endTime}";
+        String url = coderBaseUrl + sonarCoderApi;
+        ParameterizedTypeReference<Response<List<CoderSonarDto>>> responseType = new ParameterizedTypeReference<Response<List<CoderSonarDto>>>() {};
+
+        List<CoderSonarDto> coderSonarDtos = getCoderSonarData(url,param1,responseType);
+        List<SonarCoderDao> sonarCoderDaos = dataFilter(coderSonarDtos);
+        sonarCoderRepository.saveAll(sonarCoderDaos);
+    }
+
+    private SonarCoderDao dtoToDaoMapper(CoderSonarDto sonarDto) {
+        return SonarCoderDao.builder()
+                .sonarName(sonarDto.getSonarName())
+                .userId(sonarDto.getUserId())
+                .build();
+    }
+
+    private List<SonarCoderDao> dataFilter(List<CoderSonarDto> inp) {
+        Map<String,SonarCoderDao> map = new HashMap<>();
+        inp.forEach(item -> {
+            if(!map.containsKey(item.getSonarName())) {
+                map.put(item.getSonarName(),dtoToDaoMapper(item));
+            }
+        });
+        return (List<SonarCoderDao>) map.values();
+    }
+
+    private Map<String,String> buildParam(LocalDateTime start,LocalDateTime end) {
+        Map<String,String> param = new HashMap<>();
+        param.put("startTime",start.toString());
+        param.put("endTime",end.toString());
+        return param;
+    }
+
+    private List<CoderSonarDto> getCoderSonarData(String url, Map<String,String> param, ParameterizedTypeReference<Response<List<CoderSonarDto>>> responseType) {
+        ResponseEntity<Response<List<CoderSonarDto>>> r = restTemplate.exchange(url, HttpMethod.GET,null,responseType,param );
+        Response<List<CoderSonarDto>> body = r.getBody();
+        if (body == null || body.getCode()!=200) throw new GlobalException(ErrorCodeEnum.SERVICE_ERROR_C0002);
+        List<CoderSonarDto> source = body.getSource();
+        if(source == null || source.size()==0) throw new GlobalException(ErrorCodeEnum.SERVICE_ERROR_C0003);
+        return source;
+    }
+}

seec-data-server/src/main/java/cn/seecoder/data/server/service/ExtractionLogService.java

@@ -1,21 +1,23 @@
 package cn.seecoder.data.server.service;
 
+import cn.seecoder.data.server.utils.enums.TimeType;
+
 import java.time.LocalDateTime;
 
 /**
  * 注意： 该service只能在数据抽取函数( schedule.*.*Extraction )里面使用
  * */
 public interface ExtractionLogService {
-    /**
-     * 参数 back 默认为0
-     * */
+
+    //“上次” 抽取成功的时间，作为本次抽取的开始时间
     LocalDateTime getExtractionStartTime(String source);
 
-    /**
-     * @param back back==0时，返回上次成功抽取的时间（可用作本次的开始时间）；
-     *             back==n时，往前追溯第n次成功抽取的时间（用作本次开始时间），防止一些之前抽取时正在发生变化的数据抽取不到
-     * */
+    //按照抽取成功的时间节点回溯，[back=0] 表示 “上次” 抽取成功的时间 [back=1]，表示 “上上次” 抽取成功的时间
     LocalDateTime getExtractionStartTime(String source,Integer back);
 
+    //按照时间类型回溯， [back=1,timeType=DAY] 表示从上次抽取成功的时间开始，向前回溯一天
+    LocalDateTime getExtractionStartTime(String source, Integer back, TimeType timeType);
+
+    //本次抽取的时间节点，作为抽取结束时间
     LocalDateTime getExtractionEndTime(String source);
 }

seec-data-server/src/main/java/cn/seecoder/data/server/service/Impl/ExtractionLogServiceImpl.java

@@ -4,6 +4,7 @@ import cn.seecoder.data.server.domain.ExtractionLogDao;
 import cn.seecoder.data.server.repository.ExtractionLogRepository;
 import cn.seecoder.data.server.service.ExtractionLogService;
 import cn.seecoder.data.server.utils.enums.ExtractionLogStatus;
+import cn.seecoder.data.server.utils.enums.TimeType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -33,6 +34,22 @@ public class ExtractionLogServiceImpl implements ExtractionLogService {
         return extractionLogs.get().findFirst().get().getCreateAt();
     }
 
+    @Override
+    public LocalDateTime getExtractionStartTime(String source, Integer back, TimeType timeType) {
+
+        LocalDateTime lastSuccessTime = this.getExtractionStartTime(source, 0);
+        switch (timeType) {
+            case SECOND: return lastSuccessTime.minusSeconds(back);
+            case MINUTE: return lastSuccessTime.minusMinutes(back);
+            case HOUR: return lastSuccessTime.minusHours(back);
+            case DAY: return lastSuccessTime.minusDays(back);
+            case WEEK: return lastSuccessTime.minusWeeks(back);
+            case MONTH: return lastSuccessTime.minusMonths(back);
+            case YEAR: return lastSuccessTime.minusYears(back);
+            default: return lastSuccessTime;
+        }
+    }
+
     @Override
     public LocalDateTime getExtractionEndTime(String source) { //返回刚创建的时间
         PageRequest pr = PageRequest.of(0,1,Sort.by(Sort.Direction.DESC,"id"));

seec-data-server/src/main/java/cn/seecoder/data/server/utils/enums/TimeType.java

@@ -0,0 +1,11 @@
+package cn.seecoder.data.server.utils.enums;
+
+public enum TimeType {
+    SECOND,
+    MINUTE,
+    HOUR,
+    DAY,
+    WEEK,
+    MONTH,
+    YEAR
+}