fix:发现并填补coder数据抽取的巨大漏洞；sonar分数改为按时间计算

seec-data-server/src/main/java/cn/seecoder/data/server/domain/sonar/SonarCoderDao.java

@@ -15,10 +15,13 @@ import javax.persistence.*;
 @Table(name = "coder")
 public class SonarCoderDao {
 
+    @Id
+    @Column(name = "commit_id" , nullable = false)
+    private Integer commitId;
+
     @Column(name = "user_id" , nullable = false)
     private Integer userId;
 
-    @Id
     @Column(name = "sonar_name" , nullable = false)
     private String sonarName;
 }

seec-data-server/src/main/java/cn/seecoder/data/server/repository/CoderRepository.java

@@ -15,5 +15,7 @@ public interface CoderRepository extends JpaRepository<CoderDao,Integer> {
 
     List<CoderDao> findByExamIdAndProblemIdOrderByLastCommitAt(Integer examId, Integer problemId);
 
-    CoderDao findCoderDaoByRecordIdAndSubmitAt(Integer recordId, LocalDateTime submitAt); //recordId.submitAt是几乎不可能冲突的,不管它来自哪个镜像
+    List<CoderDao> findAllByUserIdAndLastCommitAtBetween(Integer userId, LocalDateTime lastCommitAt, LocalDateTime lastCommitAt2);
+
+    CoderDao findByUserIdAndExamIdAndProblemId(Integer userId, Integer examId, Integer problemId);
 }

seec-data-server/src/main/java/cn/seecoder/data/server/repository/SonarCoderRepository.java

@@ -7,6 +7,7 @@ import org.springframework.stereotype.Repository;
 import java.util.List;
 
 @Repository
-public interface SonarCoderRepository extends JpaRepository<SonarCoderDao,String> {
-    List<SonarCoderDao> findAllByUserId(Integer userId);
+public interface SonarCoderRepository extends JpaRepository<SonarCoderDao,Integer> {
+//    List<SonarCoderDao> findAllByUserIdAndCommitIdBetween(Integer userId, Integer commitId, Integer commitId2);
+    SonarCoderDao findByCommitId(Integer commitId);
 }

seec-data-server/src/main/java/cn/seecoder/data/server/schedule/CoderSchedule.java

@@ -12,6 +12,7 @@ import cn.seecoder.data.server.utils.exception.GlobalException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.format.datetime.standard.DateTimeFormatterFactory;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -20,6 +21,7 @@ import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.client.RestTemplate;
 
 import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -47,38 +49,34 @@ public class CoderSchedule {
     @Transactional(rollbackFor = Exception.class)
     public void coderDataExtraction() throws Exception {
         LocalDateTime start = extractionLogService.getExtractionStartTime(source,1, TimeType.DAY);
-        LocalDateTime middle = extractionLogService.getExtractionStartTime(source);
         LocalDateTime end = extractionLogService.getExtractionEndTime(source);
 
-        Map<String,String> param1 = new HashMap<>();
-        param1.put("startTime",start.toString());
-        param1.put("endTime",middle.toString());
+        Map<String,String> param1 = paramBuilder(start,end);
         System.out.println(param1);
 
-        Map<String,String> param2 = new HashMap<>();
-        param2.put("startTime",middle.toString());
-        param2.put("endTime",end.toString());
-        System.out.println(param2);
-
-
         String coderApi1 = "/api/statistic/getAnalysis?startAt={startTime}&endAt={endTime}";
         String url = coderBaseUrl + coderApi1;
         ParameterizedTypeReference<Response<List<CoderDto>>> responseType = new ParameterizedTypeReference<Response<List<CoderDto>>>() {};
 
-        //上次抽取的时刻可能有些状态为RUNNING数据没抽取到，现在重复抽取上次抽取时刻的当天数据，把未抽取到的数据取到
         List<CoderDto> source = getCoderApi1Data(url,param1,responseType);
         source.forEach(data -> {
-            CoderDao coderDao = coderRepository.findCoderDaoByRecordIdAndSubmitAt(data.getRecordId(),data.getSubmitAt());
+            CoderDao coderDao = coderRepository.findByUserIdAndExamIdAndProblemId(data.getUserId(),data.getExamId(),data.getProblemId());
             if (coderDao == null) {
                 coderRepository.save(coderDtoAndDaoMapper(data));
+            } else {
+                CoderDao coderDao1 = coderDtoAndDaoMapper(data);
+                coderDao1.setId(coderDao.getId());
+                coderRepository.save(coderDao1);
             }
         });
+    }
 
-        //pull本次的数据
-        List<CoderDto> source2 = getCoderApi1Data(url,param2,responseType);
-        source2.forEach(data -> {
-            coderRepository.save(coderDtoAndDaoMapper(data));
-        });
+    private Map<String,String> paramBuilder(LocalDateTime start, LocalDateTime end) {
+        Map<String,String> param = new HashMap<>();
+        DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+        param.put("startTime", start.format(df));
+        param.put("endTime", end.format(df));
+        return param;
     }
 
     private List<CoderDto> getCoderApi1Data(String url1,Map<String,String> params,ParameterizedTypeReference<Response<List<CoderDto>>> responseType) throws Exception {

seec-data-server/src/main/java/cn/seecoder/data/server/schedule/SonarSchedule.java

@@ -62,16 +62,17 @@ public class SonarSchedule {
 
     private SonarCoderDao dtoToDaoMapper(CoderSonarDto sonarDto) {
         return SonarCoderDao.builder()
+                .commitId(sonarDto.getCommitId())
                 .sonarName(sonarDto.getSonarName())
                 .userId(sonarDto.getUserId())
                 .build();
     }
 
     private List<SonarCoderDao> dataFilter(List<CoderSonarDto> inp) {
-        Map<String,SonarCoderDao> map = new HashMap<>();
+        Map<Integer,SonarCoderDao> map = new HashMap<>();
         inp.forEach(item -> {
-            if(!map.containsKey(item.getSonarName())) {
-                map.put(item.getSonarName(),dtoToDaoMapper(item));
+            if(!map.containsKey(item.getCommitId())) {
+                map.put(item.getCommitId(),dtoToDaoMapper(item));
             }
         });
         return (List<SonarCoderDao>) map.values();

seec-data-server/src/main/java/cn/seecoder/data/server/service/Impl/CoderServiceImpl.java

@@ -3,6 +3,7 @@ package cn.seecoder.data.server.service.Impl;
 import cn.seecoder.data.server.domain.CoderDao;
 import cn.seecoder.data.server.repository.CoderRepository;
 import cn.seecoder.data.server.service.CoderService;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -27,32 +28,37 @@ public class CoderServiceImpl implements CoderService {
 
     @Override
     public double getCoderGrade(Integer userId) {
-        AtomicReference<Double> grade = new AtomicReference<>(0.0);
-        AtomicReference<Double> sumGrade = new AtomicReference<>(0.0);
-        List<CoderDao> coderInfoByUser = coderRepository.findByUserId(userId);
-        coderInfoByUser.forEach(info -> {
-            double weight = 1.0;
-            Integer score = info.getScore();
-            Integer problemId = info.getProblemId();
-            Integer examId = info.getExamId();
-            LocalDateTime lastCommitAt = info.getSubmitAt();
-            List<CoderDao> coderDaoList = coderRepository.findByExamIdAndProblemIdOrderByLastCommitAt(examId, problemId);
-
-            ArrayList<LocalDateTime> commitTimeQuartiles = new ArrayList<>(3);
-            commitTimeQuartiles.add(coderDaoList.get(coderDaoList.size()/4).getSubmitAt());
-            commitTimeQuartiles.add(coderDaoList.get(coderDaoList.size()*2/4).getSubmitAt());
-            commitTimeQuartiles.add(coderDaoList.get(coderDaoList.size()*3/4).getSubmitAt());
-            if(lastCommitAt.compareTo(commitTimeQuartiles.get(0))<=0) weight = HIGH;
-            else if(lastCommitAt.compareTo(commitTimeQuartiles.get(1))<=0) weight = HIGH_MIDDLE;
-            else if(lastCommitAt.compareTo(commitTimeQuartiles.get(2))<=0) weight = MIDDLE_LOW;
-            else weight = LOW;
-
-            double finalWeight = weight;
-            grade.updateAndGet(v -> v + finalWeight * score);
-            sumGrade.updateAndGet(v -> v + MAX_SCORE);
-        });
-
-        double result = grade.get() / sumGrade.get();
-        return Math.min(result, 1.0) * 100;
+        try {
+            AtomicReference<Double> grade = new AtomicReference<>(0.0);
+            AtomicReference<Double> sumGrade = new AtomicReference<>(0.0);
+            List<CoderDao> coderInfoByUser = coderRepository.findByUserId(userId);
+            coderInfoByUser.forEach(info -> {
+                double weight = 1.0;
+                Integer score = info.getScore();
+                Integer problemId = info.getProblemId();
+                Integer examId = info.getExamId();
+                LocalDateTime lastCommitAt = info.getSubmitAt();
+                List<CoderDao> coderDaoList = coderRepository.findByExamIdAndProblemIdOrderByLastCommitAt(examId, problemId);
+
+                ArrayList<LocalDateTime> commitTimeQuartiles = new ArrayList<>(3);
+                commitTimeQuartiles.add(coderDaoList.get(coderDaoList.size()/4).getSubmitAt());
+                commitTimeQuartiles.add(coderDaoList.get(coderDaoList.size()*2/4).getSubmitAt());
+                commitTimeQuartiles.add(coderDaoList.get(coderDaoList.size()*3/4).getSubmitAt());
+                if(lastCommitAt.compareTo(commitTimeQuartiles.get(0))<=0) weight = HIGH;
+                else if(lastCommitAt.compareTo(commitTimeQuartiles.get(1))<=0) weight = HIGH_MIDDLE;
+                else if(lastCommitAt.compareTo(commitTimeQuartiles.get(2))<=0) weight = MIDDLE_LOW;
+                else weight = LOW;
+
+                double finalWeight = weight;
+                grade.updateAndGet(v -> v + finalWeight * score);
+                sumGrade.updateAndGet(v -> v + MAX_SCORE);
+            });
+
+            double result = grade.get() / sumGrade.get();
+            return Math.min(result, 1.0) * 100;
+        } catch (Exception e) {
+            LoggerFactory.getLogger(this.getClass()).error("score: coder得分计算失败" + "  userId: "+userId, e);
+            return 0.0;
+        }
     }
 }

seec-data-server/src/main/java/cn/seecoder/data/server/service/Impl/DevcloudServiceImpl.java

@@ -4,6 +4,7 @@ import cn.seecoder.data.server.dto.devcloud.DevcloudCommitCoverageDto;
 import cn.seecoder.data.server.dto.devcloud.Response;
 import cn.seecoder.data.server.service.DevcloudService;
 import com.nju.edu.gitlab.SeecoderGitlabClient;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
@@ -24,33 +25,36 @@ public class DevcloudServiceImpl implements DevcloudService {
     @Value("${sys.devcloud.url}")
     private String devcloudUrl;
 
-    private double[] dividePoint = new double[]{0.875, 0.165, 0.23};
-
     @Override
     public Double commitScoreCompute(Integer userId) { // TODO
-        // 1. get email from gitlab
-//        SeecoderGitlabClient seecoderGitlabClient = new SeecoderGitlabClient(gitlabUrl,gitlabToken);
-//        String email = "email from seecoder gitlab";
-        // 2. fetch data from devcloud
-        List<DevcloudCommitCoverageDto> list = new LinkedList<>();
-        list.add(new DevcloudCommitCoverageDto(1,60,100,5));
-        list.add(new DevcloudCommitCoverageDto(2,30,100,5));
-        list.add(new DevcloudCommitCoverageDto(3,10,100,5));
-        list.add(new DevcloudCommitCoverageDto(4,20,100,5));
-        list.add(new DevcloudCommitCoverageDto(5,50,100,5));
-        Response<List<DevcloudCommitCoverageDto>> coverageDtosResponse = Response.buildSuccess(list);
-        // 3. analysis and compute
-        List<DevcloudCommitCoverageDto> coverageDtos = coverageDtosResponse.getData();
-        AtomicInteger count = new AtomicInteger();
-        AtomicReference<Double> biasSum = new AtomicReference<>(0.0);
-        coverageDtos.forEach(data -> {
-            if(data.getAllPeople()>1) {
-                double difference = data.getCommitTimes() - data.getAllCommits() * 1.0 / data.getAllPeople();
-                biasSum.updateAndGet(v -> (double) (v + (difference > 0 ? difference / (data.getAllPeople()-1) : -difference) / data.getAllCommits() * data.getAllPeople()));
-                count.addAndGet(1);
-            }
-        });
-        double biasRate = biasSum.get()/count.get();
-        return 100 * (1 - biasRate);
+        try {
+            // 1. get email from gitlab
+            //SeecoderGitlabClient seecoderGitlabClient = new SeecoderGitlabClient(gitlabUrl,gitlabToken);
+            //String email = "email from seecoder gitlab";
+            // 2. fetch data from devcloud
+            List<DevcloudCommitCoverageDto> list = new LinkedList<>();
+            list.add(new DevcloudCommitCoverageDto(1,60,100,5));
+            list.add(new DevcloudCommitCoverageDto(2,30,100,5));
+            list.add(new DevcloudCommitCoverageDto(3,10,100,5));
+            list.add(new DevcloudCommitCoverageDto(4,20,100,5));
+            list.add(new DevcloudCommitCoverageDto(5,50,100,5));
+            Response<List<DevcloudCommitCoverageDto>> coverageDtosResponse = Response.buildSuccess(list);
+            // 3. analysis and compute
+            List<DevcloudCommitCoverageDto> coverageDtos = coverageDtosResponse.getData();
+            AtomicInteger count = new AtomicInteger();
+            AtomicReference<Double> biasSum = new AtomicReference<>(0.0);
+            coverageDtos.forEach(data -> {
+                if(data.getAllPeople()>1) {
+                    double difference = data.getCommitTimes() - data.getAllCommits() * 1.0 / data.getAllPeople();
+                    biasSum.updateAndGet(v -> (double) (v + (difference > 0 ? difference / (data.getAllPeople()-1) : -difference) / data.getAllCommits() * data.getAllPeople()));
+                    count.addAndGet(1);
+                }
+            });
+            double biasRate = biasSum.get()/count.get();
+            return 100 * (1 - biasRate);
+        } catch (Exception e) {
+            LoggerFactory.getLogger(this.getClass()).error("score: devcloud commit得分计算失败" + "  userId: "+userId,e);
+            return 0.0;
+        }
     }
 }

seec-data-server/src/main/java/cn/seecoder/data/server/service/Impl/SonarqubeServiceImpl.java

@@ -1,7 +1,9 @@
 package cn.seecoder.data.server.service.Impl;
 
+import cn.seecoder.data.server.domain.CoderDao;
 import cn.seecoder.data.server.domain.sonar.SonarCoderDao;
 import cn.seecoder.data.server.dto.sonarqube.ComponentTreeDto;
+import cn.seecoder.data.server.repository.CoderRepository;
 import cn.seecoder.data.server.repository.SonarCoderRepository;
 import cn.seecoder.data.server.service.SonarqubeService;
 import cn.seecoder.data.server.utils.enums.ErrorCodeEnum;
@@ -15,10 +17,10 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
 
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
+import java.time.LocalDateTime;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 @Service
 public class SonarqubeServiceImpl implements SonarqubeService {
@@ -26,6 +28,9 @@ public class SonarqubeServiceImpl implements SonarqubeService {
     @Autowired
     private SonarCoderRepository sonarCoderRepository;
 
+    @Autowired
+    private CoderRepository coderRepository;
+
     @Autowired
     private RestTemplate restTemplate;
 
@@ -33,10 +38,20 @@ public class SonarqubeServiceImpl implements SonarqubeService {
     private String sonarBaseUrl;
 
     @Override
-    public double getSonarScoreOfUser(int userId) { // 所有项目取平均分 ， 100.0 满分
-        List<ComponentTreeDto> componentTreeDtos = getSonarDataOfUser(userId);
-        double sum = componentTreeDtos.stream().mapToDouble(componentTreeDto -> handleMeasures(componentTreeDto.getBaseComponent().getMeasures())).sum();
-        return sum / componentTreeDtos.size();
+    public double getSonarScoreOfUser(int userId) {
+        return getSonarScoreOfUser(userId, LocalDateTime.MIN, LocalDateTime.MAX);
+    }
+
+    @Override
+    public double getSonarScoreOfUser(int userId, LocalDateTime start, LocalDateTime end) { // 所有项目取平均分 ， 100.0 满分
+        try {
+            List<ComponentTreeDto> componentTreeDtos = getSonarDataOfUser(userId, start, end);
+            double sum = componentTreeDtos.stream().mapToDouble(componentTreeDto -> handleMeasures(componentTreeDto.getBaseComponent().getMeasures())).sum();
+            return sum / componentTreeDtos.size();
+        } catch (Exception e) {
+            LoggerFactory.getLogger(this.getClass()).error("score: sonar得分计算失败" + "  userId: "+userId, e);
+            return 0.0;
+        }
     }
 
     private double handleMeasures(List<ComponentTreeDto.Measure> measures) { // 单个项目得分计算
@@ -48,17 +63,27 @@ public class SonarqubeServiceImpl implements SonarqubeService {
         return sum.get() / measures.size();
     }
 
-    private List<ComponentTreeDto> getSonarDataOfUser(int userId) { // note: 以后devcloud接入sonar之后，此处源码需要修改
-        // 1. fetch sonarName by userId
-        List<SonarCoderDao> sonarCoderDaos = sonarCoderRepository.findAllByUserId(userId);
+    private List<ComponentTreeDto> getSonarDataOfUser(int userId, LocalDateTime start, LocalDateTime end) { // note: 以后devcloud接入sonar之后，此处源码需要修改
+        // 0. 通过考试结束时间来获取所有 last commit
+        // 1. 在sonar coder表中通过last commit 获取到对应的 sonarname， 去重
+        Set<String> sonarNames = new HashSet<>();
+        Set<Integer> commitIdsSet =  coderRepository.findAllByUserIdAndLastCommitAtBetween(userId,start,end)
+                .stream().map(CoderDao::getCommitId).collect(Collectors.toSet());
+
+        commitIdsSet.forEach(commitId -> {
+            SonarCoderDao byCommitId = sonarCoderRepository.findByCommitId(commitId);
+            if(byCommitId!=null) sonarNames.add(byCommitId.getSonarName());
+        });
+
+
         // 2. fetch sonarComponentTreeDto by sonarName
         List<ComponentTreeDto> componentTreeDtos = new LinkedList<>();
-        sonarCoderDaos.forEach(sonarCoderDao -> {
+        sonarNames.forEach(sonarName -> {
             try {
-                ComponentTreeDto componentTreeDto = fetchSonarDataBySonarName(sonarCoderDao.getSonarName());
+                ComponentTreeDto componentTreeDto = fetchSonarDataBySonarName(sonarName);
                 componentTreeDtos.add(componentTreeDto);
             } catch (Exception e){
-                LoggerFactory.getLogger(getClass()).error("sonar: Cannot get data by sonarname '" + sonarCoderDao.getSonarName() + "' !" );
+                LoggerFactory.getLogger(getClass()).error("sonar: Cannot get data by sonarname '" + sonarName + "' !" );
             }
         });
         return componentTreeDtos;

seec-data-server/src/main/java/cn/seecoder/data/server/service/SonarqubeService.java

@@ -1,5 +1,8 @@
 package cn.seecoder.data.server.service;
 
+import java.time.LocalDateTime;
+
 public interface SonarqubeService {
-    double getSonarScoreOfUser(int userId);
+    public double getSonarScoreOfUser(int userId);
+    public double getSonarScoreOfUser(int userId, LocalDateTime start, LocalDateTime end);
 }

seec-data-server/src/main/resources/application.yml

@@ -20,7 +20,7 @@ jwt:
 
 sys:
   coder:
-    url: 'coder'
+    url: http://coder.seecoder.cn/
   playground:
     url: 'playground'
     token: 'token'