任务内手机号去重

services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueJpa.java

@@ -6,10 +6,12 @@ import com.baiye.feign.OrganizeClient;
 import com.baiye.model.dto.ClueDto;
 import com.baiye.model.dto.ClueQueryCriteria;
 import com.baiye.model.vo.ResSourceLabel;
+import com.baiye.module.entity.Clue;
 import com.baiye.util.AESUtils;
 import com.baiye.util.RedisUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
+import org.hibernate.query.NativeQuery;
 import org.hibernate.query.internal.NativeQueryImpl;
 import org.hibernate.transform.Transformers;
 import org.springframework.beans.factory.annotation.Value;
@@ -97,8 +99,8 @@ public class ClueJpa {
         if (clueQueryCriteria.getMemberStatus() != null) {
             sql.append("and cm.member_status = :memberStatus ");
         }
-        //拼接最后加入时间排序
-        sql.append("ORDER BY c.create_time desc ");
+        //拼接最后加入时间排序,没加id排序出现数据重复问题(注意空格)
+        sql.append("ORDER BY c.create_time desc ,c.id ");
 
         if (pageable != null) {
             sql.append("LIMIT :number,:size ");
@@ -205,4 +207,29 @@ public class ClueJpa {
         }
         return list;
     }
+
+    public List<Clue> findClue(Long taskId) {
+        StringBuilder sql = new StringBuilder();
+        sql.append("SELECT tc.id as id,tc.nid as nid FROM tb_clue_middle tcm LEFT JOIN tb_clue tc ON tcm.clue_id = tc.id WHERE 1 = 1  ");
+        if (taskId != null) {
+            sql.append("AND tcm.task_id = :taskId ");
+        }
+        Query query = entityManager.createNativeQuery(sql.toString());
+        if (taskId != null) {
+            query.setParameter("taskId", taskId);
+        }
+        query.unwrap(NativeQuery.class).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP);
+        List resultList = query.getResultList();
+        List<Clue> arrayList = new ArrayList<>();
+        for (Object obj : resultList) {
+            Map row = (Map) obj;
+            Clue clue = new Clue();
+            BigInteger clueId = (BigInteger)row.get("id");
+            String phone = (String)row.get("nid");
+            clue.setNid(phone);
+            clue.setId(clueId.longValue());
+            arrayList.add(clue);
+        }
+    return arrayList;
+    }
 }

services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueMiddleRepository.java

@@ -8,6 +8,7 @@ import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
+import java.util.Set;
 
 @Repository
 public interface ClueMiddleRepository extends JpaRepository<ClueMiddle, Long>, JpaSpecificationExecutor<ClueMiddle> {
@@ -105,4 +106,11 @@ public interface ClueMiddleRepository extends JpaRepository<ClueMiddle, Long>, J
      */
     @Query(value = "select count(*) from tb_clue_middle where task_id = ?1", nativeQuery = true)
     Integer findTaskNum(Long taskId);
+
+    /**
+     * Id批量删除
+     *
+     * @param clueIdSet
+     */
+    void deleteAllByClueIdIn(Set<Long> clueIdSet);
 }

services/ad-platform-source/src/main/java/com/baiye/module/dao/ClueRepository.java

@@ -5,7 +5,15 @@ import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
 import org.springframework.stereotype.Repository;
 
+import java.util.Set;
+
 
 @Repository
 public interface ClueRepository extends JpaRepository<Clue, Long>, JpaSpecificationExecutor<Clue> {
+    /**
+     * Id批量删除
+     *
+     * @param clueIdList
+     */
+    void deleteAllByIdIn(Set<Long> clueIdList);
 }

services/ad-platform-source/src/main/java/com/baiye/task/FileAnalysisTask.java

@@ -3,6 +3,10 @@ package com.baiye.task;
 import cn.hutool.core.date.DateUtil;
 import com.alibaba.excel.EasyExcelFactory;
 import com.baiye.module.constant.FileConstant;
+import com.baiye.module.dao.ClueJpa;
+import com.baiye.module.dao.ClueMiddleRepository;
+import com.baiye.module.dao.ClueRepository;
+import com.baiye.module.entity.Clue;
 import com.baiye.module.entity.ClueRecord;
 import com.baiye.module.entity.vo.BaseExcelVo;
 import com.baiye.module.entity.vo.KsVo;
@@ -16,10 +20,12 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
 
 import javax.annotation.Resource;
 import java.io.FileInputStream;
-import java.util.List;
+import java.util.*;
+import java.util.stream.Collectors;
 
 
 @Component
@@ -29,10 +35,20 @@ public class FileAnalysisTask {
     @Resource
     private ClueService clueService;
 
+    @Resource
+    private ClueJpa clueJpa;
+
+    @Resource
+    private ClueRepository clueRepository;
+
+    @Resource
+    private ClueMiddleRepository clueMiddleRepository;
+
     /**
      * 读取文件处理数据
      * @param clueRecords
      */
+    @Transactional(rollbackFor = Exception.class)
     @Async(value = "SendBigDataTaskExecutor")
     @SneakyThrows   //处理异常try
     public void runFileAnalysisTask(List<ClueRecord> clueRecords,String oneFileName) {
@@ -67,7 +83,9 @@ public class FileAnalysisTask {
         Long taskId = clueRecords.get(0).getTaskId();
         Long userId = clueRecords.get(0).getCreateBy();
         if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)){
-            log.info("================创建任务开始:{}================", DateUtil.now());
+            // 任务内手机号去重,先去重然后创建任务(统计任务数量)
+            distinctNid(taskId);
+            // 创建任务
             BaseExcelVo baseExcelVo = new BaseExcelVo();
             baseExcelVo.setUserId(userId);
             baseExcelVo.setTaskId(taskId);
@@ -75,4 +93,26 @@ public class FileAnalysisTask {
             clueService.saveTask(baseExcelVo);
         }
     }
+
+    /**
+     * 同一个任务内去重手机号
+     * @param taskId
+     */
+    @Transactional(rollbackFor = Exception.class)
+    public void distinctNid(Long taskId){
+        List<Clue> clueList = clueJpa.findClue(taskId);
+        Map<String, List<Clue>> map = clueList.stream().collect(Collectors.groupingBy(Clue::getNid));
+        Set<String> keySet = map.keySet();
+        for (String key : keySet) {
+            if (map.get(key).size() > 1){
+                List<Clue> clues = map.get(key);
+                clues.remove(0);
+                Set<Long> clueIdSet = new HashSet<>();
+                clues.forEach(cs -> clueIdSet.add(cs.getId()));
+                clueRepository.deleteAllByIdIn(clueIdSet);
+                log.info("===========要删除的id：{}============",clueIdSet);
+                clueMiddleRepository.deleteAllByClueIdIn(clueIdSet);
+            }
+        }
+    }
 }