|
|
|
@ -1,21 +1,24 @@
|
|
|
|
|
package com.baiye.task;
|
|
|
|
|
|
|
|
|
|
import com.alibaba.excel.EasyExcelFactory;
|
|
|
|
|
import com.baiye.exception.BadRequestException;
|
|
|
|
|
import com.baiye.feign.TaskClient;
|
|
|
|
|
import com.baiye.module.constant.FileConstant;
|
|
|
|
|
import com.baiye.module.dao.ClueJpa;
|
|
|
|
|
import com.baiye.module.dao.ClueMiddleRepository;
|
|
|
|
|
import com.baiye.module.dao.ClueRepository;
|
|
|
|
|
import com.baiye.module.entity.Clue;
|
|
|
|
|
import com.baiye.module.entity.ClueRecord;
|
|
|
|
|
import com.baiye.module.entity.Task;
|
|
|
|
|
import com.baiye.module.entity.vo.BaseExcelVo;
|
|
|
|
|
import com.baiye.module.entity.vo.KsVo;
|
|
|
|
|
import com.baiye.module.entity.vo.TaskZdyVo;
|
|
|
|
|
import com.baiye.module.entity.vo.ZdyVo;
|
|
|
|
|
import com.baiye.module.listener.KsExcelListener;
|
|
|
|
|
import com.baiye.module.listener.TaskZdyExcelListener;
|
|
|
|
|
import com.baiye.module.listener.UcCsvListener;
|
|
|
|
|
import com.baiye.module.listener.ZdyExcelListener;
|
|
|
|
|
import com.baiye.module.service.ClueService;
|
|
|
|
|
import com.baiye.util.RequestHeaderHandler;
|
|
|
|
|
import lombok.SneakyThrows;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.apache.commons.lang.StringUtils;
|
|
|
|
|
import org.springframework.scheduling.annotation.Async;
|
|
|
|
@ -35,56 +38,75 @@ public class FileAnalysisTask {
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
private ClueService clueService;
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
private ClueJpa clueJpa;
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
private ClueRepository clueRepository;
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
private ClueMiddleRepository clueMiddleRepository;
|
|
|
|
|
@Resource
|
|
|
|
|
private TaskClient taskClient;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 读取文件处理数据
|
|
|
|
|
* 异步读取文件处理数据
|
|
|
|
|
*
|
|
|
|
|
* @param clueRecords
|
|
|
|
|
* @param headerMap
|
|
|
|
|
*/
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
@Async(value = "SendBigDataTaskExecutor")
|
|
|
|
|
@SneakyThrows //处理异常try
|
|
|
|
|
public void runFileAnalysisTask(List<ClueRecord> clueRecords, String oneFileName, Map<String, String> headerMap) {
|
|
|
|
|
//@SneakyThrows
|
|
|
|
|
public void runFileAnalysisTask(List<ClueRecord> clueRecords, String oneFileName){
|
|
|
|
|
try {
|
|
|
|
|
for (ClueRecord clueRecord : clueRecords) {
|
|
|
|
|
BaseExcelVo baseExcelVo = new BaseExcelVo();
|
|
|
|
|
baseExcelVo.setUserId(clueRecord.getCreateBy());
|
|
|
|
|
baseExcelVo.setClueRecordId(clueRecord.getId());
|
|
|
|
|
baseExcelVo.setOrigin(clueRecord.getUploadType());
|
|
|
|
|
baseExcelVo.setTaskId(clueRecord.getTaskId());
|
|
|
|
|
baseExcelVo.setOneFileName(oneFileName);
|
|
|
|
|
|
|
|
|
|
for (ClueRecord clueRecord : clueRecords) {
|
|
|
|
|
BaseExcelVo baseExcelVo = new BaseExcelVo();
|
|
|
|
|
baseExcelVo.setUserId(clueRecord.getCreateBy());
|
|
|
|
|
baseExcelVo.setClueRecordId(clueRecord.getId());
|
|
|
|
|
baseExcelVo.setOrigin(clueRecord.getUploadType());
|
|
|
|
|
baseExcelVo.setTaskId(clueRecord.getTaskId());
|
|
|
|
|
baseExcelVo.setOneFileName(oneFileName);
|
|
|
|
|
|
|
|
|
|
FileInputStream file = new FileInputStream(clueRecord.getNewFileUrl());
|
|
|
|
|
switch (clueRecord.getUploadType()){
|
|
|
|
|
case FileConstant.ZERO_NUMBER:
|
|
|
|
|
log.info("==============开始导入自定义表单============{}",Thread.currentThread().getName());
|
|
|
|
|
EasyExcelFactory.read(file, ZdyVo.class, new ZdyExcelListener(clueService,baseExcelVo)).build().readAll();
|
|
|
|
|
break;
|
|
|
|
|
case FileConstant.ONE_NUMBER:
|
|
|
|
|
log.info("==============开始导入UC表单============{}",Thread.currentThread().getName());
|
|
|
|
|
new UcCsvListener(clueService,baseExcelVo).readUcCsv(clueRecord.getNewFileUrl());
|
|
|
|
|
break;
|
|
|
|
|
case FileConstant.TWO_NUMBER:
|
|
|
|
|
log.info("==============开始导入快手表单============{}",Thread.currentThread().getName());
|
|
|
|
|
EasyExcelFactory.read(file, KsVo.class, new KsExcelListener(clueService,baseExcelVo)).build().readAll();
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
FileInputStream file = new FileInputStream(clueRecord.getNewFileUrl());
|
|
|
|
|
switch (clueRecord.getUploadType()) {
|
|
|
|
|
case FileConstant.ZERO_NUMBER:
|
|
|
|
|
log.info("==============开始导入自定义表单============{}", Thread.currentThread().getName());
|
|
|
|
|
EasyExcelFactory.read(file, ZdyVo.class, new ZdyExcelListener(clueService, baseExcelVo)).build().readAll();
|
|
|
|
|
break;
|
|
|
|
|
case FileConstant.ONE_NUMBER:
|
|
|
|
|
log.info("==============开始导入UC表单============{}", Thread.currentThread().getName());
|
|
|
|
|
new UcCsvListener(clueService, baseExcelVo).readUcCsv(clueRecord.getNewFileUrl());
|
|
|
|
|
break;
|
|
|
|
|
case FileConstant.TWO_NUMBER:
|
|
|
|
|
log.info("==============开始导入快手表单============{}", Thread.currentThread().getName());
|
|
|
|
|
EasyExcelFactory.read(file, KsVo.class, new KsExcelListener(clueService, baseExcelVo)).build().readAll();
|
|
|
|
|
break;
|
|
|
|
|
case FileConstant.FIVE_NUMBER:
|
|
|
|
|
log.info("==============开始导入动态任务表单============{}", Thread.currentThread().getName());
|
|
|
|
|
EasyExcelFactory.read(file, TaskZdyVo.class, new TaskZdyExcelListener(clueService, baseExcelVo)).build().readAll();
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
log.info("=========读取资源错误============{}",e);
|
|
|
|
|
taskClient.sendErrMessage("导入资源错误,请检查数据格式后重新导入", clueRecords.get(0).getCreateBy());
|
|
|
|
|
throw new BadRequestException("=========读取资源错误============");
|
|
|
|
|
}
|
|
|
|
|
//动态任务不需要创建任务,更新后结束方法
|
|
|
|
|
if (clueRecords.get(0).getUploadType() == FileConstant.FIVE_NUMBER){
|
|
|
|
|
Long taskId = clueRecords.get(0).getTaskId();
|
|
|
|
|
Integer taskNum = clueMiddleRepository.findTaskNum(taskId);
|
|
|
|
|
Task task = new Task();
|
|
|
|
|
task.setId(taskId);
|
|
|
|
|
task.setTotalNumber(taskNum);
|
|
|
|
|
task.setIsDistribution(1);
|
|
|
|
|
taskClient.updateTask(task);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
//创建任务
|
|
|
|
|
Long taskId = clueRecords.get(0).getTaskId();
|
|
|
|
|
Long userId = clueRecords.get(0).getCreateBy();
|
|
|
|
|
if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)){
|
|
|
|
|
if (taskId != null && userId != null && StringUtils.isNotBlank(oneFileName)) {
|
|
|
|
|
// 任务内手机号去重,先去重然后创建任务(统计任务数量)
|
|
|
|
|
distinctNid(taskId);
|
|
|
|
|
// 创建任务
|
|
|
|
@ -92,24 +114,24 @@ public class FileAnalysisTask {
|
|
|
|
|
baseExcelVo.setUserId(userId);
|
|
|
|
|
baseExcelVo.setTaskId(taskId);
|
|
|
|
|
baseExcelVo.setOneFileName(oneFileName);
|
|
|
|
|
RequestHeaderHandler.setHeaderMap(headerMap);
|
|
|
|
|
clueService.saveTask(baseExcelVo);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 同一个任务内去重手机号
|
|
|
|
|
*
|
|
|
|
|
* @param taskId
|
|
|
|
|
*/
|
|
|
|
|
@Transactional(propagation = Propagation.REQUIRED)
|
|
|
|
|
public void distinctNid(Long taskId){
|
|
|
|
|
public void distinctNid(Long taskId) {
|
|
|
|
|
Set<Long> taskIds = new HashSet();
|
|
|
|
|
taskIds.add(taskId);
|
|
|
|
|
List<Clue> clueList = clueJpa.findClue(taskIds,null);
|
|
|
|
|
List<Clue> clueList = clueJpa.findClue(taskIds, null);
|
|
|
|
|
Map<String, List<Clue>> map = clueList.stream().collect(Collectors.groupingBy(Clue::getNid));
|
|
|
|
|
Set<String> keySet = map.keySet();
|
|
|
|
|
for (String key : keySet) {
|
|
|
|
|
if (map.get(key).size() > 1){
|
|
|
|
|
if (map.get(key).size() > 1) {
|
|
|
|
|
List<Clue> clues = map.get(key);
|
|
|
|
|
clues.remove(0);
|
|
|
|
|
Set<Long> clueIdSet = new HashSet<>();
|
|
|
|
|