From 88d66726c49c53edf24b4c5852ee67aaa48552a8 Mon Sep 17 00:00:00 2001 From: yqy Date: Thu, 23 Dec 2021 17:15:44 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B5=84=E6=BA=90=E6=9C=8D=E5=8A=A1=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/UploadFileServiceImpl.java | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java diff --git a/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java new file mode 100644 index 00000000..04da60c1 --- /dev/null +++ b/services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java @@ -0,0 +1,283 @@ +package com.baiye.module.service.impl; + +import cn.hutool.core.text.csv.CsvData; +import cn.hutool.core.text.csv.CsvRow; +import cn.hutool.core.text.csv.CsvUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.poi.excel.ExcelReader; +import cn.hutool.poi.excel.ExcelUtil; +import com.baiye.exception.BadRequestException; +import com.baiye.feign.TaskClient; +import com.baiye.http.CommonResponse; +import com.baiye.http.ResponseCode; +import com.baiye.module.dao.ClueRecordRepository; +import com.baiye.module.entity.ClueRecord; +import com.baiye.module.entity.Task; +import com.baiye.module.service.UploadFileService; +import com.baiye.task.FileAnalysisTask; +import com.baiye.module.constant.FileConstant; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import java.io.*; +import java.nio.charset.Charset; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; + +@Service +@Slf4j +public class UploadFileServiceImpl implements UploadFileService { + + @Resource + private FileAnalysisTask fileAnalysisTask; + @Resource + private ClueRecordRepository clueRecordRepository; + @Resource + private TaskClient taskClient; + + @Value(value = "${storage.url}") + private String URL; + @Value(value = "${storage.de_symbol}") + private String DE_SYMBOL; + @Value("${snowflake.workerId}") + private int workerId; + @Value("${snowflake.datacenterId}") + private int datacenterId; + + private static SimpleDateFormat timeOne = new SimpleDateFormat("yyyyMMddHHmmssSSS"); + private static SimpleDateFormat timeTwo=new SimpleDateFormat("yyyy-MM-dd"); + private static Random random = new Random(); + + + /** + * 文件上传 + * @param files + * @param uploadType + * @param userId + * @return + */ + @Override + @SneakyThrows //处理异常try + @Transactional(rollbackFor = Exception.class) + public ResponseEntity singleFileUpload(MultipartFile[] files, Integer uploadType,Long userId) { + + List clueRecords = new ArrayList<>(); + int totalNum = 0; + // fixme controller层已经判空，可直接取第一个文件名拼接任务名称 + String oneFileName = files[0].getOriginalFilename(); + long taskId = IdUtil.getSnowflake(workerId, datacenterId).nextId(); + //处理文件数据 + for (MultipartFile file : files) { + String name=file.getOriginalFilename(); + int lastIndexOf = name.lastIndexOf("."); + String nameStr = name.substring(lastIndexOf); + // 校验文件格式 + switch (uploadType){ + case FileConstant.ONE_NUMBER: + if (!nameStr.equals(".csv")){ + throw new BadRequestException("文件格式错误！, 请上传csv格式"); + } + break; + default: + if (!((nameStr.equals(".xlsx")|| nameStr.equals(".xls")))){ + throw new BadRequestException("文件格式错误！, 请上传xlsx、xls、csv格式"); + } + break; + } + //检测excel + if (nameStr.equals(".xlsx") || nameStr.equals(".xls")) { + ExcelReader reader = ExcelUtil.getReader(file.getInputStream()); + //检验excel内容格式(表头信息) + List excelHeadList = reader.readRow(0); + Boolean flag = testingFormat(excelHeadList, uploadType, FileConstant.ZERO_NUMBER); + if (!flag){ + throw new BadRequestException("文件内容格式上传错误"); + } + //检测文件不超过100w行 + int rowCount = reader.getRowCount() - FileConstant.ONE_NUMBER; + if (reader.getSheetCount() > FileConstant.ONE_NUMBER) { + // fixme 检测多个sheet，i从1开始，自动过滤第一个sheet(第一个上面已经读过一次) + for (int i = 1; i < reader.getSheetCount(); i++) { + ExcelReader readerSheet = ExcelUtil.getReader(file.getInputStream(),i); + //-1减去表头行计算 + rowCount = readerSheet.getRowCount() - FileConstant.ONE_NUMBER + rowCount; + } + } + if (rowCount > FileConstant.ONE_MILLION_NUMBER) { + log.info("============excel行数大于100w====================="); + throw new BadRequestException("文件行数不能超过100w行"); + } + totalNum += rowCount; + } + //存入本地 + String currentDate = timeTwo.format(new Date()); + String newRealPath = URL + currentDate; + File newRealPathFile = new File(newRealPath); + if (!newRealPathFile.exists()){ + newRealPathFile.mkdir(); + } + String newName = randomFileName(name); + File newFile = new File(newRealPathFile, newName); + file.transferTo(newFile); + + //检测csv文件(不满足记得delete临时文件) + if (nameStr.equals(".csv")) { + CsvData data = CsvUtil.getReader().read(newFile, Charset.forName("gbk")); + List rows = data.getRows(); + //检验csv内容格式(表头信息),传的是行数，下面取第一行判断 + Boolean flag = testingFormat(rows, uploadType, FileConstant.ONE_NUMBER); + if (!flag){ + newFile.delete(); + throw new BadRequestException("文件内容格式上传错误"); + } + //行数不能超过100w行 + if (rows.size() > FileConstant.ONE_MILLION_NUMBER) { + log.info("============csv行数大于100w====================="); + newFile.delete(); + throw new BadRequestException("文件行数不能超过100w行"); + } + totalNum += (rows.size() - 1); + } + //保存上传记录信息 + long clueRecordId = IdUtil.getSnowflake(workerId, datacenterId).nextId(); + ClueRecord clueRecord = new ClueRecord(); + clueRecord.setId(clueRecordId); + clueRecord.setOldFile(name); + clueRecord.setNewFileUrl(newRealPath + DE_SYMBOL + newName); + clueRecord.setStatus(true); + clueRecord.setUploadType(uploadType); + clueRecord.setCreateBy(userId); + clueRecord.setTaskId(taskId); + + clueRecords.add(clueRecord); + clueRecordRepository.save(clueRecord); + } + //创建任务 + saveTask(userId,taskId,oneFileName,totalNum); + //异步读取文件 + fileAnalysisTask.runFileAnalysisTask(clueRecords); + return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.SUCCESS),HttpStatus.OK); + } + + /** + * 创建任务 + * @param userId + * @param taskId + * @param oneFileName + * @param totalNum + */ + public void saveTask(Long userId, long taskId ,String oneFileName,int totalNum){ + int lastIndexOf = oneFileName.lastIndexOf("."); + String nameStr = oneFileName.substring(0,lastIndexOf); + String taskName = nameStr + timeOne.format(new Date()); + Task task = new Task(); + task.setId(taskId); + task.setCreateBy(userId); + task.setTaskName(taskName); + task.setTotalNumber(totalNum); + taskClient.saveTask(task); + } + + /** + * 检测文件内容格式 + * @param list + * @param uploadType + * @param format + * @return + */ + public Boolean testingFormat(List list,Integer uploadType,Integer format){ + //0检测excel + if (format == FileConstant.ZERO_NUMBER){ + switch (uploadType){ + //检测自定义表单头 + case FileConstant.ZERO_NUMBER: + if (list.size() >=6) { + String zdyName = String.valueOf(list.get(0)); + String zdyNid = String.valueOf(list.get(1)); + String zdyWx = String.valueOf(list.get(2)); + String zdyAmount = String.valueOf(list.get(3)); + String zdyCollectTime = String.valueOf(list.get(4)); + String zdyAddress = String.valueOf(list.get(5)); + if (!(zdyName.equals("姓名") && zdyNid.equals("电话") && zdyWx.equals("微信") && zdyCollectTime.equals("日期") + && zdyAddress.equals("线索归属地") && zdyAmount.equals("金额"))) { + log.info("======================自定义单内容格式不正确====================="); + return false; + } + }else{ + return false; + } + break; + //检测快手表单头 + case FileConstant.TWO_NUMBER: + if (list.size() >= 25) { + String ksName = String.valueOf(list.get(3)); + String ksNid = String.valueOf(list.get(4)); + String ksWx = String.valueOf(list.get(5)); + String ksCollectTime = String.valueOf(list.get(11)); + String ksAddress = String.valueOf(list.get(24)); + if (!(ksName.equals("姓名") && ksNid.equals("电话") && ksWx.equals("微信") && ksCollectTime.equals("收集日期") + && ksAddress.equals("线索归属地"))) { + log.info("======================快手表单内容格式不正确====================="); + return false; + } + }else { + return false; + } + break; + default: + break; + } + } + //1检测csv + if (format == FileConstant.ONE_NUMBER){ + switch (uploadType){ + //检测UC表单头 + case FileConstant.ONE_NUMBER: + CsvRow csvRow = (CsvRow) list.get(0); + if (csvRow.size() >= 6) { + String collectTime = csvRow.get(2); + String name = csvRow.get(3); + String nid = csvRow.get(4); + String amount = csvRow.get(5); + if (!(name.equals("第一行") && nid.equals("第二行") && collectTime.equals("时间") && amount.equals("第三行"))) { + log.info("======================UC表单内容格式不正确====================="); + return false; + } + }else { + return false; + } + break; + default: + break; + } + } + return true; + } + + /** + * 生成 时间+随机数 文件名 + * @param old + * @return + */ + public static String randomFileName(String old){ + + // 1.得到老文件.后面的文件夹后缀名 如：.jsp .png + String suffix = old.substring(old.lastIndexOf("."), old.length()); + // 2.4位的随机数1000--10000的数字 + int num = random.nextInt(9000)+1000; + // 3.得到当前时间 + String format = timeOne.format(new Date()); + // 4.时间 + _ + 随机数 + 后缀名 返回一个新的文件名 + return format+"_"+num+suffix; + } +}