BY-Team
/
ad-platform
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

资源服务代码提交

Browse Source
master
yqy 3 years ago
parent 91aee40a6a
commit 88d66726c4
1 changed files with 283 additions and 0 deletions
Show Stats Download Patch File Download Diff File

283
services/ad-platform-source/src/main/java/com/baiye/module/service/impl/UploadFileServiceImpl.java
Unescape Escape View File

@ -0,0 +1,283 @@
package com.baiye.module.service.impl;
import cn.hutool.core.text.csv.CsvData;
import cn.hutool.core.text.csv.CsvRow;
import cn.hutool.core.text.csv.CsvUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.poi.excel.ExcelReader;
import cn.hutool.poi.excel.ExcelUtil;
import com.baiye.exception.BadRequestException;
import com.baiye.feign.TaskClient;
import com.baiye.http.CommonResponse;
import com.baiye.http.ResponseCode;
import com.baiye.module.dao.ClueRecordRepository;
import com.baiye.module.entity.ClueRecord;
import com.baiye.module.entity.Task;
import com.baiye.module.service.UploadFileService;
import com.baiye.task.FileAnalysisTask;
import com.baiye.module.constant.FileConstant;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.*;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
@Service
@Slf4j
public class UploadFileServiceImpl implements UploadFileService {
@Resource
private FileAnalysisTask fileAnalysisTask;
@Resource
private ClueRecordRepository clueRecordRepository;
@Resource
private TaskClient taskClient;
@Value(value = "${storage.url}")
private String URL;
@Value(value = "${storage.de_symbol}")
private String DE_SYMBOL;
@Value("${snowflake.workerId}")
private int workerId;
@Value("${snowflake.datacenterId}")
private int datacenterId;
private static SimpleDateFormat timeOne = new SimpleDateFormat("yyyyMMddHHmmssSSS");
private static SimpleDateFormat timeTwo=new SimpleDateFormat("yyyy-MM-dd");
private static Random random = new Random();
/**
* 文件上传
* @param files
* @param uploadType
* @param userId
* @return
*/
@Override
@SneakyThrows //处理异常try
@Transactional(rollbackFor = Exception.class)
public ResponseEntity<Object> singleFileUpload(MultipartFile[] files, Integer uploadType,Long userId) {
List<ClueRecord> clueRecords = new ArrayList<>();
int totalNum = 0;
// fixme controller层已经判空，可直接取第一个文件名拼接任务名称
String oneFileName = files[0].getOriginalFilename();
long taskId = IdUtil.getSnowflake(workerId, datacenterId).nextId();
//处理文件数据
for (MultipartFile file : files) {
String name=file.getOriginalFilename();
int lastIndexOf = name.lastIndexOf(".");
String nameStr = name.substring(lastIndexOf);
// 校验文件格式
switch (uploadType){
case FileConstant.ONE_NUMBER:
if (!nameStr.equals(".csv")){
throw new BadRequestException("文件格式错误！, 请上传csv格式");
}
break;
default:
if (!((nameStr.equals(".xlsx")|| nameStr.equals(".xls")))){
throw new BadRequestException("文件格式错误！, 请上传xlsx、xls、csv格式");
}
break;
}
//检测excel
if (nameStr.equals(".xlsx") || nameStr.equals(".xls")) {
ExcelReader reader = ExcelUtil.getReader(file.getInputStream());
//检验excel内容格式(表头信息)
List<Object> excelHeadList = reader.readRow(0);
Boolean flag = testingFormat(excelHeadList, uploadType, FileConstant.ZERO_NUMBER);
if (!flag){
throw new BadRequestException("文件内容格式上传错误");
}
//检测文件不超过100w行
int rowCount = reader.getRowCount() - FileConstant.ONE_NUMBER;
if (reader.getSheetCount() > FileConstant.ONE_NUMBER) {
// fixme 检测多个sheet，i从1开始，自动过滤第一个sheet(第一个上面已经读过一次)
for (int i = 1; i < reader.getSheetCount(); i++) {
ExcelReader readerSheet = ExcelUtil.getReader(file.getInputStream(),i);
//-1减去表头行计算
rowCount = readerSheet.getRowCount() - FileConstant.ONE_NUMBER + rowCount;
}
}
if (rowCount > FileConstant.ONE_MILLION_NUMBER) {
log.info("============excel行数大于100w=====================");
throw new BadRequestException("文件行数不能超过100w行");
}
totalNum += rowCount;
}
//存入本地
String currentDate = timeTwo.format(new Date());
String newRealPath = URL + currentDate;
File newRealPathFile = new File(newRealPath);
if (!newRealPathFile.exists()){
newRealPathFile.mkdir();
}
String newName = randomFileName(name);
File newFile = new File(newRealPathFile, newName);
file.transferTo(newFile);
//检测csv文件(不满足记得delete临时文件)
if (nameStr.equals(".csv")) {
CsvData data = CsvUtil.getReader().read(newFile, Charset.forName("gbk"));
List<CsvRow> rows = data.getRows();
//检验csv内容格式(表头信息),传的是行数，下面取第一行判断
Boolean flag = testingFormat(rows, uploadType, FileConstant.ONE_NUMBER);
if (!flag){
newFile.delete();
throw new BadRequestException("文件内容格式上传错误");
}
//行数不能超过100w行
if (rows.size() > FileConstant.ONE_MILLION_NUMBER) {
log.info("============csv行数大于100w=====================");
newFile.delete();
throw new BadRequestException("文件行数不能超过100w行");
}
totalNum += (rows.size() - 1);
}
//保存上传记录信息
long clueRecordId = IdUtil.getSnowflake(workerId, datacenterId).nextId();
ClueRecord clueRecord = new ClueRecord();
clueRecord.setId(clueRecordId);
clueRecord.setOldFile(name);
clueRecord.setNewFileUrl(newRealPath + DE_SYMBOL + newName);
clueRecord.setStatus(true);
clueRecord.setUploadType(uploadType);
clueRecord.setCreateBy(userId);
clueRecord.setTaskId(taskId);
clueRecords.add(clueRecord);
clueRecordRepository.save(clueRecord);
}
//创建任务
saveTask(userId,taskId,oneFileName,totalNum);
//异步读取文件
fileAnalysisTask.runFileAnalysisTask(clueRecords);
return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.SUCCESS),HttpStatus.OK);
}
/**
* 创建任务
* @param userId
* @param taskId
* @param oneFileName
* @param totalNum
*/
public void saveTask(Long userId, long taskId ,String oneFileName,int totalNum){
int lastIndexOf = oneFileName.lastIndexOf(".");
String nameStr = oneFileName.substring(0,lastIndexOf);
String taskName = nameStr + timeOne.format(new Date());
Task task = new Task();
task.setId(taskId);
task.setCreateBy(userId);
task.setTaskName(taskName);
task.setTotalNumber(totalNum);
taskClient.saveTask(task);
}
/**
* 检测文件内容格式
* @param list
* @param uploadType
* @param format
* @return
*/
public Boolean testingFormat(List list,Integer uploadType,Integer format){
//0检测excel
if (format == FileConstant.ZERO_NUMBER){
switch (uploadType){
//检测自定义表单头
case FileConstant.ZERO_NUMBER:
if (list.size() >=6) {
String zdyName = String.valueOf(list.get(0));
String zdyNid = String.valueOf(list.get(1));
String zdyWx = String.valueOf(list.get(2));
String zdyAmount = String.valueOf(list.get(3));
String zdyCollectTime = String.valueOf(list.get(4));
String zdyAddress = String.valueOf(list.get(5));
if (!(zdyName.equals("姓名") && zdyNid.equals("电话") && zdyWx.equals("微信") && zdyCollectTime.equals("日期")
&& zdyAddress.equals("线索归属地") && zdyAmount.equals("金额"))) {
log.info("======================自定义单内容格式不正确=====================");
return false;
}
}else{
return false;
}
break;
//检测快手表单头
case FileConstant.TWO_NUMBER:
if (list.size() >= 25) {
String ksName = String.valueOf(list.get(3));
String ksNid = String.valueOf(list.get(4));
String ksWx = String.valueOf(list.get(5));
String ksCollectTime = String.valueOf(list.get(11));
String ksAddress = String.valueOf(list.get(24));
if (!(ksName.equals("姓名") && ksNid.equals("电话") && ksWx.equals("微信") && ksCollectTime.equals("收集日期")
&& ksAddress.equals("线索归属地"))) {
log.info("======================快手表单内容格式不正确=====================");
return false;
}
}else {
return false;
}
break;
default:
break;
}
}
//1检测csv
if (format == FileConstant.ONE_NUMBER){
switch (uploadType){
//检测UC表单头
case FileConstant.ONE_NUMBER:
CsvRow csvRow = (CsvRow) list.get(0);
if (csvRow.size() >= 6) {
String collectTime = csvRow.get(2);
String name = csvRow.get(3);
String nid = csvRow.get(4);
String amount = csvRow.get(5);
if (!(name.equals("第一行") && nid.equals("第二行") && collectTime.equals("时间") && amount.equals("第三行"))) {
log.info("======================UC表单内容格式不正确=====================");
return false;
}
}else {
return false;
}
break;
default:
break;
}
}
return true;
}
/**
* 生成 时间+随机数 文件名
* @param old
* @return
*/
public static String randomFileName(String old){
// 1.得到老文件.后面的文件夹后缀名 如：.jsp .png
String suffix = old.substring(old.lastIndexOf("."), old.length());
// 2.4位的随机数1000--10000的数字
int num = random.nextInt(9000)+1000;
// 3.得到当前时间
String format = timeOne.format(new Date());
// 4.时间 + _ + 随机数 + 后缀名 返回一个新的文件名
return format+"_"+num+suffix;
}
}
Write Preview
Loading…
Cancel
Save