|
|
|
|
@ -10,18 +10,20 @@ import cn.hutool.core.util.IdUtil;
|
|
|
|
|
import cn.hutool.poi.excel.ExcelReader;
|
|
|
|
|
import cn.hutool.poi.excel.ExcelUtil;
|
|
|
|
|
import com.baiye.exception.BadRequestException;
|
|
|
|
|
import com.baiye.feign.TaskClient;
|
|
|
|
|
import com.baiye.http.CommonResponse;
|
|
|
|
|
import com.baiye.http.ResponseCode;
|
|
|
|
|
import com.baiye.module.dao.ClueRecordRepository;
|
|
|
|
|
import com.baiye.module.entity.ClueRecord;
|
|
|
|
|
import com.baiye.module.entity.Task;
|
|
|
|
|
import com.baiye.module.entity.vo.BaseExcelVo;
|
|
|
|
|
import com.baiye.module.service.ClueService;
|
|
|
|
|
import com.baiye.module.service.UploadFileService;
|
|
|
|
|
import com.baiye.task.FileAnalysisTask;
|
|
|
|
|
import com.baiye.module.constant.FileConstant;
|
|
|
|
|
import com.baiye.util.FileZipUtil;
|
|
|
|
|
import com.baiye.util.TestingUtil;
|
|
|
|
|
import lombok.SneakyThrows;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.apache.commons.lang.StringUtils;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
|
|
import org.springframework.http.HttpStatus;
|
|
|
|
|
import org.springframework.http.ResponseEntity;
|
|
|
|
|
@ -48,7 +50,7 @@ public class UploadFileServiceImpl implements UploadFileService {
|
|
|
|
|
@Resource
|
|
|
|
|
private ClueRecordRepository clueRecordRepository;
|
|
|
|
|
@Resource
|
|
|
|
|
private TaskClient taskClient;
|
|
|
|
|
private ClueService clueService;
|
|
|
|
|
|
|
|
|
|
@Value(value = "${storage.url}")
|
|
|
|
|
private String URL;
|
|
|
|
|
@ -77,52 +79,19 @@ public class UploadFileServiceImpl implements UploadFileService {
|
|
|
|
|
@Transactional(rollbackFor = Exception.class)
|
|
|
|
|
public ResponseEntity<Object> singleFileUpload(MultipartFile[] files, Integer uploadType, Long userId) {
|
|
|
|
|
|
|
|
|
|
List<ClueRecord> clueRecords = new ArrayList<>();
|
|
|
|
|
int totalNum = 0;
|
|
|
|
|
List<ClueRecord> clueRecordList = new ArrayList<>();
|
|
|
|
|
String oneFileName = files[0].getOriginalFilename();
|
|
|
|
|
long taskId = IdUtil.getSnowflake(workerId, datacenterId).nextId();
|
|
|
|
|
Long taskId = IdUtil.getSnowflake(workerId, datacenterId).nextId();
|
|
|
|
|
//处理文件数据
|
|
|
|
|
for (MultipartFile file : files) {
|
|
|
|
|
String name = file.getOriginalFilename();
|
|
|
|
|
int lastIndexOf = name.lastIndexOf(".");
|
|
|
|
|
String nameStr = name.substring(lastIndexOf);
|
|
|
|
|
// 校验文件格式
|
|
|
|
|
switch (uploadType) {
|
|
|
|
|
case FileConstant.ONE_NUMBER:
|
|
|
|
|
if (!nameStr.equals(".csv")) {
|
|
|
|
|
throw new BadRequestException("文件格式错误！, 请上传csv格式");
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
if (!((nameStr.equals(".xlsx") || nameStr.equals(".xls")))) {
|
|
|
|
|
throw new BadRequestException("文件格式错误！, 请上传xlsx、xls、csv格式");
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
testingSuffix(uploadType, nameStr);
|
|
|
|
|
//检测excel
|
|
|
|
|
if (nameStr.equals(".xlsx") || nameStr.equals(".xls")) {
|
|
|
|
|
ExcelReader reader = ExcelUtil.getReader(file.getInputStream());
|
|
|
|
|
//检验excel内容格式(表头信息)
|
|
|
|
|
List<Object> excelHeadList = reader.readRow(0);
|
|
|
|
|
Boolean flag = testingFormat(excelHeadList, uploadType, FileConstant.ZERO_NUMBER);
|
|
|
|
|
if (!flag) {
|
|
|
|
|
throw new BadRequestException("文件内容格式上传错误");
|
|
|
|
|
}
|
|
|
|
|
//检测文件不超过100w行
|
|
|
|
|
int rowCount = reader.getRowCount() - FileConstant.ONE_NUMBER;
|
|
|
|
|
if (reader.getSheetCount() > FileConstant.ONE_NUMBER) {
|
|
|
|
|
// 检测多个sheet，i从1开始，自动过滤第一个sheet(第一个上面已经读过一次)
|
|
|
|
|
for (int i = 1; i < reader.getSheetCount(); i++) {
|
|
|
|
|
ExcelReader readerSheet = ExcelUtil.getReader(file.getInputStream(), i);
|
|
|
|
|
//-1减去表头行计算
|
|
|
|
|
rowCount = readerSheet.getRowCount() - FileConstant.ONE_NUMBER + rowCount;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (rowCount > FileConstant.ONE_MILLION_NUMBER) {
|
|
|
|
|
log.info("============excel行数大于100w=====================");
|
|
|
|
|
throw new BadRequestException("文件行数不能超过100w行");
|
|
|
|
|
}
|
|
|
|
|
totalNum += rowCount;
|
|
|
|
|
testingExcel(file, uploadType);
|
|
|
|
|
}
|
|
|
|
|
//存入本地
|
|
|
|
|
String currentDate = timeTwo.format(new Date());
|
|
|
|
|
@ -134,24 +103,9 @@ public class UploadFileServiceImpl implements UploadFileService {
|
|
|
|
|
String newName = randomFileName(name);
|
|
|
|
|
File newFile = new File(newRealPathFile, newName);
|
|
|
|
|
file.transferTo(newFile);
|
|
|
|
|
|
|
|
|
|
//检测csv文件(不满足记得delete临时文件)
|
|
|
|
|
//检测csv文件
|
|
|
|
|
if (nameStr.equals(".csv")) {
|
|
|
|
|
CsvData data = CsvUtil.getReader().read(newFile, Charset.forName("gbk"));
|
|
|
|
|
List<CsvRow> rows = data.getRows();
|
|
|
|
|
//检验csv内容格式(表头信息),传的是行数，下面取第一行判断
|
|
|
|
|
Boolean flag = testingFormat(rows, uploadType, FileConstant.ONE_NUMBER);
|
|
|
|
|
if (!flag) {
|
|
|
|
|
newFile.delete();
|
|
|
|
|
throw new BadRequestException("文件内容格式上传错误");
|
|
|
|
|
}
|
|
|
|
|
//行数不能超过100w行
|
|
|
|
|
if (rows.size() > FileConstant.ONE_MILLION_NUMBER) {
|
|
|
|
|
log.info("============csv行数大于100w=====================");
|
|
|
|
|
newFile.delete();
|
|
|
|
|
throw new BadRequestException("文件行数不能超过100w行");
|
|
|
|
|
}
|
|
|
|
|
totalNum += (rows.size() - 1);
|
|
|
|
|
testingCsv(newFile, uploadType);
|
|
|
|
|
}
|
|
|
|
|
//保存上传记录信息
|
|
|
|
|
long clueRecordId = IdUtil.getSnowflake(workerId, datacenterId).nextId();
|
|
|
|
|
@ -163,116 +117,97 @@ public class UploadFileServiceImpl implements UploadFileService {
|
|
|
|
|
clueRecord.setUploadType(uploadType);
|
|
|
|
|
clueRecord.setCreateBy(userId);
|
|
|
|
|
clueRecord.setTaskId(taskId);
|
|
|
|
|
|
|
|
|
|
clueRecords.add(clueRecord);
|
|
|
|
|
clueRecordList.add(clueRecord);
|
|
|
|
|
clueRecordRepository.save(clueRecord);
|
|
|
|
|
}
|
|
|
|
|
//创建任务
|
|
|
|
|
saveTask(userId, taskId, oneFileName, totalNum);
|
|
|
|
|
//异步读取文件
|
|
|
|
|
fileAnalysisTask.runFileAnalysisTask(clueRecords);
|
|
|
|
|
//异步读取文件并创建任务
|
|
|
|
|
fileAnalysisTask.runFileAnalysisTask(clueRecordList, oneFileName);
|
|
|
|
|
return new ResponseEntity<>(CommonResponse.createBySuccess(ResponseCode.SUCCESS), HttpStatus.OK);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 创建任务
|
|
|
|
|
* 检测导入类型文件后缀是否合法
|
|
|
|
|
*
|
|
|
|
|
* @param userId
|
|
|
|
|
* @param taskId
|
|
|
|
|
* @param oneFileName
|
|
|
|
|
* @param totalNum
|
|
|
|
|
* @param uploadType
|
|
|
|
|
*/
|
|
|
|
|
public void saveTask(Long userId, long taskId, String oneFileName, int totalNum) {
|
|
|
|
|
int lastIndexOf = oneFileName.lastIndexOf(".");
|
|
|
|
|
String nameStr = oneFileName.substring(0, lastIndexOf);
|
|
|
|
|
String taskName = nameStr + timeOne.format(new Date());
|
|
|
|
|
Task task = new Task();
|
|
|
|
|
task.setId(taskId);
|
|
|
|
|
task.setCreateBy(userId);
|
|
|
|
|
task.setTaskName(taskName);
|
|
|
|
|
task.setTotalNumber(totalNum);
|
|
|
|
|
taskClient.saveTask(task);
|
|
|
|
|
public void testingSuffix(Integer uploadType, String nameStr) {
|
|
|
|
|
switch (uploadType) {
|
|
|
|
|
case FileConstant.ONE_NUMBER:
|
|
|
|
|
if (!nameStr.equals(".csv")) {
|
|
|
|
|
throw new BadRequestException("文件格式错误！, 请上传csv格式");
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
if (!((nameStr.equals(".xlsx") || nameStr.equals(".xls")))) {
|
|
|
|
|
throw new BadRequestException("文件格式错误！, 请上传xlsx、xls格式");
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 检测文件内容格式
|
|
|
|
|
* 检测Excel文件格式，内容，量
|
|
|
|
|
*
|
|
|
|
|
* @param list
|
|
|
|
|
* @param uploadType
|
|
|
|
|
* @param format
|
|
|
|
|
* @return
|
|
|
|
|
* @param file 文件
|
|
|
|
|
* @param uploadType 线索文件类型(UC..)
|
|
|
|
|
*/
|
|
|
|
|
public Boolean testingFormat(List list, Integer uploadType, Integer format) {
|
|
|
|
|
//0检测excel
|
|
|
|
|
if (format == FileConstant.ZERO_NUMBER) {
|
|
|
|
|
switch (uploadType) {
|
|
|
|
|
//检测自定义表单头
|
|
|
|
|
case FileConstant.ZERO_NUMBER:
|
|
|
|
|
if (list.size() >= 6) {
|
|
|
|
|
String zdyName = String.valueOf(list.get(0));
|
|
|
|
|
String zdyNid = String.valueOf(list.get(1));
|
|
|
|
|
String zdyWx = String.valueOf(list.get(2));
|
|
|
|
|
String zdyAmount = String.valueOf(list.get(3));
|
|
|
|
|
String zdyCollectTime = String.valueOf(list.get(4));
|
|
|
|
|
String zdyAddress = String.valueOf(list.get(5));
|
|
|
|
|
if (!(zdyName.equals("姓名") && zdyNid.equals("电话") && zdyWx.equals("微信") && zdyCollectTime.equals("日期")
|
|
|
|
|
&& zdyAddress.equals("线索归属地") && zdyAmount.equals("金额"))) {
|
|
|
|
|
log.info("======================自定义单内容格式不正确=====================");
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
//检测快手表单头
|
|
|
|
|
case FileConstant.TWO_NUMBER:
|
|
|
|
|
if (list.size() >= 25) {
|
|
|
|
|
String ksName = String.valueOf(list.get(3));
|
|
|
|
|
String ksNid = String.valueOf(list.get(4));
|
|
|
|
|
String ksWx = String.valueOf(list.get(5));
|
|
|
|
|
String ksCollectTime = String.valueOf(list.get(11));
|
|
|
|
|
String ksAddress = String.valueOf(list.get(24));
|
|
|
|
|
if (!(ksName.equals("姓名") && ksNid.equals("电话") && ksWx.equals("微信") && ksCollectTime.equals("收集日期")
|
|
|
|
|
&& ksAddress.equals("线索归属地"))) {
|
|
|
|
|
log.info("======================快手表单内容格式不正确=====================");
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
public void testingExcel(MultipartFile file, Integer uploadType) throws IOException {
|
|
|
|
|
ExcelReader reader = ExcelUtil.getReader(file.getInputStream());
|
|
|
|
|
//检验excel内容格式(表头信息)
|
|
|
|
|
List<Object> excelHeadList = reader.readRow(0);
|
|
|
|
|
Boolean flag = TestingUtil.testingFormat(excelHeadList, uploadType, FileConstant.ZERO_NUMBER);
|
|
|
|
|
if (!flag) {
|
|
|
|
|
throw new BadRequestException("文件内容格式上传错误");
|
|
|
|
|
}
|
|
|
|
|
//1检测csv
|
|
|
|
|
if (format == FileConstant.ONE_NUMBER) {
|
|
|
|
|
switch (uploadType) {
|
|
|
|
|
//检测UC表单头
|
|
|
|
|
case FileConstant.ONE_NUMBER:
|
|
|
|
|
CsvRow csvRow = (CsvRow) list.get(0);
|
|
|
|
|
if (csvRow.size() >= 6) {
|
|
|
|
|
String collectTime = csvRow.get(2);
|
|
|
|
|
String name = csvRow.get(3);
|
|
|
|
|
String nid = csvRow.get(4);
|
|
|
|
|
String amount = csvRow.get(5);
|
|
|
|
|
if (!(name.equals("第一行") && nid.equals("第二行") && collectTime.equals("时间") && amount.equals("第三行"))) {
|
|
|
|
|
log.info("======================UC表单内容格式不正确=====================");
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
//检测文件不超过100w行
|
|
|
|
|
int rowCount = reader.getRowCount() - FileConstant.ONE_NUMBER;
|
|
|
|
|
if (reader.getSheetCount() > FileConstant.ONE_NUMBER) {
|
|
|
|
|
// 检测多个sheet，i从1开始，自动过滤第一个sheet(第一个上面已经读过一次)
|
|
|
|
|
for (int i = 1; i < reader.getSheetCount(); i++) {
|
|
|
|
|
ExcelReader readerSheet = ExcelUtil.getReader(file.getInputStream(), i);
|
|
|
|
|
//-1减去表头行计算
|
|
|
|
|
rowCount = readerSheet.getRowCount() - FileConstant.ONE_NUMBER + rowCount;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
if (rowCount <= FileConstant.ONE_NUMBER) {
|
|
|
|
|
throw new BadRequestException("文件没有内容，无法生成任务");
|
|
|
|
|
}
|
|
|
|
|
if (rowCount > FileConstant.ONE_MILLION_NUMBER) {
|
|
|
|
|
log.info("============excel行数大于100w=====================");
|
|
|
|
|
throw new BadRequestException("文件行数不能超过100w行");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 检测CSV文件格式，内容，量(不满足要delete临时文件)
|
|
|
|
|
*
|
|
|
|
|
* @param newFile 文件
|
|
|
|
|
* @param uploadType 线索文件类型(UC..)
|
|
|
|
|
*/
|
|
|
|
|
public void testingCsv(File newFile, Integer uploadType) {
|
|
|
|
|
CsvData data = CsvUtil.getReader().read(newFile, Charset.forName("gbk"));
|
|
|
|
|
List<CsvRow> rows = data.getRows();
|
|
|
|
|
//检验csv内容格式(表头信息),传的是行数，下面取第一行判断
|
|
|
|
|
Boolean flag = TestingUtil.testingFormat(rows, uploadType, FileConstant.ONE_NUMBER);
|
|
|
|
|
if (!flag) {
|
|
|
|
|
newFile.delete();
|
|
|
|
|
throw new BadRequestException("文件内容格式上传错误");
|
|
|
|
|
}
|
|
|
|
|
if (rows.size() <= FileConstant.ONE_NUMBER) {
|
|
|
|
|
newFile.delete();
|
|
|
|
|
throw new BadRequestException("文件没有内容，无法生成任务");
|
|
|
|
|
}
|
|
|
|
|
//行数不能超过100w行
|
|
|
|
|
if (rows.size() > FileConstant.ONE_MILLION_NUMBER) {
|
|
|
|
|
log.info("============csv行数大于100w=====================");
|
|
|
|
|
newFile.delete();
|
|
|
|
|
throw new BadRequestException("文件行数不能超过100w行");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 压缩前一天文件夹(每天3点)
|
|
|
|
|
*
|
|
|
|
|
* @throws Exception
|
|
|
|
|
*/
|
|
|
|
|
@Scheduled(cron = "0 0 3 * * ? ")
|
|
|
|
|
@ -281,8 +216,8 @@ public class UploadFileServiceImpl implements UploadFileService {
|
|
|
|
|
DateTime date = DateUtil.yesterday();
|
|
|
|
|
String path = URL + dateFormat.format(date);
|
|
|
|
|
boolean directory = FileUtil.isDirectory(path);
|
|
|
|
|
if (directory){
|
|
|
|
|
FileZipUtil.zipFile(new File(path),"zip");
|
|
|
|
|
if (directory) {
|
|
|
|
|
FileZipUtil.zipFile(new File(path), "zip");
|
|
|
|
|
}
|
|
|
|
|
log.info("-----------------------------压缩文件夹开始时间{}-----------------------------", DateUtil.now());
|
|
|
|
|
}
|
|
|
|
|
|
yqy
3 years ago