|
|
|
|
@ -1,14 +1,37 @@
|
|
|
|
|
package me.zhengjie.task;
|
|
|
|
|
|
|
|
|
|
import cn.hutool.core.util.RandomUtil;
|
|
|
|
|
import cn.hutool.system.OsInfo;
|
|
|
|
|
import cn.hutool.system.SystemUtil;
|
|
|
|
|
import com.alibaba.fastjson.JSON;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import me.zhengjie.modules.abmessage.domain.AbMessage;
|
|
|
|
|
import me.zhengjie.modules.abmessage.service.AbMessageService;
|
|
|
|
|
import me.zhengjie.modules.form.domain.FormMessage;
|
|
|
|
|
import me.zhengjie.modules.form.service.FormMessageService;
|
|
|
|
|
import me.zhengjie.modules.taskrecord.domain.TaskRecord;
|
|
|
|
|
import org.apache.poi.ss.formula.functions.T;
|
|
|
|
|
import me.zhengjie.modules.taskrecord.service.TaskRecordService;
|
|
|
|
|
import me.zhengjie.utils.FileUtil;
|
|
|
|
|
import me.zhengjie.utils.StringUtils;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
|
|
import org.springframework.data.domain.Example;
|
|
|
|
|
import org.springframework.data.domain.PageRequest;
|
|
|
|
|
import org.springframework.data.domain.Pageable;
|
|
|
|
|
import org.springframework.data.domain.Slice;
|
|
|
|
|
import org.springframework.scheduling.annotation.Async;
|
|
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
import org.springframework.util.CollectionUtils;
|
|
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.nio.file.Files;
|
|
|
|
|
import java.nio.file.Paths;
|
|
|
|
|
import java.nio.file.StandardOpenOption;
|
|
|
|
|
import java.time.Instant;
|
|
|
|
|
import java.time.LocalDateTime;
|
|
|
|
|
import java.time.ZoneOffset;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 生成本地文件的异步执行任务类
|
|
|
|
|
@ -17,6 +40,65 @@ import java.util.List;
|
|
|
|
|
@Slf4j
|
|
|
|
|
public class ProduceLocalFileTask {
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 默认的生成文件前缀
|
|
|
|
|
*/
|
|
|
|
|
private static final String PRE_FILE = "out_task_";
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 字段分隔符 (目前用于随机文件名中,...) [后续使用范围进行自行补充]
|
|
|
|
|
*/
|
|
|
|
|
private static final String SEPARATOR = "_";
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* AB任务标识
|
|
|
|
|
*/
|
|
|
|
|
private static final String TAG_AB = "AB";
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* FORM任务标识
|
|
|
|
|
*/
|
|
|
|
|
private static final String TAG_FORM = "FORM";
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 随机字符串的长度 (目前用于随机文件名中,...) [后续使用范围进行自行补充]
|
|
|
|
|
*/
|
|
|
|
|
private static final int RANDOM_STRING_LENGTH = 6;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 分页查询起始页 - 默认从0开始计算
|
|
|
|
|
*/
|
|
|
|
|
private static final int START_PAGE = 0;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 分页查询 - 单次2000条
|
|
|
|
|
*/
|
|
|
|
|
private static final int SIZE_PAGE = 2000;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Windows环境下的路径信息
|
|
|
|
|
*/
|
|
|
|
|
@Value(value = "file.windows.path")
|
|
|
|
|
private String windowsFilePath;
|
|
|
|
|
/**
|
|
|
|
|
* Windows环境下的路径信息
|
|
|
|
|
*/
|
|
|
|
|
@Value(value = "file.linux.path")
|
|
|
|
|
private String linuxFilePath;
|
|
|
|
|
/**
|
|
|
|
|
* Windows环境下的路径信息
|
|
|
|
|
*/
|
|
|
|
|
@Value(value = "file.mac.path")
|
|
|
|
|
private String macFilePath;
|
|
|
|
|
|
|
|
|
|
@Autowired
|
|
|
|
|
private TaskRecordService taskRecordService;
|
|
|
|
|
@Autowired
|
|
|
|
|
private AbMessageService abMessageService;
|
|
|
|
|
@Autowired
|
|
|
|
|
private FormMessageService formMessageService;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 任务处理入口,主要用于时间记录
|
|
|
|
|
*
|
|
|
|
|
@ -24,15 +106,174 @@ public class ProduceLocalFileTask {
|
|
|
|
|
* @param task
|
|
|
|
|
*/
|
|
|
|
|
@Async(value = "ProduceLocalFileTaskExecutor")
|
|
|
|
|
public void doRunTask(TaskRecord task){
|
|
|
|
|
public void doRunTask(TaskRecord task, String tag){
|
|
|
|
|
Long satrtMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
|
|
|
|
|
log.info("====== [ task start running, task name is {} ] ======", "ProduceLocalFileTask");
|
|
|
|
|
runTask(task);
|
|
|
|
|
runTask(task, tag);
|
|
|
|
|
Long endMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
|
|
|
|
|
log.info("====== [ task start end, task name is {},cost milliSecond is {} ] ======", "ProduceLocalFileTask", (endMilliSecond - satrtMilliSecond));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void runTask(TaskRecord task) {
|
|
|
|
|
private void runTask(TaskRecord task, String tag) {
|
|
|
|
|
// 转换需要的查询参数
|
|
|
|
|
String params = task.getParams();
|
|
|
|
|
TaskRecord taskRecord = new TaskRecord();
|
|
|
|
|
// 组装分页查询参数
|
|
|
|
|
Pageable pageable = PageRequest.of(START_PAGE, SIZE_PAGE);
|
|
|
|
|
// 设置不同的查询条件,去查AB表和Form表的数据
|
|
|
|
|
// 构建写入文件的路径
|
|
|
|
|
buildFileWritePath(taskRecord);
|
|
|
|
|
if (StringUtils.equalsIgnoreCase(TAG_AB, tag)){
|
|
|
|
|
// Json参数解析
|
|
|
|
|
AbMessage abMessage = new AbMessage();
|
|
|
|
|
if (StringUtils.isNotBlank(params)){
|
|
|
|
|
abMessage = JSON.parseObject(params, AbMessage.class);
|
|
|
|
|
}
|
|
|
|
|
// 构建分页查询,并写入到文件中
|
|
|
|
|
buildABQueryToFile(taskRecord, pageable, abMessage);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (StringUtils.equalsIgnoreCase(TAG_FORM, tag)){
|
|
|
|
|
// Json参数解析
|
|
|
|
|
FormMessage formMessage = new FormMessage();
|
|
|
|
|
if (StringUtils.isNotBlank(params)){
|
|
|
|
|
formMessage = JSON.parseObject(params, FormMessage.class);
|
|
|
|
|
}
|
|
|
|
|
// 构建分页查询,并写入到文件中
|
|
|
|
|
buildFromQueryToFile(taskRecord, pageable, formMessage);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void buildFromQueryToFile(TaskRecord taskRecord, Pageable pageable, FormMessage formMessage) {
|
|
|
|
|
if (formMessage == null){
|
|
|
|
|
log.error("============[Parse formMessage fail, please check formMessage info ]============");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
Example<FormMessage> example = Example.of(formMessage);
|
|
|
|
|
while(true){
|
|
|
|
|
// AB表查询
|
|
|
|
|
Slice<FormMessage> slice = formMessageService.queryAll(example, pageable);
|
|
|
|
|
if (slice == null){
|
|
|
|
|
log.error("============[Query FormMessage fail, please check example info : {} ]============", example.getProbe().toString());
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
List<FormMessage> abMessageList = slice.getContent();
|
|
|
|
|
if (!CollectionUtils.isEmpty(abMessageList)){
|
|
|
|
|
// 只获取需要的字段
|
|
|
|
|
List<String> collect = abMessageList.stream()
|
|
|
|
|
.map(FormMessage::getUid)
|
|
|
|
|
.collect(Collectors.toList());
|
|
|
|
|
// 设置总数
|
|
|
|
|
taskRecord.setTotal((long) slice.getSize());
|
|
|
|
|
// 解析并写入文件
|
|
|
|
|
writeToFile(taskRecord, collect);
|
|
|
|
|
}
|
|
|
|
|
if (!slice.hasNext()){
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
pageable = slice.nextPageable();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void buildABQueryToFile(TaskRecord taskRecord, Pageable pageable, AbMessage abMessage) {
|
|
|
|
|
if (abMessage == null){
|
|
|
|
|
log.error("============[Parse abMessage fail, please check abMessage info ]============");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
Example<AbMessage> example = Example.of(abMessage);
|
|
|
|
|
while(true){
|
|
|
|
|
// AB表查询
|
|
|
|
|
Slice<AbMessage> slice = abMessageService.queryAll(example, pageable);
|
|
|
|
|
if (slice == null){
|
|
|
|
|
log.error("============[Query abMessage fail, please check example info : {} ]============", example.getProbe().toString());
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
List<AbMessage> abMessageList = slice.getContent();
|
|
|
|
|
if (!CollectionUtils.isEmpty(abMessageList)){
|
|
|
|
|
// 只获取需要的字段
|
|
|
|
|
List<String> collect = abMessageList.stream()
|
|
|
|
|
.map(AbMessage::getUid)
|
|
|
|
|
.collect(Collectors.toList());
|
|
|
|
|
// 设置总数
|
|
|
|
|
taskRecord.setTotal((long) slice.getSize());
|
|
|
|
|
// 解析并写入文件
|
|
|
|
|
writeToFile(taskRecord, collect);
|
|
|
|
|
}
|
|
|
|
|
if (!slice.hasNext()){
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
pageable = slice.nextPageable();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 解析文件然后入库
|
|
|
|
|
*
|
|
|
|
|
* @param taskRecord 需要构建的任务包体
|
|
|
|
|
*/
|
|
|
|
|
private void buildFileWritePath(TaskRecord taskRecord){
|
|
|
|
|
// 获取环境配置信息
|
|
|
|
|
OsInfo osInfo = SystemUtil.getOsInfo();
|
|
|
|
|
String fullPath;
|
|
|
|
|
if (osInfo.isWindows()){
|
|
|
|
|
// 构建存储文件
|
|
|
|
|
fullPath = windowsFilePath + buildOnlyFileNameRule();
|
|
|
|
|
}else if (osInfo.isLinux()){
|
|
|
|
|
// 构建存储文件
|
|
|
|
|
fullPath = linuxFilePath + buildOnlyFileNameRule();
|
|
|
|
|
}else if (osInfo.isMac()){
|
|
|
|
|
// 构建存储文件
|
|
|
|
|
// 构建存储文件
|
|
|
|
|
fullPath = macFilePath + buildOnlyFileNameRule();
|
|
|
|
|
}else {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
taskRecord.setLocalFilePath(fullPath);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 待发送记录写入文件中,并记录课包信息
|
|
|
|
|
*
|
|
|
|
|
* @param taskRecord 课包记录信息
|
|
|
|
|
* @param collect 一个分页下的数据
|
|
|
|
|
*/
|
|
|
|
|
private void writeToFile(TaskRecord taskRecord, List<String> collect){
|
|
|
|
|
String fullPath = taskRecord.getLocalFilePath();
|
|
|
|
|
if (StringUtils.isBlank(fullPath)){
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
// 构建存储文件
|
|
|
|
|
try {
|
|
|
|
|
if (!FileUtil.exist(fullPath)){
|
|
|
|
|
Files.write(Paths.get(fullPath), collect, StandardOpenOption.CREATE_NEW);
|
|
|
|
|
TaskRecord update = taskRecordService.update(taskRecord);
|
|
|
|
|
if (update == null){
|
|
|
|
|
log.error("============[Write taskRecord fail, please check taskRecord info : {} ]============", taskRecord.toString());
|
|
|
|
|
}
|
|
|
|
|
}else{
|
|
|
|
|
Files.write(Paths.get(fullPath), collect, StandardOpenOption.APPEND);
|
|
|
|
|
TaskRecord update = taskRecordService.update(taskRecord);
|
|
|
|
|
if (update == null){
|
|
|
|
|
log.error("============[Write taskRecord fail, please check taskRecord info : {} ]============", taskRecord.toString());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
log.error("write prepare send file fail, please check param, fullPath is {}, {}", fullPath, e );
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 构建随机文件名规则
|
|
|
|
|
*
|
|
|
|
|
* @return 构建好的随机文件名
|
|
|
|
|
*/
|
|
|
|
|
private String buildOnlyFileNameRule(){
|
|
|
|
|
String fileName = PRE_FILE +
|
|
|
|
|
RandomUtil.randomString(RANDOM_STRING_LENGTH) +
|
|
|
|
|
SEPARATOR +
|
|
|
|
|
Instant.now().getEpochSecond();
|
|
|
|
|
return fileName;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
