更新了自动检测删除的逻辑和优化了发送前去重的逻辑
parent
c377c2ca51
commit
491c9d4d76
@ -0,0 +1,122 @@
|
||||
package me.zhengjie.modules.common.handler;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import me.zhengjie.modules.tmpfilerecord.service.TempFileRecordService;
|
||||
import me.zhengjie.modules.tmpfilerecord.service.dto.TempFileRecordDto;
|
||||
import me.zhengjie.modules.tmpfilerecord.service.dto.TempFileRecordQueryCriteria;
|
||||
import me.zhengjie.utils.FileUtil;
|
||||
import me.zhengjie.utils.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 定时检查过期文件并进行删除 - 基类
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class DeleteFileScheduler {
|
||||
|
||||
/**
|
||||
* 文件执行操作成功标识
|
||||
*/
|
||||
private static final int FILE_SUCCESS_TAG = 1;
|
||||
|
||||
/**
|
||||
* 文件路径的分隔符
|
||||
*/
|
||||
private static final String PATH_SPLIT_TAG = ",";
|
||||
|
||||
|
||||
@Autowired
|
||||
private TempFileRecordService tempFileRecordService;
|
||||
|
||||
/**
|
||||
* 在每天11点进行任务删除
|
||||
*/
|
||||
// @Scheduled(cron = "0 0 23 ? * *")
|
||||
public void deleteOrUpdateFilePerDayAtElevenClock() {
|
||||
log.info("======== [start run delete/update file scheduler task.] ========");
|
||||
// 先进行分类
|
||||
Map<Integer, List<TempFileRecordDto>> resultMap = checkDayOfDBDay();
|
||||
// 然后进行修改或者删除处理
|
||||
handleOfDays(resultMap);
|
||||
log.info("======== [end run delete/update file scheduler task.] ========");
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void handleOfDays(Map<Integer, List<TempFileRecordDto>> resultMap) {
|
||||
|
||||
for (Map.Entry<Integer, List<TempFileRecordDto>> each : resultMap.entrySet()) {
|
||||
if (each.getKey() == 0) {
|
||||
List<TempFileRecordDto> value = each.getValue();
|
||||
// 获取所有的需要删除记录的路径
|
||||
List<String> pathCollect = value.stream()
|
||||
.map(TempFileRecordDto::getFilePaths)
|
||||
.collect(Collectors.toList());
|
||||
delExpireFiles(pathCollect);
|
||||
// 对存储天数为0的记录进行删除
|
||||
tempFileRecordService.deleteAll(
|
||||
value.stream()
|
||||
.map(TempFileRecordDto::getId)
|
||||
.toArray(Integer[]::new)
|
||||
);
|
||||
}
|
||||
if (each.getKey() == 1) {
|
||||
// 每天对存储天数进行减一
|
||||
List<TempFileRecordDto> value = each.getValue();
|
||||
value.forEach(
|
||||
one-> one.setDays(one.getDays() - 1)
|
||||
);
|
||||
tempFileRecordService.batchUpdate(value);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private void delExpireFiles(List<String> pathCollect) {
|
||||
pathCollect.forEach(
|
||||
// 直接把文件夹进行删除
|
||||
each->{
|
||||
if (each.contains(PATH_SPLIT_TAG)){
|
||||
// 一个目录下有多个文件,多个文件删除
|
||||
String[] split = each.split(PATH_SPLIT_TAG);
|
||||
if (split.length > 0){
|
||||
String dirPath = StringUtils.substringBeforeLast(split[0], File.separator);
|
||||
if (StringUtils.isNotBlank(dirPath)){
|
||||
FileUtil.del(dirPath);
|
||||
}
|
||||
}
|
||||
}else{
|
||||
String dirPath = StringUtils.substringBeforeLast(each, File.separator);
|
||||
FileUtil.del(dirPath);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
private Map<Integer, List<TempFileRecordDto>> checkDayOfDBDay() {
|
||||
// 获取所有的记录
|
||||
TempFileRecordQueryCriteria tempFileRecordQueryCriteria = new TempFileRecordQueryCriteria();
|
||||
tempFileRecordQueryCriteria.setFileStatus(FILE_SUCCESS_TAG);
|
||||
List<TempFileRecordDto> tempFileRecordDtos = tempFileRecordService.queryAll(tempFileRecordQueryCriteria);
|
||||
// 筛选出Days小于等于1 的 打标识为0 与 其他的打标记为1
|
||||
Map<Integer, List<TempFileRecordDto>> collect = tempFileRecordDtos.stream()
|
||||
.collect(Collectors.groupingBy(
|
||||
item -> {
|
||||
if (item.getDays() <= 1) {
|
||||
return 0;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
));
|
||||
return collect;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
package me.zhengjie;
|
||||
|
||||
import cn.hutool.core.util.RandomUtil;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class StreamTest {
|
||||
|
||||
private static Set<String> uids;
|
||||
|
||||
private static List<String> base;
|
||||
|
||||
static {
|
||||
uids = new HashSet<>(50_0000);
|
||||
uids.add("111111111111111111111111");
|
||||
for (int i = 0; i < 500000; i++) {
|
||||
uids.add(RandomUtil.randomString(24));
|
||||
}
|
||||
|
||||
base = new ArrayList<>(2000_0000);
|
||||
base.add("111111111111111111111111");
|
||||
for (int i = 0; i < 2000_0000; i++) {
|
||||
base.add(RandomUtil.randomString(24));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 测试去重写法
|
||||
* 测试结论:
|
||||
* 1. 一定要在大集合里面去重小集合
|
||||
* 2. 数据结构大集合采用 List 小集合采用 Set
|
||||
*/
|
||||
@Test
|
||||
public void testFilter(){
|
||||
Long start = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
|
||||
System.out.println(" start , " + start);
|
||||
// Set<String> collect = uids.stream().parallel().filter(base::contains).collect(Collectors.toSet());
|
||||
Set<String> collect = base.stream().parallel().filter(uids::contains).collect(Collectors.toSet()); // 2119 334
|
||||
|
||||
System.out.println("uids size :" + uids.size() + ", base size: " + base.size() + ", 重复"+ collect.size());
|
||||
Long end = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
|
||||
System.out.println(" end, "+ end +", cost , " + (end - start));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package me.zhengjie;
|
||||
|
||||
import me.zhengjie.modules.common.handler.DeleteFileScheduler;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
public class TestScheduler extends MailTest{
|
||||
|
||||
@Autowired
|
||||
private DeleteFileScheduler deleteFileScheduler;
|
||||
|
||||
|
||||
@Test
|
||||
public void testDel(){
|
||||
deleteFileScheduler.deleteOrUpdateFilePerDayAtElevenClock();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue