From 54a4f1b6a90a1fb5176f3761341f1375a44e838e Mon Sep 17 00:00:00 2001 From: bynt Date: Mon, 3 Jul 2023 18:36:08 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=80=BB=E8=BE=91=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../callback/MobileMonitorCallbackNewAPI.java | 6 +++ .../adcallback/service/BdBackDateService.java | 17 +------- .../adcallback/service/DslBaseService.java | 41 +++++++++++++++++++ .../service/JlV1BackDataService.java | 18 +------- .../service/JlV2BackDataService.java | 17 +------- .../adcallback/service/KsBackDateService.java | 18 +------- .../adcallback/service/UcBackDateService.java | 18 +------- .../service/impl/BdBackDataServiceImpl.java | 11 +++++ .../service/impl/JlV1BackDataServiceImpl.java | 23 +++++++++++ .../service/impl/JlV2BackDataServiceImpl.java | 11 +++++ .../service/impl/KsBackDataServiceImpl.java | 11 +++++ .../service/impl/UcBackDataServiceImpl.java | 11 +++++ .../task/DeleteDeliveryDataTask.java | 34 +++++++++++++++ .../baiyee/adcallback/task/TfSourceTask.java | 2 +- 14 files changed, 156 insertions(+), 82 deletions(-) create mode 100644 src/main/java/com/baiyee/adcallback/service/DslBaseService.java diff --git a/src/main/java/com/baiyee/adcallback/api/callback/MobileMonitorCallbackNewAPI.java b/src/main/java/com/baiyee/adcallback/api/callback/MobileMonitorCallbackNewAPI.java index 8d855f1..99204cf 100644 --- a/src/main/java/com/baiyee/adcallback/api/callback/MobileMonitorCallbackNewAPI.java +++ b/src/main/java/com/baiyee/adcallback/api/callback/MobileMonitorCallbackNewAPI.java @@ -89,6 +89,9 @@ public class MobileMonitorCallbackNewAPI { vo.setVpoint(vpoint); } + if (vo.getOs().equalsIgnoreCase("OS")){ + log.error("======================== the ta error as {} ========================", vo.getTag()); + } // 重点参数校验规则 - 通过则入监控回流Source库 - 较完整字段 if (CheckPlatPointParamsUtil.checkJLPlatPointV2Param(vo.getOs(), vo.getImei(), vo.getIdfa(), vo.getIdfaMD5(), vo.getMac())) { originalityManageService.handleNewJLV2MonitorUrlCallbackDataThenSave(vo); @@ -148,6 +151,9 @@ public class MobileMonitorCallbackNewAPI { vo.setVpoint(vpoint); } + if (vo.getOs().equalsIgnoreCase("OS")){ + log.error("======================== the ta error as {} ========================", vo.getTag()); + } // 重点参数校验规则 - 通过则入监控回流Source库 - 较完整字段 if (CheckPlatPointParamsUtil.checkJLPlatPointV2Param(vo.getOs(), vo.getImei(), vo.getIdfa(), vo.getIdfaMD5(), vo.getMac())) { originalityManageService.handleNewJLV1MonitorUrlCallbackDataThenSave(vo); diff --git a/src/main/java/com/baiyee/adcallback/service/BdBackDateService.java b/src/main/java/com/baiyee/adcallback/service/BdBackDateService.java index 945bc12..2e33cf1 100644 --- a/src/main/java/com/baiyee/adcallback/service/BdBackDateService.java +++ b/src/main/java/com/baiyee/adcallback/service/BdBackDateService.java @@ -1,24 +1,9 @@ package com.baiyee.adcallback.service; -import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; - -import java.util.Date; -import java.util.List; - /** * @author Enzo * @date : 2023/6/21 */ -public interface BdBackDateService { +public interface BdBackDateService extends DslBaseService { - /** - * 条件查询 - * - * @param startTime begin time - * @param endTime end time - * @param tagList tag - * @param flag num or link - * @return - */ - List queryByTagAndTimeAndFlag(Date startTime, Date endTime, List tagList, Integer flag); } diff --git a/src/main/java/com/baiyee/adcallback/service/DslBaseService.java b/src/main/java/com/baiyee/adcallback/service/DslBaseService.java new file mode 100644 index 0000000..14b8c64 --- /dev/null +++ b/src/main/java/com/baiyee/adcallback/service/DslBaseService.java @@ -0,0 +1,41 @@ +package com.baiyee.adcallback.service; + +import cn.hutool.core.date.DateTime; +import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; + +import java.util.Date; +import java.util.List; + +/** + * @author Enzo + * @date : 2023/7/3 + */ +public interface DslBaseService { + + + /** + * 条件查询 + * + * @param startTime begin time + * @param endTime end time + * @param tagList tag + * @param flag num or link + * @return + */ + List queryByTagAndTimeAndFlag(Date startTime, Date endTime, List tagList, Integer flag); + + /** + * 删除数据 + * @param date + * @param num + * @return + */ + Integer deleteDateByDateAndLimitNum(DateTime date, Integer num); + + /** + * 查询数量 + * @param date + * @return + */ + Long countByDate(DateTime date); +} diff --git a/src/main/java/com/baiyee/adcallback/service/JlV1BackDataService.java b/src/main/java/com/baiyee/adcallback/service/JlV1BackDataService.java index b04ddc0..3969a31 100644 --- a/src/main/java/com/baiyee/adcallback/service/JlV1BackDataService.java +++ b/src/main/java/com/baiyee/adcallback/service/JlV1BackDataService.java @@ -1,24 +1,10 @@ package com.baiyee.adcallback.service; -import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; - -import java.util.Date; -import java.util.List; - /** * @author Enzo * @date : 2023/6/21 */ -public interface JlV1BackDataService { - /** - * 条件查询 - * - * @param startTime begin time - * @param endTime end time - * @param tagList tag - * @param flag num or link - * @return - */ - List queryByTagAndTimeAndFlag(Date startTime, Date endTime, List tagList, Integer flag); +public interface JlV1BackDataService extends DslBaseService { + } diff --git a/src/main/java/com/baiyee/adcallback/service/JlV2BackDataService.java b/src/main/java/com/baiyee/adcallback/service/JlV2BackDataService.java index 8f748d3..f627383 100644 --- a/src/main/java/com/baiyee/adcallback/service/JlV2BackDataService.java +++ b/src/main/java/com/baiyee/adcallback/service/JlV2BackDataService.java @@ -1,24 +1,9 @@ package com.baiyee.adcallback.service; -import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; - -import java.util.Date; -import java.util.List; - /** * @author Enzo * @date : 2023/6/21 */ -public interface JlV2BackDataService { +public interface JlV2BackDataService extends DslBaseService { - /** - * 条件查询 - * - * @param startTime begin time - * @param endTime end time - * @param tagList tag - * @param flag num or link - * @return - */ - List queryByTagAndTimeAndFlag(Date startTime, Date endTime, List tagList, Integer flag); } diff --git a/src/main/java/com/baiyee/adcallback/service/KsBackDateService.java b/src/main/java/com/baiyee/adcallback/service/KsBackDateService.java index e2997c7..f651dbb 100644 --- a/src/main/java/com/baiyee/adcallback/service/KsBackDateService.java +++ b/src/main/java/com/baiyee/adcallback/service/KsBackDateService.java @@ -1,24 +1,10 @@ package com.baiyee.adcallback.service; -import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; - -import java.util.Date; -import java.util.List; - /** * @author Enzo * @date : 2023/6/21 */ -public interface KsBackDateService { +public interface KsBackDateService extends DslBaseService { + - /** - * 条件查询 - * - * @param startTime begin time - * @param endTime end time - * @param tagList tag - * @param flag num or link - * @return - */ - List queryByTagAndTimeAndFlag(Date startTime, Date endTime, List tagList, Integer flag); } diff --git a/src/main/java/com/baiyee/adcallback/service/UcBackDateService.java b/src/main/java/com/baiyee/adcallback/service/UcBackDateService.java index 4fa81d0..662956f 100644 --- a/src/main/java/com/baiyee/adcallback/service/UcBackDateService.java +++ b/src/main/java/com/baiyee/adcallback/service/UcBackDateService.java @@ -1,24 +1,8 @@ package com.baiyee.adcallback.service; -import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; - -import java.util.Date; -import java.util.List; - /** * @author Enzo * @date : 2023/6/21 */ -public interface UcBackDateService { - - /** - * 条件查询 - * - * @param startTime begin time - * @param endTime end time - * @param tagList tag - * @param flag num or link - * @return - */ - List queryByTagAndTimeAndFlag(Date startTime, Date endTime, List tagList, Integer flag); +public interface UcBackDateService extends DslBaseService { } diff --git a/src/main/java/com/baiyee/adcallback/service/impl/BdBackDataServiceImpl.java b/src/main/java/com/baiyee/adcallback/service/impl/BdBackDataServiceImpl.java index bbc34f2..26fc995 100644 --- a/src/main/java/com/baiyee/adcallback/service/impl/BdBackDataServiceImpl.java +++ b/src/main/java/com/baiyee/adcallback/service/impl/BdBackDataServiceImpl.java @@ -1,6 +1,7 @@ package com.baiyee.adcallback.service.impl; import cn.hutool.core.convert.Convert; +import cn.hutool.core.date.DateTime; import cn.hutool.core.text.CharSequenceUtil; import com.baiyee.adcallback.api.constants.SqlConstant; import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; @@ -69,4 +70,14 @@ public class BdBackDataServiceImpl extends AbstractService dslBaseServices; + + /** * 凌晨五点 查询定时任务 删除记录 */ @Scheduled(cron = "0 0 5 * * ?") @Transactional(rollbackFor = Exception.class) + public void deleteDataTask() { for (TableEnum value : TableEnum.values()) { + //数据中需要删除的数据量 long expiredCount; // 已经删除数据量 @@ -70,4 +80,28 @@ public class DeleteDeliveryDataTask { BigInteger bigInteger = (BigInteger) selectQueryCount.getSingleResult(); return bigInteger.longValue(); } + + + /** + * 凌晨五点 查询定时任务 删除记录 + */ + // @Scheduled(cron = "0 0 4 * * ?") + @Scheduled(cron = "0 0/1 * * * ? ") + @Transactional(rollbackFor = Exception.class) + public void deleteNewDataTask() { + for (DslBaseService dslService : dslBaseServices) { + // 数据中需要删除的数据量 + long expiredCount; + // 已经删除数据量 + long totalDeleted = 0L; + // 时间偏移2个月 + DateTime date = DateUtil.offsetMonth(DateUtil.date(), -2); + do { + int executeUpdate = dslService.deleteDateByDateAndLimitNum(date, 50000); + totalDeleted += executeUpdate; + expiredCount = dslService.countByDate(date); + } while (expiredCount > 0); + log.info("======== delete tag{} table size as [{}] ========", dslService, totalDeleted); + } + } } diff --git a/src/main/java/com/baiyee/adcallback/task/TfSourceTask.java b/src/main/java/com/baiyee/adcallback/task/TfSourceTask.java index c9bc7c7..b59cb60 100644 --- a/src/main/java/com/baiyee/adcallback/task/TfSourceTask.java +++ b/src/main/java/com/baiyee/adcallback/task/TfSourceTask.java @@ -53,7 +53,7 @@ public class TfSourceTask { *

* 所有今天的数据压缩成投放包 */ - @Scheduled(cron = "0 0 1 * * ? ") + // @Scheduled(cron = "0 0 1 * * ? ") @Transactional(rollbackFor = Exception.class) public void tfSource() { log.info("++++++++++++++++++++++TfSource Sync start time {} ++++++++++++++++", DateUtil.now());