Merge remote-tracking branch 'origin/master'

src/main/java/com/baiyee/adcallback/service/DslBaseService.java

@@ -30,7 +30,7 @@ public interface DslBaseService {
      * @param num
      * @return
      */
-    Integer deleteDateByDateAndLimitNum(DateTime date, Integer num);
+    Long deleteDateByDateAndLimitNum(DateTime date, Integer num);
 
     /**
      * 查询数量

src/main/java/com/baiyee/adcallback/service/impl/BdBackDataServiceImpl.java

@@ -23,6 +23,7 @@ import com.querydsl.sql.SQLExpressions;
 import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
 
+import java.sql.Timestamp;
 import java.util.Date;
 import java.util.List;
 
@@ -34,6 +35,7 @@ import java.util.List;
 @Service
 @RequiredArgsConstructor
 public class BdBackDataServiceImpl extends AbstractService<TbBdNewBackdataEntity, Long> implements BdBackDateService {
+    QTbBdNewBackdataEntity bdNewBackdata = QTbBdNewBackdataEntity.tbBdNewBackdataEntity;
     @Override
     public List<DmpMonitorJpaDto> queryByTagAndTimeAndFlag
             (Date startTime, Date endTime, List<String> tagList, Integer flag) {
@@ -41,7 +43,7 @@ public class BdBackDataServiceImpl extends AbstractService<TbBdNewBackdataEntity
         return this.applyJPA(query -> {
             StringPath t3 = Expressions.stringPath("t3");
             StringTemplate dateExpr;
-            QTbBdNewBackdataEntity bdNewBackdata = QTbBdNewBackdataEntity.tbBdNewBackdataEntity;
+
             if (flag == 1) {
                 dateExpr = Expressions.stringTemplate(SqlConstant.DATE_EXPR, bdNewBackdata.gmtCreate);
                 SimpleTemplate<String> t3Tag = Expressions.template(String.class, "t3.tag");
@@ -72,12 +74,13 @@ public class BdBackDataServiceImpl extends AbstractService<TbBdNewBackdataEntity
     }
 
     @Override
-    public Integer deleteDateByDateAndLimitNum(DateTime date, Integer num) {
-        return null;
+    public Long deleteDateByDateAndLimitNum(DateTime date, Integer num) {
+        Timestamp timestamp = new Timestamp(date.getTime());
+        return this.newQuery().delete(bdNewBackdata).where(bdNewBackdata.gmtCreate.gt(timestamp)).execute();
     }
 
     @Override
     public Long countByDate(DateTime date) {
-        return null;
+        return 0L;
     }
 }

src/main/java/com/baiyee/adcallback/service/impl/JlV1BackDataServiceImpl.java

@@ -21,8 +21,13 @@ import com.querydsl.core.types.dsl.StringPath;
 import com.querydsl.core.types.dsl.StringTemplate;
 import com.querydsl.sql.SQLExpressions;
 import lombok.RequiredArgsConstructor;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import javax.persistence.Query;
 import java.util.Date;
 import java.util.List;
 
@@ -34,13 +39,18 @@ import java.util.List;
 @RequiredArgsConstructor
 public class JlV1BackDataServiceImpl extends AbstractService<TbJLBackDataV1Entity, Long> implements JlV1BackDataService {
 
+
+    @PersistenceContext
+    protected EntityManager entityManager;
+    QTbJLBackDataV1Entity v1Entity = QTbJLBackDataV1Entity.tbJLBackDataV1Entity;
+
     @Override
     public List<DmpMonitorJpaDto> queryByTagAndTimeAndFlag
             (Date startTime, Date endTime, List<String> tagList, Integer flag) {
         return this.applyJPA(query -> {
             StringPath t3 = Expressions.stringPath("t3");
             StringTemplate dateExpr;
-            QTbJLBackDataV1Entity v1Entity = QTbJLBackDataV1Entity.tbJLBackDataV1Entity;
+
             if (flag == 1) {
                 dateExpr = Expressions.stringTemplate(SqlConstant.DATE_EXPR, v1Entity.gmtCreate);
                 SimpleTemplate<String> t3Tag = Expressions.template(String.class, "t3.tag");
@@ -48,9 +58,9 @@ public class JlV1BackDataServiceImpl extends AbstractService<TbJLBackDataV1Entit
                 SimpleTemplate<Date> t3Date = Expressions.template(Date.class, "t3.gmtCreate");
                 // 封装子查询
                 SubQueryExpression<Tuple> gmtCreate = SQLExpressions.select
-                        (v1Entity.imei, v1Entity.tag,v1Entity.gmtCreate.max().as(SqlConstant.CREATE_TIME)).from
+                        (v1Entity.imei, v1Entity.tag, v1Entity.gmtCreate.max().as(SqlConstant.CREATE_TIME)).from
                                 (v1Entity).where(v1Entity.tag.in(tagList).and(v1Entity.imei.ne(CharSequenceUtil.EMPTY).and
-                                        (v1Entity.imei.ne(DataConstant.IMEI_NULL_TAG)).and(v1Entity.imei.ne(DataConstant.IMEI_TAG)))).groupBy(v1Entity.imei,v1Entity.tag);
+                                        (v1Entity.imei.ne(DataConstant.IMEI_NULL_TAG)).and(v1Entity.imei.ne(DataConstant.IMEI_TAG)))).groupBy(v1Entity.imei, v1Entity.tag);
 
                 // 结果返回
                 List<QueryQuantityVO> voList = query.select(Projections.bean
@@ -64,31 +74,19 @@ public class JlV1BackDataServiceImpl extends AbstractService<TbJLBackDataV1Entit
 
             List<QueryImeiVO> imeiVOList = query.select(Projections.bean
                     (QueryImeiVO.class, v1Entity.imei, v1Entity.tag, dateExpr.as(SqlConstant.CREATE_TIME))).distinct().from(v1Entity).where
-                            (v1Entity.tag.in(tagList).and(v1Entity.imei.ne(CharSequenceUtil.EMPTY).and(v1Entity.imei.ne(DataConstant.IMEI_NULL_TAG)).and(v1Entity.imei.ne
-                                    (DataConstant.IMEI_TAG)))).groupBy(v1Entity.imei,v1Entity.tag).fetch();
+                    (v1Entity.tag.in(tagList).and(v1Entity.imei.ne(CharSequenceUtil.EMPTY).and(v1Entity.imei.ne(DataConstant.IMEI_NULL_TAG)).and(v1Entity.imei.ne
+                            (DataConstant.IMEI_TAG)))).groupBy(v1Entity.imei, v1Entity.tag).fetch();
             return Convert.toList(DmpMonitorJpaDto.class, imeiVOList);
         });
     }
 
     @Override
-    public Integer deleteDateByDateAndLimitNum(DateTime date, Integer num) {
-
-
-        //querydsl查询实体
-        QTbJLBackDataV1Entity v1Entity = QTbJLBackDataV1Entity.tbJLBackDataV1Entity;
-
-        this.newQuery()
-                //删除对象
-                .delete(v1Entity)
-                //删除条件
-                .where(v1Entity.gmtCreate.gt(date))
-                //执行删除
-                .execute();
-        return null;
+    public Long deleteDateByDateAndLimitNum(DateTime date, Integer num) {
+        return this.newQuery().delete(v1Entity).where(v1Entity.gmtCreate.gt(date)).execute();
     }
 
     @Override
     public Long countByDate(DateTime date) {
-        return null;
+        return 0L;
     }
 }

src/main/java/com/baiyee/adcallback/service/impl/JlV2BackDataServiceImpl.java

@@ -33,7 +33,7 @@ import java.util.List;
 @Service
 @RequiredArgsConstructor
 public class JlV2BackDataServiceImpl extends AbstractService<TbJLBackDataV2Entity,Long> implements JlV2BackDataService {
-
+    QTbJLBackDataV2Entity v2Entity = QTbJLBackDataV2Entity.tbJLBackDataV2Entity;
     @Override
     public List<DmpMonitorJpaDto> queryByTagAndTimeAndFlag
             (Date startTime, Date endTime, List<String> tagList, Integer flag) {
@@ -41,7 +41,7 @@ public class JlV2BackDataServiceImpl extends AbstractService<TbJLBackDataV2Entit
         return this.applyJPA(query -> {
             StringPath t3 = Expressions.stringPath("t3");
             StringTemplate dateExpr;
-            QTbJLBackDataV2Entity v2Entity = QTbJLBackDataV2Entity.tbJLBackDataV2Entity;
+
             if (flag == 1) {
                 dateExpr = Expressions.stringTemplate(SqlConstant.DATE_EXPR, v2Entity.gmtCreate);
                 SimpleTemplate<String> t3Tag = Expressions.template(String.class, "t3.tag");
@@ -72,12 +72,12 @@ public class JlV2BackDataServiceImpl extends AbstractService<TbJLBackDataV2Entit
     }
 
     @Override
-    public Integer deleteDateByDateAndLimitNum(DateTime date, Integer num) {
-        return null;
+    public Long deleteDateByDateAndLimitNum(DateTime date, Integer num) {
+        return this.newQuery().delete(v2Entity).where(v2Entity.gmtCreate.gt(date)).execute();
     }
 
     @Override
     public Long countByDate(DateTime date) {
-        return null;
+        return 0L;
     }
 }

src/main/java/com/baiyee/adcallback/service/impl/KsBackDataServiceImpl.java

@@ -23,6 +23,7 @@ import com.querydsl.sql.SQLExpressions;
 import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
 
+import java.sql.Timestamp;
 import java.util.Date;
 import java.util.List;
 
@@ -34,7 +35,7 @@ import java.util.List;
 @Service
 @RequiredArgsConstructor
 public class KsBackDataServiceImpl extends AbstractService<TbKSNewBackdataEntity,Long> implements KsBackDateService {
-
+    QTbKSNewBackdataEntity ksNewBackdata = QTbKSNewBackdataEntity.tbKSNewBackdataEntity;
     @Override
     public List<DmpMonitorJpaDto> queryByTagAndTimeAndFlag
             (Date startTime, Date endTime, List<String> tagList, Integer flag) {
@@ -42,7 +43,7 @@ public class KsBackDataServiceImpl extends AbstractService<TbKSNewBackdataEntity
         return this.applyJPA(query -> {
             StringPath t3 = Expressions.stringPath("t3");
             StringTemplate dateExpr;
-            QTbKSNewBackdataEntity ksNewBackdata = QTbKSNewBackdataEntity.tbKSNewBackdataEntity;
+
             if (flag == 1) {
                 dateExpr = Expressions.stringTemplate(SqlConstant.DATE_EXPR, ksNewBackdata.gmtCreate);
                 SimpleTemplate<String> t3Tag = Expressions.template(String.class, "t3.tag");
@@ -73,12 +74,13 @@ public class KsBackDataServiceImpl extends AbstractService<TbKSNewBackdataEntity
     }
 
     @Override
-    public Integer deleteDateByDateAndLimitNum(DateTime date, Integer num) {
-        return null;
+    public Long deleteDateByDateAndLimitNum(DateTime date, Integer num) {
+        Timestamp timestamp = new Timestamp(date.getTime());
+        return this.newQuery().delete(ksNewBackdata).where(ksNewBackdata.gmtCreate.gt(timestamp)).execute();
     }
 
     @Override
     public Long countByDate(DateTime date) {
-        return null;
+        return 0L;
     }
 }

src/main/java/com/baiyee/adcallback/service/impl/UcBackDataServiceImpl.java

@@ -23,6 +23,7 @@ import com.querydsl.sql.SQLExpressions;
 import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
 
+import java.sql.Timestamp;
 import java.util.Date;
 import java.util.List;
 
@@ -34,7 +35,7 @@ import java.util.List;
 @Service
 @RequiredArgsConstructor
 public class UcBackDataServiceImpl extends AbstractService<TbUcBackdataEntity,Long> implements UcBackDateService {
-
+    QTbUcBackdataEntity ucBackdata = QTbUcBackdataEntity.tbUcBackdataEntity;
     @Override
     public List<DmpMonitorJpaDto> queryByTagAndTimeAndFlag
             (Date startTime, Date endTime, List<String> tagList, Integer flag) {
@@ -42,7 +43,7 @@ public class UcBackDataServiceImpl extends AbstractService<TbUcBackdataEntity,Lo
         return this.applyJPA(query -> {
             StringPath t3 = Expressions.stringPath("t3");
             StringTemplate dateExpr;
-            QTbUcBackdataEntity ucBackdata = QTbUcBackdataEntity.tbUcBackdataEntity;
+
             if (flag == 1) {
                 dateExpr = Expressions.stringTemplate(SqlConstant.DATE_EXPR, ucBackdata.gmtCreate);
                 SimpleTemplate<String> t3Tag = Expressions.template(String.class, "t3.tag");
@@ -73,12 +74,13 @@ public class UcBackDataServiceImpl extends AbstractService<TbUcBackdataEntity,Lo
     }
 
     @Override
-    public Integer deleteDateByDateAndLimitNum(DateTime date, Integer num) {
-        return null;
+    public Long deleteDateByDateAndLimitNum(DateTime date, Integer num) {
+        Timestamp timestamp = new Timestamp(date.getTime());
+        return this.newQuery().delete(ucBackdata).where(ucBackdata.gmtCreate.gt(timestamp)).execute();
     }
 
     @Override
     public Long countByDate(DateTime date) {
-        return null;
+        return 0L;
     }
 }

src/main/java/com/baiyee/adcallback/task/DeleteDeliveryDataTask.java

@@ -55,7 +55,7 @@ public class DeleteDeliveryDataTask {
                 Query deleteTagByQuery = entityManager.createNativeQuery
                         (SqlConstant.DELETE_STR
                                 .concat(tableName)
-                                .concat(SqlConstant.WHERE_STR)
+                                        .concat(SqlConstant.WHERE_STR)
                                 .concat(" limit 50000"));
                 deleteTagByQuery.setParameter("DATE", date.toDateStr());
                 int executeUpdate = deleteTagByQuery.executeUpdate();
@@ -83,25 +83,17 @@ public class DeleteDeliveryDataTask {
 
 
     /**
-     * 凌晨五点 查询定时任务 删除记录
+     * 每六个小时删除一批数据
+     *  XXX 大数据应该修改代码逻辑
      */
-    // @Scheduled(cron = "0 0 4 * * ?")
-    @Scheduled(cron = "0 0/1 * * * ? ")
+    @Scheduled(cron = "0 0 0/6 * * ?")
     @Transactional(rollbackFor = Exception.class)
     public void deleteNewDataTask() {
         for (DslBaseService dslService : dslBaseServices) {
-            // 数据中需要删除的数据量
-            long expiredCount;
-            // 已经删除数据量
-            long totalDeleted = 0L;
             // 时间偏移2个月
             DateTime date = DateUtil.offsetMonth(DateUtil.date(), -2);
-            do {
-                int executeUpdate = dslService.deleteDateByDateAndLimitNum(date, 50000);
-                totalDeleted += executeUpdate;
-                expiredCount = dslService.countByDate(date);
-            } while (expiredCount > 0);
-            log.info("========  delete tag{} table size as [{}]   ========", dslService, totalDeleted);
+            Long executeUpdate = dslService.deleteDateByDateAndLimitNum(date, 50000);
+            log.info("========  delete service {} table size as [{}]   ========", dslService, executeUpdate);
         }
     }
 }

src/main/resources/application-dev.yml

@@ -74,36 +74,42 @@ spring:
             props:
               # 分片算法的行表达式（算法自行定义，此处为方便演示效果）
               algorithm-expression: db$->{id % 2}
+              allow-range-query-with-inline-sharding: true
           jl-v1-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_jl_v1_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           jl-v2-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_jl_v2_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           bd-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_bd_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           gdt-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_gdt_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           ks-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_ks_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
         # 分布式序列算法配置
         key-generators:
           snowflake:

src/main/resources/application-prod.yml

@@ -72,36 +72,42 @@ spring:
             props:
               # 分片算法的行表达式（算法自行定义，此处为方便演示效果）
               algorithm-expression: db$->{id % 2}
+              allow-range-query-with-inline-sharding: true
           jl-v1-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_jl_v1_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           jl-v2-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_jl_v2_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           bd-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_bd_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           gdt-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_gdt_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
           ks-table-inline:
             # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/
             type: INLINE
             props:
               # 分片算法的行表达式
               algorithm-expression: tb_ks_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16}
+              allow-range-query-with-inline-sharding: true
         # 分布式序列算法配置
         key-generators:
           snowflake:

src/test/java/com/baiyee/adcallback/DeleteTest.java

@@ -0,0 +1,46 @@
+package com.baiyee.adcallback;
+
+import cn.hutool.core.date.DateTime;
+import cn.hutool.core.date.DateUtil;
+import com.baiyee.adcallback.service.JlV1BackDataService;
+import com.baiyee.adcallback.service.UcBackDateService;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.annotation.Resource;
+
+/**
+ * @author Enzo
+ * @date : 2023/7/4
+ */
+
+@Slf4j
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = AdCallbackApplication.class,
+        webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+public class DeleteTest {
+
+
+    @Resource
+    private JlV1BackDataService jlV1BackDataService;
+
+
+    @Resource
+    private UcBackDateService ucBackDateService;
+
+    @Test
+    @Transactional(rollbackFor = Exception.class)
+    public void test1() {
+
+        DateTime dateTime = DateUtil.offsetMonth(DateUtil.date(), -1);
+        ucBackDateService.deleteDateByDateAndLimitNum(dateTime, 5000);
+
+    }
+
+
+
+}

src/test/java/com/baiyee/adcallback/TagTest.java

@@ -9,6 +9,7 @@ import com.baiyee.adcallback.api.vo.QueryQuantityVO;
 import com.baiyee.adcallback.repository.TbJLV1BackdataEntityRepository;
 import com.baiyee.adcallback.repository.entity.QTbJLBackDataV1Entity;
 import com.baiyee.adcallback.repository.entity.TbJLBackDataV1Entity;
+import com.baiyee.adcallback.service.JlV1BackDataService;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.querydsl.core.Tuple;
@@ -53,6 +54,10 @@ public class TagTest {
     private EntityManager entityManager;
 
 
+    @Resource
+    private  JlV1BackDataService jlV1BackDataService;
+
+
     @Resource
     private TbJLV1BackdataEntityRepository jlv1BackdataEntityRepository;
 
@@ -90,7 +95,7 @@ public class TagTest {
 
         List<QueryQuantityVO> num = jpasqlQuery.select(Projections.bean(QueryQuantityVO.class, tbJLBackDataV1Entity.imei.count().as("num"),
                 tbJLBackDataV1Entity.tag, dateExpr.as("gmtCreate"))).from(tbJLBackDataV1Entity).innerJoin(gmtCreate, t3).on(tbJLBackDataV1Entity.imei.eq
-                (t3Imei).and(tbJLBackDataV1Entity.gmtCreate.eq(t3Date))).groupBy(dateExpr ,tbJLBackDataV1Entity.tag).fetch();
+                (t3Imei).and(tbJLBackDataV1Entity.gmtCreate.eq(t3Date))).groupBy(dateExpr, tbJLBackDataV1Entity.tag).fetch();
 
         System.out.println(num);