添加去重操作

pom.xml

@@ -67,6 +67,11 @@
           <version>${mysql-connector.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>22.0</version>
+        </dependency>
 
     </dependencies>
 

src/main/java/com/baiyee/adcallback/AdCallbackApplication.java

@@ -2,17 +2,16 @@ package com.baiyee.adcallback;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
 /**
  * 广告回调程序
- *
+ * 开启定时任务功能
+ * @EnableScheduling
  * @author q
  * @date 2022/01/05
  */
 @SpringBootApplication
-// 开启定时任务功能
 @EnableScheduling
 public class AdCallbackApplication {
 

src/main/java/com/baiyee/adcallback/api/controller/DmpMonitorNumController.java

@@ -3,6 +3,7 @@ package com.baiyee.adcallback.api.controller;
 import com.baiyee.adcallback.api.dto.DmpMonitorNumDto;
 import com.baiyee.adcallback.service.DmpMonitorNumService;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -14,6 +15,7 @@ import org.springframework.web.bind.annotation.RestController;
  * @author wujingtao
  * @date 2022/03/01
  */
+@Slf4j
 @RestController
 @RequiredArgsConstructor
 @RequestMapping("/api/dmp")

src/main/java/com/baiyee/adcallback/config/pojo/DeliveryStatisticDTO.java

@@ -0,0 +1,36 @@
+package com.baiyee.adcallback.config.pojo;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * @author Enzo
+ * @date : 2022/4/6
+ */
+@Data
+public class DeliveryStatisticDTO implements Serializable {
+
+    private static final long serialVersionUID = 4421574173345246118L;
+
+    private String imei;
+
+    private String tag;
+
+    private String gmtCreate;
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DeliveryStatisticDTO that = (DeliveryStatisticDTO) o;
+        return Objects.equals(imei, that.imei) && Objects.equals(tag, that.tag);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(imei, tag);
+    }
+
+}

src/main/java/com/baiyee/adcallback/repository/TbDeliveryStatisticEntityRepository.java

@@ -6,6 +6,8 @@ import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
 
+import java.util.List;
+
 /**
  * @author wujingtao
  * @date 2022/04/06
@@ -27,4 +29,13 @@ public interface TbDeliveryStatisticEntityRepository extends JpaRepository<TbDel
     @Modifying
     @Query(value = "update TbDeliveryStatisticEntity set dayNum = dayNum -1")
     void updateAllByDayNum();
+
+    /**
+     * tag查询数据
+     * @param tagStr
+     * @param i
+     * @return
+     */
+    @Query(value = "from TbDeliveryStatisticEntity where tag = ?1 and dayNum > ?2")
+    List<TbDeliveryStatisticEntity> findByTag(String tagStr, int i);
 }

src/main/java/com/baiyee/adcallback/repository/entity/TbGdtBackdataEntity.java

@@ -113,7 +113,7 @@ public class TbGdtBackdataEntity {
     @Column(name = "ip", nullable = false, length = 255)
     private String ip;
     @Basic
-    @Column(name = "user_agent", nullable = false, length = 255)
+    @Column(name = "user_agent", nullable = false, length = 512)
     private String userAgent;
     @Basic
     @Column(name = "callback", nullable = false, length = 255)

src/main/java/com/baiyee/adcallback/service/impl/DmpMonitorNumServiceImpl.java

@@ -1,6 +1,7 @@
 package com.baiyee.adcallback.service.impl;
 
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.convert.Convert;
 import cn.hutool.core.date.DateUnit;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.text.StrPool;
@@ -9,8 +10,16 @@ import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto;
 import com.baiyee.adcallback.api.dto.DmpMonitorNumDto;
 import com.baiyee.adcallback.common.enums.DeliveryPlatformEnum;
 import com.baiyee.adcallback.common.enums.TableEnum;
+import com.baiyee.adcallback.config.pojo.DeliveryStatisticDTO;
+import com.baiyee.adcallback.repository.TbDeliveryStatisticEntityRepository;
+import com.baiyee.adcallback.repository.entity.TbDeliveryStatisticEntity;
 import com.baiyee.adcallback.service.DmpMonitorNumService;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.hibernate.query.internal.NativeQueryImpl;
 import org.hibernate.transform.Transformers;
 import org.springframework.stereotype.Service;
@@ -21,12 +30,14 @@ import javax.persistence.PersistenceContext;
 import javax.persistence.Query;
 import java.math.BigInteger;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 
 /**
  * @author wujingtao
  * @date 2022/03/01
  */
+@Slf4j
 @Service
 @RequiredArgsConstructor
 public class DmpMonitorNumServiceImpl implements DmpMonitorNumService {
@@ -34,6 +45,9 @@ public class DmpMonitorNumServiceImpl implements DmpMonitorNumService {
     @PersistenceContext
     EntityManager entityManager;
 
+    private final TbDeliveryStatisticEntityRepository tbDeliveryStatisticEntityRepository;
+
+
     @Override
     public Map<String, Object> queryNumByTags(DmpMonitorNumDto dmpMonitorNumDto) {
         HashMap<String, Object> map = new HashMap<>();
@@ -177,6 +191,7 @@ public class DmpMonitorNumServiceImpl implements DmpMonitorNumService {
         query.setParameter("tag", tags);
         query.unwrap(NativeQueryImpl.class).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP);
         List list = query.getResultList();
+        log.error("the result list ");
         return getDmpMonitorJpaDto(list, flag);
     }
 
@@ -262,19 +277,63 @@ public class DmpMonitorNumServiceImpl implements DmpMonitorNumService {
 
     private List<DmpMonitorJpaDto> getDmpMonitorJpaDto(List list, Integer flag) {
         List<DmpMonitorJpaDto> tradeList = new ArrayList<>();
+        Multimap<String, DmpMonitorJpaDto> multimap = ArrayListMultimap.create();
         for (Object obj : list) {
             Map row = (Map) obj;
             DmpMonitorJpaDto trade = new DmpMonitorJpaDto();
+            trade.setGmtCreate(row.get("date").toString());
+            trade.setTag(row.get("tag").toString());
             if (1 == flag) {
                 BigInteger num = (BigInteger) row.get("num");
                 trade.setNum(num.longValue());
             } else {
                 trade.setImei(row.get("imei").toString());
+                multimap.put(row.get("tag").toString(), trade);
             }
-            trade.setGmtCreate(row.get("date").toString());
-            trade.setTag(row.get("tag").toString());
             tradeList.add(trade);
         }
+        if (flag == 2 && !multimap.isEmpty()) {
+            List<DmpMonitorJpaDto> newDataList = Lists.newArrayList();
+            List<DeliveryStatisticDTO> saveDataList = Lists.newArrayList();
+            // 转换map
+            Map<String, Collection<DmpMonitorJpaDto>> stringCollectionMap = multimap.asMap();
+            for (Map.Entry<String, Collection<DmpMonitorJpaDto>> stringCollectionEntry : stringCollectionMap.entrySet()) {
+                String tagStr = stringCollectionEntry.getKey();
+                // 去除重复数据
+                Set<DmpMonitorJpaDto> dmpMonitorJpaDtoSet =
+                        Sets.newHashSet(stringCollectionEntry.getValue());
+                // 转换set
+                Set<DeliveryStatisticDTO> rawDataSet =
+                        Convert.toSet(DeliveryStatisticDTO.class, dmpMonitorJpaDtoSet);
+                // 查询数据
+                List<DeliveryStatisticDTO> tagLists =
+                        Convert.toList(DeliveryStatisticDTO.class,
+                                tbDeliveryStatisticEntityRepository.findByTag(tagStr, 2));
+
+                Set<DeliveryStatisticDTO> newDataSet = Sets.newHashSet(tagLists);
+                // 差集去除数据
+                List<DeliveryStatisticDTO> differentData
+                        = Lists.newArrayList(Sets.difference(rawDataSet, newDataSet));
+                // 保存将要插入数据
+                saveDataList.addAll(differentData);
+                List<DmpMonitorJpaDto> monitorJpaDtoList =
+                        Convert.toList(DmpMonitorJpaDto.class, differentData);
+                // 返回数据
+                newDataList.addAll(monitorJpaDtoList);
+            }
+            log.error("================ the save datalist size as {} ================",saveDataList.size());
+            if (CollUtil.isNotEmpty(saveDataList)) {
+                // 异步插入数据
+                CompletableFuture.runAsync(() -> {
+                    List<TbDeliveryStatisticEntity> deliveryStatisticEntityList =
+                            Convert.toList(TbDeliveryStatisticEntity.class, saveDataList);
+                    // 批量保存
+                    tbDeliveryStatisticEntityRepository.saveAll(deliveryStatisticEntityList);
+                });
+            }
+            log.error("================ the new datalist size as {} ================",newDataList.size());
+            return newDataList;
+        }
         return tradeList;
     }
 }

src/main/java/com/baiyee/adcallback/task/DeliveryStatisticTask.java

@@ -25,7 +25,7 @@ public class DeliveryStatisticTask {
      * 一，删除剩余天数为0的数据
      * 二，所有的天数减1
      */
-//    @Scheduled(cron = "0 0 23 * * ? ")
+    @Scheduled(cron = "0 0 23 * * ? ")
     @Transactional(rollbackFor = Exception.class)
     public void autoDeal() {
         log.info("++++++++++++++++++++++DeliveryStatisticTask Sync start time {} ++++++++++++++++", DateUtil.now());

src/main/resources/application-dev.yml

@@ -6,7 +6,7 @@ spring:
     druid:
       db-type: com.alibaba.druid.pool.DruidDataSource
       driverClassName: com.mysql.cj.jdbc.Driver
-      url: jdbc:mysql://localhost/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true
+      url: jdbc:mysql://118.178.137.129/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true
       username: root
       password: root
       # 初始连接数

src/main/resources/application.yml

@@ -4,7 +4,7 @@ spring:
   application:
     name: ad-callback
   profiles:
-    active: dev
+    active: prod
   jackson:
     time-zone: GMT+8
     date-format: yyyy-MM-dd HH:mm:ss

src/test/java/com/baiyee/adcallback/AdCallbackApplicationTests.java

@@ -1,15 +1,23 @@
 package com.baiyee.adcallback;
 
-import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONUtil;
-import org.junit.jupiter.api.Test;
+import com.baiyee.adcallback.api.constants.SqlConstant;
-
+import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto;
-import java.util.ArrayList;
+import com.baiyee.adcallback.common.enums.TableEnum;
-import java.util.HashMap;
+import com.baiyee.adcallback.config.pojo.DeliveryStatisticDTO;
-import java.util.List;
+import com.google.common.collect.ArrayListMultimap;
-import java.util.Map;
+import com.google.common.collect.Lists;
-
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
 public class AdCallbackApplicationTests {
     static class User{
         private String name;
@@ -34,12 +42,11 @@ public class AdCallbackApplicationTests {
 
     public static void main(String[] args) {
 
-        User user = new User();
+        String sqlString = SqlConstant.PARSE_DATA_SQL.replaceAll(SqlConstant.TABLE_STR, TableEnum.GD.getTableName());
-        user.setName("123");
+        sqlString = new StringBuffer(sqlString.replaceAll
-        user.setAge(0);
+                (SqlConstant.IMEI_STR, SqlConstant.MUID_STR)).insert
-
+                (sqlString.indexOf("GROUP BY"), SqlConstant.PHONE_STR).toString();
-
+        System.out.println(sqlString);
-        System.out.println(JSONUtil.toJsonStr(user));
     }
 
 
@@ -75,4 +82,46 @@ public class AdCallbackApplicationTests {
     }
 
 
+    @Test
+    public void map() {
+        Multimap<String,String> multimap= ArrayListMultimap.create();
+        multimap.put("Fruits","Banana");
+        multimap.put("Fruits","Apple");
+        multimap.put("Fruits","Apple");
+
+        Map<String, Collection<String>> stringCollectionMap = multimap.asMap();
+        log.debug("");
+
+    }
+
+    @Test
+    public void intersection() {
+        DeliveryStatisticDTO dmpMonitorJpaDto1 = new DeliveryStatisticDTO();
+        DeliveryStatisticDTO dmpMonitorJpaDto2 = new DeliveryStatisticDTO();
+        dmpMonitorJpaDto1.setImei("12345");
+
+        dmpMonitorJpaDto1.setTag("678910");
+        dmpMonitorJpaDto1.setGmtCreate("2022-4-6 17:58:52");
+
+        dmpMonitorJpaDto2.setImei("12345");
+
+
+        dmpMonitorJpaDto2.setTag("9101234");
+        List<DeliveryStatisticDTO> dmpMonitorJpaDtos1 = Lists.newArrayList(dmpMonitorJpaDto1);
+        dmpMonitorJpaDtos1.add(dmpMonitorJpaDto2);
+        Set<DeliveryStatisticDTO> dmpMonitorJpaDtos = Sets.newHashSet(dmpMonitorJpaDtos1);
+        DeliveryStatisticDTO dmpMonitorJpaDto3 = new DeliveryStatisticDTO();
+        dmpMonitorJpaDto3.setImei("12345");
+        dmpMonitorJpaDto3.setTag("678910");
+        List<DeliveryStatisticDTO> dmpMonitorJpaDtos2 = Lists.newArrayList(dmpMonitorJpaDto3);
+        Set<DeliveryStatisticDTO> dmpMonitorJpaDtos3 = Sets.newHashSet(dmpMonitorJpaDtos2);
+        Sets.SetView<DeliveryStatisticDTO> difference = Sets.difference(dmpMonitorJpaDtos, dmpMonitorJpaDtos3);
+        Lists.newArrayList(difference);
+
+        List<DeliveryStatisticDTO> monitorJpaDtoList = CollUtil.subtractToList(dmpMonitorJpaDtos, dmpMonitorJpaDtos3);
+
+
+        log.debug("");
+
+    }
 }