From 7680c9a2be27e77266b7053e8a55f71024910cdc Mon Sep 17 00:00:00 2001 From: bynt Date: Wed, 6 Apr 2022 19:32:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=8E=BB=E9=87=8D=E6=93=8D?= =?UTF-8?q?=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 5 ++ .../adcallback/AdCallbackApplication.java | 5 +- .../controller/DmpMonitorNumController.java | 2 + .../config/pojo/DeliveryStatisticDTO.java | 36 +++++++++ .../TbDeliveryStatisticEntityRepository.java | 11 +++ .../entity/TbGdtBackdataEntity.java | 2 +- .../impl/DmpMonitorNumServiceImpl.java | 63 ++++++++++++++- .../task/DeliveryStatisticTask.java | 2 +- src/main/resources/application-dev.yml | 2 +- src/main/resources/application.yml | 2 +- .../AdCallbackApplicationTests.java | 79 +++++++++++++++---- 11 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 src/main/java/com/baiyee/adcallback/config/pojo/DeliveryStatisticDTO.java diff --git a/pom.xml b/pom.xml index a0668ae..4fe05c8 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,11 @@ ${mysql-connector.version} + + com.google.guava + guava + 22.0 + diff --git a/src/main/java/com/baiyee/adcallback/AdCallbackApplication.java b/src/main/java/com/baiyee/adcallback/AdCallbackApplication.java index 1a869d0..8a55ab8 100644 --- a/src/main/java/com/baiyee/adcallback/AdCallbackApplication.java +++ b/src/main/java/com/baiyee/adcallback/AdCallbackApplication.java @@ -2,17 +2,16 @@ package com.baiyee.adcallback; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cache.annotation.EnableCaching; import org.springframework.scheduling.annotation.EnableScheduling; /** * 广告回调程序 - * + * 开启定时任务功能 + * @EnableScheduling * @author q * @date 2022/01/05 */ @SpringBootApplication -// 开启定时任务功能 @EnableScheduling public class AdCallbackApplication { diff --git a/src/main/java/com/baiyee/adcallback/api/controller/DmpMonitorNumController.java b/src/main/java/com/baiyee/adcallback/api/controller/DmpMonitorNumController.java index a4afa2b..5bd5a24 100644 --- a/src/main/java/com/baiyee/adcallback/api/controller/DmpMonitorNumController.java +++ b/src/main/java/com/baiyee/adcallback/api/controller/DmpMonitorNumController.java @@ -3,6 +3,7 @@ package com.baiyee.adcallback.api.controller; import com.baiyee.adcallback.api.dto.DmpMonitorNumDto; import com.baiyee.adcallback.service.DmpMonitorNumService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; @@ -14,6 +15,7 @@ import org.springframework.web.bind.annotation.RestController; * @author wujingtao * @date 2022/03/01 */ +@Slf4j @RestController @RequiredArgsConstructor @RequestMapping("/api/dmp") diff --git a/src/main/java/com/baiyee/adcallback/config/pojo/DeliveryStatisticDTO.java b/src/main/java/com/baiyee/adcallback/config/pojo/DeliveryStatisticDTO.java new file mode 100644 index 0000000..3a35273 --- /dev/null +++ b/src/main/java/com/baiyee/adcallback/config/pojo/DeliveryStatisticDTO.java @@ -0,0 +1,36 @@ +package com.baiyee.adcallback.config.pojo; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Objects; + +/** + * @author Enzo + * @date : 2022/4/6 + */ +@Data +public class DeliveryStatisticDTO implements Serializable { + + private static final long serialVersionUID = 4421574173345246118L; + + private String imei; + + private String tag; + + private String gmtCreate; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeliveryStatisticDTO that = (DeliveryStatisticDTO) o; + return Objects.equals(imei, that.imei) && Objects.equals(tag, that.tag); + } + + @Override + public int hashCode() { + return Objects.hash(imei, tag); + } + +} diff --git a/src/main/java/com/baiyee/adcallback/repository/TbDeliveryStatisticEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbDeliveryStatisticEntityRepository.java index e16b542..8b456ee 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbDeliveryStatisticEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbDeliveryStatisticEntityRepository.java @@ -6,6 +6,8 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import java.util.List; + /** * @author wujingtao * @date 2022/04/06 @@ -27,4 +29,13 @@ public interface TbDeliveryStatisticEntityRepository extends JpaRepository findByTag(String tagStr, int i); } diff --git a/src/main/java/com/baiyee/adcallback/repository/entity/TbGdtBackdataEntity.java b/src/main/java/com/baiyee/adcallback/repository/entity/TbGdtBackdataEntity.java index 7ad32e8..972fb3a 100644 --- a/src/main/java/com/baiyee/adcallback/repository/entity/TbGdtBackdataEntity.java +++ b/src/main/java/com/baiyee/adcallback/repository/entity/TbGdtBackdataEntity.java @@ -113,7 +113,7 @@ public class TbGdtBackdataEntity { @Column(name = "ip", nullable = false, length = 255) private String ip; @Basic - @Column(name = "user_agent", nullable = false, length = 255) + @Column(name = "user_agent", nullable = false, length = 512) private String userAgent; @Basic @Column(name = "callback", nullable = false, length = 255) diff --git a/src/main/java/com/baiyee/adcallback/service/impl/DmpMonitorNumServiceImpl.java b/src/main/java/com/baiyee/adcallback/service/impl/DmpMonitorNumServiceImpl.java index a72b65f..d6b02ef 100644 --- a/src/main/java/com/baiyee/adcallback/service/impl/DmpMonitorNumServiceImpl.java +++ b/src/main/java/com/baiyee/adcallback/service/impl/DmpMonitorNumServiceImpl.java @@ -1,6 +1,7 @@ package com.baiyee.adcallback.service.impl; import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.convert.Convert; import cn.hutool.core.date.DateUnit; import cn.hutool.core.date.DateUtil; import cn.hutool.core.text.StrPool; @@ -9,8 +10,16 @@ import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; import com.baiyee.adcallback.api.dto.DmpMonitorNumDto; import com.baiyee.adcallback.common.enums.DeliveryPlatformEnum; import com.baiyee.adcallback.common.enums.TableEnum; +import com.baiyee.adcallback.config.pojo.DeliveryStatisticDTO; +import com.baiyee.adcallback.repository.TbDeliveryStatisticEntityRepository; +import com.baiyee.adcallback.repository.entity.TbDeliveryStatisticEntity; import com.baiyee.adcallback.service.DmpMonitorNumService; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.hibernate.query.internal.NativeQueryImpl; import org.hibernate.transform.Transformers; import org.springframework.stereotype.Service; @@ -21,12 +30,14 @@ import javax.persistence.PersistenceContext; import javax.persistence.Query; import java.math.BigInteger; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; /** * @author wujingtao * @date 2022/03/01 */ +@Slf4j @Service @RequiredArgsConstructor public class DmpMonitorNumServiceImpl implements DmpMonitorNumService { @@ -34,6 +45,9 @@ public class DmpMonitorNumServiceImpl implements DmpMonitorNumService { @PersistenceContext EntityManager entityManager; + private final TbDeliveryStatisticEntityRepository tbDeliveryStatisticEntityRepository; + + @Override public Map queryNumByTags(DmpMonitorNumDto dmpMonitorNumDto) { HashMap map = new HashMap<>(); @@ -177,6 +191,7 @@ public class DmpMonitorNumServiceImpl implements DmpMonitorNumService { query.setParameter("tag", tags); query.unwrap(NativeQueryImpl.class).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP); List list = query.getResultList(); + log.error("the result list "); return getDmpMonitorJpaDto(list, flag); } @@ -262,19 +277,63 @@ public class DmpMonitorNumServiceImpl implements DmpMonitorNumService { private List getDmpMonitorJpaDto(List list, Integer flag) { List tradeList = new ArrayList<>(); + Multimap multimap = ArrayListMultimap.create(); for (Object obj : list) { Map row = (Map) obj; DmpMonitorJpaDto trade = new DmpMonitorJpaDto(); + trade.setGmtCreate(row.get("date").toString()); + trade.setTag(row.get("tag").toString()); if (1 == flag) { BigInteger num = (BigInteger) row.get("num"); trade.setNum(num.longValue()); } else { trade.setImei(row.get("imei").toString()); + multimap.put(row.get("tag").toString(), trade); } - trade.setGmtCreate(row.get("date").toString()); - trade.setTag(row.get("tag").toString()); tradeList.add(trade); } + if (flag == 2 && !multimap.isEmpty()) { + List newDataList = Lists.newArrayList(); + List saveDataList = Lists.newArrayList(); + // 转换map + Map> stringCollectionMap = multimap.asMap(); + for (Map.Entry> stringCollectionEntry : stringCollectionMap.entrySet()) { + String tagStr = stringCollectionEntry.getKey(); + // 去除重复数据 + Set dmpMonitorJpaDtoSet = + Sets.newHashSet(stringCollectionEntry.getValue()); + // 转换set + Set rawDataSet = + Convert.toSet(DeliveryStatisticDTO.class, dmpMonitorJpaDtoSet); + // 查询数据 + List tagLists = + Convert.toList(DeliveryStatisticDTO.class, + tbDeliveryStatisticEntityRepository.findByTag(tagStr, 2)); + + Set newDataSet = Sets.newHashSet(tagLists); + // 差集去除数据 + List differentData + = Lists.newArrayList(Sets.difference(rawDataSet, newDataSet)); + // 保存将要插入数据 + saveDataList.addAll(differentData); + List monitorJpaDtoList = + Convert.toList(DmpMonitorJpaDto.class, differentData); + // 返回数据 + newDataList.addAll(monitorJpaDtoList); + } + log.error("================ the save datalist size as {} ================",saveDataList.size()); + if (CollUtil.isNotEmpty(saveDataList)) { + // 异步插入数据 + CompletableFuture.runAsync(() -> { + List deliveryStatisticEntityList = + Convert.toList(TbDeliveryStatisticEntity.class, saveDataList); + // 批量保存 + tbDeliveryStatisticEntityRepository.saveAll(deliveryStatisticEntityList); + }); + } + log.error("================ the new datalist size as {} ================",newDataList.size()); + return newDataList; + } return tradeList; } } diff --git a/src/main/java/com/baiyee/adcallback/task/DeliveryStatisticTask.java b/src/main/java/com/baiyee/adcallback/task/DeliveryStatisticTask.java index c8df2af..fc4caa0 100644 --- a/src/main/java/com/baiyee/adcallback/task/DeliveryStatisticTask.java +++ b/src/main/java/com/baiyee/adcallback/task/DeliveryStatisticTask.java @@ -25,7 +25,7 @@ public class DeliveryStatisticTask { * 一，删除剩余天数为0的数据 * 二，所有的天数减1 */ -// @Scheduled(cron = "0 0 23 * * ? ") + @Scheduled(cron = "0 0 23 * * ? ") @Transactional(rollbackFor = Exception.class) public void autoDeal() { log.info("++++++++++++++++++++++DeliveryStatisticTask Sync start time {} ++++++++++++++++", DateUtil.now()); diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 8170133..a006fea 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -6,7 +6,7 @@ spring: druid: db-type: com.alibaba.druid.pool.DruidDataSource driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://localhost/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true + url: jdbc:mysql://118.178.137.129/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true username: root password: root # 初始连接数 diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 15fe98a..d79242b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -4,7 +4,7 @@ spring: application: name: ad-callback profiles: - active: dev + active: prod jackson: time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss diff --git a/src/test/java/com/baiyee/adcallback/AdCallbackApplicationTests.java b/src/test/java/com/baiyee/adcallback/AdCallbackApplicationTests.java index 82e1602..cdc5640 100644 --- a/src/test/java/com/baiyee/adcallback/AdCallbackApplicationTests.java +++ b/src/test/java/com/baiyee/adcallback/AdCallbackApplicationTests.java @@ -1,17 +1,25 @@ package com.baiyee.adcallback; -import cn.hutool.core.collection.ListUtil; +import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.baiyee.adcallback.api.constants.SqlConstant; +import com.baiyee.adcallback.api.dto.DmpMonitorJpaDto; +import com.baiyee.adcallback.common.enums.TableEnum; +import com.baiyee.adcallback.config.pojo.DeliveryStatisticDTO; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j public class AdCallbackApplicationTests { - static class User { + static class User{ private String name; private Integer age; @@ -34,12 +42,11 @@ public class AdCallbackApplicationTests { public static void main(String[] args) { - User user = new User(); - user.setName("123"); - user.setAge(0); - - - System.out.println(JSONUtil.toJsonStr(user)); + String sqlString = SqlConstant.PARSE_DATA_SQL.replaceAll(SqlConstant.TABLE_STR, TableEnum.GD.getTableName()); + sqlString = new StringBuffer(sqlString.replaceAll + (SqlConstant.IMEI_STR, SqlConstant.MUID_STR)).insert + (sqlString.indexOf("GROUP BY"), SqlConstant.PHONE_STR).toString(); + System.out.println(sqlString); } @@ -75,4 +82,46 @@ public class AdCallbackApplicationTests { } + @Test + public void map() { + Multimap multimap= ArrayListMultimap.create(); + multimap.put("Fruits","Banana"); + multimap.put("Fruits","Apple"); + multimap.put("Fruits","Apple"); + + Map> stringCollectionMap = multimap.asMap(); + log.debug(""); + + } + + @Test + public void intersection() { + DeliveryStatisticDTO dmpMonitorJpaDto1 = new DeliveryStatisticDTO(); + DeliveryStatisticDTO dmpMonitorJpaDto2 = new DeliveryStatisticDTO(); + dmpMonitorJpaDto1.setImei("12345"); + + dmpMonitorJpaDto1.setTag("678910"); + dmpMonitorJpaDto1.setGmtCreate("2022-4-6 17:58:52"); + + dmpMonitorJpaDto2.setImei("12345"); + + + dmpMonitorJpaDto2.setTag("9101234"); + List dmpMonitorJpaDtos1 = Lists.newArrayList(dmpMonitorJpaDto1); + dmpMonitorJpaDtos1.add(dmpMonitorJpaDto2); + Set dmpMonitorJpaDtos = Sets.newHashSet(dmpMonitorJpaDtos1); + DeliveryStatisticDTO dmpMonitorJpaDto3 = new DeliveryStatisticDTO(); + dmpMonitorJpaDto3.setImei("12345"); + dmpMonitorJpaDto3.setTag("678910"); + List dmpMonitorJpaDtos2 = Lists.newArrayList(dmpMonitorJpaDto3); + Set dmpMonitorJpaDtos3 = Sets.newHashSet(dmpMonitorJpaDtos2); + Sets.SetView difference = Sets.difference(dmpMonitorJpaDtos, dmpMonitorJpaDtos3); + Lists.newArrayList(difference); + + List monitorJpaDtoList = CollUtil.subtractToList(dmpMonitorJpaDtos, dmpMonitorJpaDtos3); + + + log.debug(""); + + } }