diff --git a/pom.xml b/pom.xml index 207d624..51c98a7 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ 1.1.22 5.7.16 true + 5.1.1 8.0.13 true @@ -64,7 +65,7 @@ mysql mysql-connector-java - ${mysql-connector.version} + ${mysql-connector.version} @@ -77,6 +78,20 @@ zip4j 1.3.2 + + + + + org.apache.shardingsphere + shardingsphere-jdbc-core-spring-boot-starter + ${shardingsphere.version} + + + + + org.codehaus.groovy + groovy + diff --git a/src/main/java/com/baiyee/adcallback/config/DruidConfig.java b/src/main/java/com/baiyee/adcallback/config/DruidConfig.java new file mode 100644 index 0000000..68c7cdd --- /dev/null +++ b/src/main/java/com/baiyee/adcallback/config/DruidConfig.java @@ -0,0 +1,65 @@ +package com.baiyee.adcallback.config; + +import com.alibaba.druid.spring.boot.autoconfigure.properties.DruidStatProperties; +import com.alibaba.druid.util.Utils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.servlet.*; +import java.io.IOException; + + +/** + * @author Enzo + * @date : 2023/6/7 + */ + +@Configuration +public class DruidConfig { + + + /** + * 去除监控页面底部的广告 + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + @Bean + @ConditionalOnProperty(name = "spring.datasource.druid.stat-view-servlet.enabled", havingValue = "true") + public FilterRegistrationBean removeDruidFilterRegistrationBean(DruidStatProperties properties) { + // 获取web监控页面的参数 + DruidStatProperties.StatViewServlet config = properties.getStatViewServlet(); + // 提取common.js的配置路径 + String pattern = config.getUrlPattern() != null ? config.getUrlPattern() : "/druid/*"; + String commonJsPattern = pattern.replaceAll("\\*", "js/common.js"); + final String filePath = "support/http/resources/js/common.js"; + // 创建filter进行过滤 + Filter filter = new Filter() { + @Override + public void init(javax.servlet.FilterConfig filterConfig) throws ServletException { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + chain.doFilter(request, response); + // 重置缓冲区，响应头不会被重置 + response.resetBuffer(); + // 获取common.js + String text = Utils.readFromResource(filePath); + // 正则替换banner, 除去底部的广告信息 + text = text.replaceAll("
", ""); + text = text.replaceAll("powered.*?shrek.wang", ""); + response.getWriter().write(text); + } + + @Override + public void destroy() { + } + }; + FilterRegistrationBean registrationBean = new FilterRegistrationBean(); + registrationBean.setFilter(filter); + registrationBean.addUrlPatterns(commonJsPattern); + return registrationBean; + } +} diff --git a/src/main/java/com/baiyee/adcallback/config/DruidShardingJdbcDataSourceConfiguration.java b/src/main/java/com/baiyee/adcallback/config/DruidShardingJdbcDataSourceConfiguration.java new file mode 100644 index 0000000..49be3a1 --- /dev/null +++ b/src/main/java/com/baiyee/adcallback/config/DruidShardingJdbcDataSourceConfiguration.java @@ -0,0 +1,27 @@ +package com.baiyee.adcallback.config; + +import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure; +import com.alibaba.druid.spring.boot.autoconfigure.properties.DruidStatProperties; +import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidFilterConfiguration; +import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidSpringAopConfiguration; +import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidStatViewServletConfiguration; +import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidWebStatFilterConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * @author Enzo + * @date : 2023/6/7 + */ +@Configuration +@ConditionalOnClass(DruidDataSourceAutoConfigure.class) +@EnableConfigurationProperties({DruidStatProperties.class}) +@Import({ + DruidSpringAopConfiguration.class, + DruidStatViewServletConfiguration.class, + DruidWebStatFilterConfiguration.class, + DruidFilterConfiguration.class}) +public class DruidShardingJdbcDataSourceConfiguration { +} diff --git a/src/main/java/com/baiyee/adcallback/repository/TbBDNewBackdataEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbBDNewBackdataEntityRepository.java index 81d727b..2b36c6f 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbBDNewBackdataEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbBDNewBackdataEntityRepository.java @@ -25,7 +25,7 @@ public interface TbBDNewBackdataEntityRepository extends JpaRepository= :startDate and gmtCreate < :endDate and tag in :tag group by imei,tag") List queryPlateFormDTO(@Param("imeiStrFirst") String imeiStrFirst, @Param("imeiSecond") String imeiSecond, @Param("startDate") DateTime startDate, @Param("endDate") DateTime endDate, @Param("tag") List tag); diff --git a/src/main/java/com/baiyee/adcallback/repository/TbGdtNewBackdataEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbGdtNewBackdataEntityRepository.java index 54f851d..5d6bfbf 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbGdtNewBackdataEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbGdtNewBackdataEntityRepository.java @@ -25,9 +25,9 @@ public interface TbGdtNewBackdataEntityRepository extends JpaRepository= :startDate and gmtCreate < :endDate and tag in :tag and deviceOsType = :models group by muid,tag") + @Query("select new com.baiyee.adcallback.api.dto.PlateFormDTO(imei as imei,tag as tag) from " + + " TbGdtNewBackdataEntity where imei != :imeiStrFirst and imei != :imeiSecond " + + "and gmtCreate >= :startDate and gmtCreate < :endDate and tag in :tag and deviceOsType = :models group by imei,tag") List queryPlateFormDTO(@Param("imeiStrFirst") String imeiStrFirst, @Param("imeiSecond") String imeiSecond, @Param("startDate") DateTime startDate, @Param("endDate") DateTime endDate, @Param("tag") List tag, @Param("models") String models); @Query(value = "SELECT * FROM tb_gdt_new_backdata WHERE imei != '' AND imei != '__imei__' AND gmt_create > ?1 AND gmt_create < ?2", nativeQuery = true) diff --git a/src/main/java/com/baiyee/adcallback/repository/TbJLV1BackdataEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbJLV1BackdataEntityRepository.java index 4ffb334..f08092b 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbJLV1BackdataEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbJLV1BackdataEntityRepository.java @@ -24,6 +24,7 @@ public interface TbJLV1BackdataEntityRepository extends JpaRepository= :startDate and gmtCreate < :endDate and tag in :tag group by imei,tag") List queryPlateFormDTO(@Param("imeiStrFirst") String imeiStrFirst, @Param("imeiSecond") String imeiSecond, @Param("startDate") DateTime startDate, @Param("endDate") DateTime endDate, @Param("tag") List tag); @Query(value = "SELECT * FROM tb_jl_v1_backdata WHERE imei != '' AND imei != '__IMEI__' AND gmt_create > ?1 AND gmt_create < ?2", nativeQuery = true) List queryByDate(String startDate, String endDate); + + List queryBytag(String s); } diff --git a/src/main/java/com/baiyee/adcallback/repository/TbJLV2BackdataEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbJLV2BackdataEntityRepository.java index 0cb8673..0ebb76c 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbJLV2BackdataEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbJLV2BackdataEntityRepository.java @@ -28,7 +28,7 @@ public interface TbJLV2BackdataEntityRepository extends JpaRepository= :startDate and gmtCreate < :endDate and tag in :tag group by imei,tag") List queryPlateFormDTO(@Param("imeiStrFirst") String imeiStrFirst, @Param("imeiSecond") String imeiSecond, @Param("startDate") DateTime startDate, @Param("endDate") DateTime endDate, @Param("tag") List tag); diff --git a/src/main/java/com/baiyee/adcallback/repository/TbKSNewBackdataEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbKSNewBackdataEntityRepository.java index c54010e..76a7834 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbKSNewBackdataEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbKSNewBackdataEntityRepository.java @@ -25,7 +25,7 @@ public interface TbKSNewBackdataEntityRepository extends JpaRepository= :startDate and gmtCreate < :endDate and tag in :tag group by imei,tag") List queryPlateFormDTO(@Param("imeiStrFirst") String imeiStrFirst, @Param("imeiSecond") String imeiSecond, @Param("startDate") DateTime startDate, @Param("endDate") DateTime endDate, @Param("tag") List tag); diff --git a/src/main/java/com/baiyee/adcallback/repository/TbPlatformDeliveryEntityRepository.java b/src/main/java/com/baiyee/adcallback/repository/TbPlatformDeliveryEntityRepository.java index 0bab603..0257349 100644 --- a/src/main/java/com/baiyee/adcallback/repository/TbPlatformDeliveryEntityRepository.java +++ b/src/main/java/com/baiyee/adcallback/repository/TbPlatformDeliveryEntityRepository.java @@ -28,7 +28,6 @@ public interface TbPlatformDeliveryEntityRepository extends JpaRepository findByTag(Long userId, String tagStr, int num); - /** * 删除剩余天数为0 的数据 * diff --git a/src/main/java/com/baiyee/adcallback/service/BackDataEntityService.java b/src/main/java/com/baiyee/adcallback/service/BackDataEntityService.java index e2b2229..47b242b 100644 --- a/src/main/java/com/baiyee/adcallback/service/BackDataEntityService.java +++ b/src/main/java/com/baiyee/adcallback/service/BackDataEntityService.java @@ -22,5 +22,19 @@ public interface BackDataEntityService { * @param value * @return */ - List queryPlateFormDTO(String imeiFirst, String imeiSecond, DateTime date, DateTime offsetMinute, List tagStr, Integer value); + @Deprecated + List queryPlateFormDTO(String imeiFirst, String imeiSecond, DateTime date, DateTime offsetMinute, List tagStr, Integer value); + + /** + * 新库查询指定时间内的imei号 + * + * @param imeiFirst + * @param imeiSecond + * @param date + * @param offsetMinute + * @param tagStr + * @param value + * @return + */ + List queryNewPlateFormDTO(String imeiFirst, String imeiSecond, DateTime offsetMinute, DateTime date, List tagStr, Integer value); } diff --git a/src/main/java/com/baiyee/adcallback/service/impl/BackDataEntityServiceImpl.java b/src/main/java/com/baiyee/adcallback/service/impl/BackDataEntityServiceImpl.java index 2a208bf..caafa47 100644 --- a/src/main/java/com/baiyee/adcallback/service/impl/BackDataEntityServiceImpl.java +++ b/src/main/java/com/baiyee/adcallback/service/impl/BackDataEntityServiceImpl.java @@ -8,6 +8,7 @@ import com.google.common.collect.Lists; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import java.util.List; @@ -33,9 +34,20 @@ public class BackDataEntityServiceImpl implements BackDataEntityService { private final TbVivoBackdataEntityRepository tbVivoBackdataEntityRepository; + private final TbJLV1BackdataEntityRepository jlv1BackdataEntityRepository; + + private final TbJLV2BackdataEntityRepository jlv2BackdataEntityRepository; + + private final TbGdtNewBackdataEntityRepository tbGdtNewBackdataEntityRepository; + + private final TbKSNewBackdataEntityRepository tbKSNewBackdataEntityRepository; + + private final TbBDNewBackdataEntityRepository tbBDNewBackdataEntityRepository; + + @Override - public List queryPlateFormDTO - (String imeiFirst, String imeiSecond, DateTime date, DateTime offsetMinute, List tagStr, Integer value) { + @Deprecated + public List queryPlateFormDTO(String imeiFirst, String imeiSecond, DateTime date, DateTime offsetMinute, List tagStr, Integer value) { List formDTOList = Lists.newArrayList(); switch (value) { @@ -61,11 +73,57 @@ public class BackDataEntityServiceImpl implements BackDataEntityService { break; case 6: // 广点通 - formDTOList = tbGdtBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr,"android"); + formDTOList = tbGdtBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr, "android"); break; default: } return formDTOList; } + + @Override + public List queryNewPlateFormDTO(String imeiFirst, String imeiSecond, DateTime offsetMinute, DateTime date, List tagStr, Integer value) { + + List newFormDTOList = Lists.newArrayList(); + switch (value) { + case 1: + List v1TagList = Lists.newArrayList(); + List v2TagList = Lists.newArrayList(); + for (String tag : tagStr) { + ((tag.lastIndexOf("v2") > 0) ? v1TagList : v2TagList).add(tag); + } + // 巨量 + if (!CollectionUtils.isEmpty(v1TagList)) { + newFormDTOList.addAll(jlv1BackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, v1TagList)); + } + if (!CollectionUtils.isEmpty(v2TagList)) { + newFormDTOList.addAll(jlv2BackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, v2TagList)); + } + break; + case 2: + // 快手 + newFormDTOList = tbKSNewBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr); + break; + case 3: + // UC + newFormDTOList = tbUcBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr); + break; + case 4: + // VIVO + newFormDTOList = tbVivoBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr); + break; + case 5: + // 百度 + newFormDTOList = tbBDNewBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr); + break; + case 6: + // 广点通 + newFormDTOList = tbGdtNewBackdataEntityRepository.queryPlateFormDTO(imeiFirst, imeiSecond, date, offsetMinute, tagStr, "android"); + break; + default: + } + return newFormDTOList; + } + + } diff --git a/src/main/java/com/baiyee/adcallback/task/NewPlatFormRecordTask.java b/src/main/java/com/baiyee/adcallback/task/NewPlatFormRecordTask.java new file mode 100644 index 0000000..2bbd768 --- /dev/null +++ b/src/main/java/com/baiyee/adcallback/task/NewPlatFormRecordTask.java @@ -0,0 +1,165 @@ +package com.baiyee.adcallback.task; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.convert.Convert; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.text.CharSequenceUtil; +import cn.hutool.core.text.StrPool; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import com.baiyee.adcallback.api.dto.PlateFormDTO; +import com.baiyee.adcallback.api.dto.RepeatFormDTO; +import com.baiyee.adcallback.common.enums.DeliveryPlatformEnum; +import com.baiyee.adcallback.config.pojo.PlatformTransmitDTO; +import com.baiyee.adcallback.repository.entity.TbPlatformCompanyEntity; +import com.baiyee.adcallback.repository.entity.TbPlatformDeliveryEntity; +import com.baiyee.adcallback.service.AdPlatformDeliveryService; +import com.baiyee.adcallback.service.AdPlatformService; +import com.baiyee.adcallback.service.BackDataEntityService; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +/** + * @author Enzo + * @date : 2022/10/13 + */ + +@Slf4j +@Component +@RequiredArgsConstructor +public class NewPlatFormRecordTask { + + @Value("${platform.authToken}") + private String authToken; + + @Value("${platform.url}") + private String url; + + private DateTime beginTime; + + private final AdPlatformService adPlatformService; + + private final BackDataEntityService backDataEntityService; + + private final AdPlatformDeliveryService adPlatformDeliveryService; + + /** + * 十分钟执行任务 + */ + @Scheduled(cron = "0 0/9 * * * ? ") + @Transactional(rollbackFor = Exception.class) + public void deliveryPlatform() { + List platforms + = adPlatformService.findByPlatTag(1); + if (CollUtil.isNotEmpty(platforms)) { + for (TbPlatformCompanyEntity platform : platforms) { + DateTime date = DateUtil.date(); + if (CollUtil.isNotEmpty(platform.getTagStr())) { + String tagStr = platform.getTagStr().get(0); + if (tagStr.contains(StrPool.DASHED)) { + Integer value = DeliveryPlatformEnum.getValue + (tagStr.substring(0, tagStr.indexOf(StrPool.DASHED))); + // 偏移十分钟 + DateTime offsetMinute = ObjectUtil.isNull(beginTime) ? + DateUtil.offsetMinute(date, -9) : + DateUtil.offsetMinute(beginTime, -9); + beginTime = offsetMinute; + log.info("================ the begin time as {} end time as {} ================", offsetMinute, date); + List dtoList = backDataEntityService.queryNewPlateFormDTO + (CharSequenceUtil.EMPTY, "__IMEI__", offsetMinute, date, platform.getTagStr(), value); + log.info("================ the dto list as {} ================", dtoList.size()); + if (CollUtil.isNotEmpty(dtoList)) { + // tag进行分组 + Map> listMap = dtoList.stream() + .collect(Collectors.groupingBy(PlateFormDTO::getTag)); + + List saveDataList = Lists.newArrayList(); + for (Map.Entry> stringListEntry : listMap.entrySet()) { + List list = listMap.get(stringListEntry.getKey()); + List repeatFormDTOList = Convert.toList(RepeatFormDTO.class, list); + repeatFormDTOList.forEach(dto -> dto.setUserId(platform.getUserId())); + // 去除重复数据 + Set originSet = + Sets.newHashSet(repeatFormDTOList); + // 查询数据 + List tagLists = + adPlatformDeliveryService.findByTag(platform.getUserId(), stringListEntry.getKey(), 0); + + Set querySet = Sets.newHashSet(tagLists); + // 差集去除数据 + List differentData + = Lists.newArrayList(Sets.difference(originSet, querySet)); + // 保存将要插入数据 + saveDataList.addAll(differentData); + } + // 分批发送 + List> partition = Lists.partition(saveDataList, 500); + int num = 0; + for (List formDTOList : partition) { + PlatformTransmitDTO dto = new PlatformTransmitDTO(); + dto.setList(formDTOList); + dto.setAuthToken(authToken); + dto.setUserId(platform.getUserId()); + dto.setTaskId(platform.getTaskId()); + String post = HttpUtil.post(url, JSONUtil.toJsonStr(dto)); + Object status = new JSONObject(post).get("status"); + if (ObjectUtil.isNotNull(status) && + Integer.parseInt(status.toString()) != 0) { + num++; + } + } + if (num == 0 && CollUtil.isNotEmpty(saveDataList)) { + // 异步插入数据 + CompletableFuture.runAsync(() -> { + List deliveryStatisticEntityList = + Convert.toList(TbPlatformDeliveryEntity.class, saveDataList); + // 批量保存 + adPlatformDeliveryService.saveAll(deliveryStatisticEntityList); + }); + beginTime = date; + } + log.error("================ the new datalist size as {} ================", saveDataList.size()); + } + if (CollUtil.isEmpty(dtoList)) { + beginTime = date; + } + } + } + } + } + } + + + /** + * 修改数据 + */ + @Scheduled(cron = "0 30 23 * * ? ") + @Transactional(rollbackFor = Exception.class) + public void autoDeal() { + log.info("++++++++++++++++++++++ change plate form sync start time {} ++++++++++++++++", DateUtil.now()); + + //删除剩余天数为0的数据 + adPlatformDeliveryService.deleteAllByDayNum(0); + + //所有的数据剩余天数-1 + adPlatformDeliveryService.updateAllByDayNum(); + + log.info("++++++++++++++++++++++ change plate form Sync end time {} ++++++++++++++++", DateUtil.now()); + } + +} diff --git a/src/main/java/com/baiyee/adcallback/task/PlatFormRecordTask.java b/src/main/java/com/baiyee/adcallback/task/PlatFormRecordTask.java index 2a61c20..a73e643 100644 --- a/src/main/java/com/baiyee/adcallback/task/PlatFormRecordTask.java +++ b/src/main/java/com/baiyee/adcallback/task/PlatFormRecordTask.java @@ -60,7 +60,9 @@ public class PlatFormRecordTask { /** * 十分钟执行任务 + * 采用新库 */ + @Deprecated @Scheduled(cron = "0 0/10 * * * ? ") @Transactional(rollbackFor = Exception.class) public void deliveryPlatform() { @@ -145,21 +147,4 @@ public class PlatFormRecordTask { } - /** - * 修改数据 - */ - @Scheduled(cron = "0 30 23 * * ? ") - @Transactional(rollbackFor = Exception.class) - public void autoDeal() { - log.info("++++++++++++++++++++++ change plate form sync start time {} ++++++++++++++++", DateUtil.now()); - - //删除剩余天数为0的数据 - adPlatformDeliveryService.deleteAllByDayNum(0); - - //所有的数据剩余天数-1 - adPlatformDeliveryService.updateAllByDayNum(); - - log.info("++++++++++++++++++++++ change plate form Sync end time {} ++++++++++++++++", DateUtil.now()); - } - } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 0936e22..7408ec5 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -2,52 +2,229 @@ x-debug: mode: '1' #配置数据源 spring: - datasource: - druid: - db-type: com.alibaba.druid.pool.DruidDataSource - driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://localhost/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true - username: root - password: root - # 初始连接数 - initial-size: 5 - # 最小连接数 - min-idle: 10 - # 最大连接数 - max-active: 20 - # 获取连接超时时间 - max-wait: 5000 - # 连接有效性检测时间 - time-between-eviction-runs-millis: 60000 - # 连接在池中最小生存的时间 - min-evictable-idle-time-millis: 300000 - # 连接在池中最大生存的时间 - max-evictable-idle-time-millis: 900000 - test-while-idle: true - test-on-borrow: false - test-on-return: false - # # 检测连接是否有效 - validation-query: select 1 - # 配置监控统计 - webStatFilter: - enabled: true - stat-view-servlet: - enabled: true - url-pattern: /druid/* - reset-enable: false - filter: - stat: - enabled: true - # 记录慢SQL - log-slow-sql: true - slow-sql-millis: 1000 - merge-sql: true + autoconfigure: + exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure + shardingsphere: + # 数据源配置 + datasource: + common: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.cj.jdbc.Driver + # 初始连接数 + initial-size: 6 + # 最小连接池数量 + min-idle: 3 + # 最大连接池数量 + maxActive: 20 + # 配置获取连接等待超时的时间 + maxWait: 60000 + # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 + timeBetweenEvictionRunsMillis: 60000 + # 配置一个连接在池中最小生存的时间，单位是毫秒 + minEvictableIdleTimeMillis: 300000 + # 配置一个连接在池中最大生存的时间，单位是毫秒 + maxEvictableIdleTimeMillis: 900000 + #Oracle需要打开注释 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + # 打开PSCache，并且指定每个连接上PSCache的大小 + poolPreparedStatements: true + maxPoolPreparedStatementPerConnectionSize: 20 + # 配置监控统计拦截的filters，去掉后监控界面sql无法统计，'wall'用于防火墙 + filters: stat,wall,slf4j + # 通过connectProperties属性来打开mergeSql功能；慢SQL记录 + connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000 wall: - config: - multi-statement-allow: true - connect-properties: druid.stat.mergeSql=false + multi-statement-allow: true + filter: + stat: + enabled: true + # 记录慢SQL + logSlowSql: true + slowSqlMillis: 1000 + mergeSql: true + # 数据源名称，多数据源以逗号分隔 + names: db0,db1 + db0: + # 数据库连接池类名称 + type: com.alibaba.druid.pool.DruidDataSource + # 数据库驱动类名 + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://8.130.96.163:3306/db_ad?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai + username: root + password: y7z7noq2 + db1: + # 数据库连接池类名称 + type: com.alibaba.druid.pool.DruidDataSource + # 数据库驱动类名 + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://8.130.96.163:3306/db_ad1?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai + username: root + password: y7z7noq2 + # 规则配置 + rules: + sharding: + # 分片算法配置 + sharding-algorithms: + database-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式（算法自行定义，此处为方便演示效果） + algorithm-expression: db$->{id % 2} + jl-v1-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_jl_v1_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + jl-v2-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_jl_v2_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + bd-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_bd_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + gdt-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_gdt_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + ks-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_ks_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + # 分布式序列算法配置 + key-generators: + snowflake: + # 分布式序列算法（雪花算法：SNOWFLAKE； UUID：UUID）（注：UUID没有props配置） https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/keygen/ + type: SNOWFLAKE + # 分布式序列算法属性配置 + props: + # 工作机器唯一标识 + worker-id: 32 + tables: + # 逻辑表名称 + tb_jl_v1_backdata: + # 行表达式标识符可以使用 ${...} 或 $->{...}，但前者与 Spring 本身的属性文件占位符冲突，因此在 Spring 环境中使用行表达式标识符建议使用 $->{...} + actual-data-nodes: db$->{0..1}.tb_jl_v1_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: jl-v1-table-inline + # 逻辑表名称 + tb_jl_v2_backdata: + actual-data-nodes: db$->{0..1}.tb_jl_v2_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: jl-v2-table-inline + # 逻辑表名称 + tb_bd_new_backdata: + actual-data-nodes: db$->{0..1}.tb_bd_new_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: bd-table-inline + # 逻辑表名称 + tb_gdt_new_backdata: + actual-data-nodes: db$->{0..1}.tb_gdt_new_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: gdt-table-inline + # 逻辑表名称 + tb_ks_new_backdata: + actual-data-nodes: db$->{0..1}.tb_ks_new_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: ks-table-inline + + # 属性配置 + props: + # 展示修改以后的sql语句 + sql-show: true platform: authToken: JI8AeA7POKsdGcBC url: http://8.130.96.163:8866/api/taskImei/callback/add +tf: + source: + path: /home/www/ad-bak/source/ diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 67e6af1..d678bbc 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -1,53 +1,233 @@ #配置数据源 spring: - datasource: - druid: - db-type: com.alibaba.druid.pool.DruidDataSource - driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://localhost:3306/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true - username: root - password: Yuyou@2020 - # 初始连接数 - initial-size: 5 - # 最小连接数 - min-idle: 10 - # 最大连接数 - max-active: 20 - # 获取连接超时时间 - max-wait: 5000 - # 连接有效性检测时间 - time-between-eviction-runs-millis: 60000 - # 连接在池中最小生存的时间 - min-evictable-idle-time-millis: 300000 - # 连接在池中最大生存的时间 - max-evictable-idle-time-millis: 900000 - test-while-idle: true - test-on-borrow: false - test-on-return: false - # 检测连接是否有效 - validation-query: select 1 - # 配置监控统计 - webStatFilter: - enabled: true - stat-view-servlet: - enabled: true - url-pattern: /druid/* - reset-enable: false - filter: - stat: - enabled: true - # 记录慢SQL - log-slow-sql: true - slow-sql-millis: 1000 - merge-sql: true + autoconfigure: + exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure + shardingsphere: + # 数据源配置 + datasource: + common: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.cj.jdbc.Driver + # 初始连接数 + initial-size: 6 + # 最小连接池数量 + min-idle: 3 + # 最大连接池数量 + maxActive: 20 + # 配置获取连接等待超时的时间 + maxWait: 60000 + # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 + timeBetweenEvictionRunsMillis: 60000 + # 配置一个连接在池中最小生存的时间，单位是毫秒 + minEvictableIdleTimeMillis: 300000 + # 配置一个连接在池中最大生存的时间，单位是毫秒 + maxEvictableIdleTimeMillis: 900000 + #Oracle需要打开注释 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + # 打开PSCache，并且指定每个连接上PSCache的大小 + poolPreparedStatements: true + maxPoolPreparedStatementPerConnectionSize: 20 + # 配置监控统计拦截的filters，去掉后监控界面sql无法统计，'wall'用于防火墙 + filters: stat,wall,slf4j + # 通过connectProperties属性来打开mergeSql功能；慢SQL记录 + connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000 wall: - config: - multi-statement-allow: true - connect-properties: druid.stat.mergeSql=false + multi-statement-allow: true + filter: + stat: + enabled: true + # 记录慢SQL + logSlowSql: true + slowSqlMillis: 1000 + mergeSql: true + # 数据源名称，多数据源以逗号分隔 + names: db0,db1 + db0: + # 数据库连接池类名称 + type: com.alibaba.druid.pool.DruidDataSource + # 数据库驱动类名 + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://localhost:3306/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true + username: root + password: Yuyou@2020 + db1: + # 数据库连接池类名称 + type: com.alibaba.druid.pool.DruidDataSource + # 数据库驱动类名 + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://localhost:3306/db_ad?serverTimezone=Asia/Shanghai&useUnicode=true&characterEncoding=utf8&useSSL=false&rewriteBatchedStatements=true + username: root + password: Yuyou@2020 + # 规则配置 + rules: + sharding: + # 分片算法配置 + sharding-algorithms: + database-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式（算法自行定义，此处为方便演示效果） + algorithm-expression: db$->{id % 2} + jl-v1-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_jl_v1_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + jl-v2-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_jl_v2_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + bd-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_bd_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + gdt-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_gdt_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + ks-table-inline: + # 分片算法类型 https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/sharding/ + type: INLINE + props: + # 分片算法的行表达式 + algorithm-expression: tb_ks_new_backdata_$->{(tag.hashCode() & Integer.MAX_VALUE) % 16} + # 分布式序列算法配置 + key-generators: + snowflake: + # 分布式序列算法（雪花算法：SNOWFLAKE； UUID：UUID）（注：UUID没有props配置） https://shardingsphere.apache.org/document/current/cn/user-manual/shardingsphere-jdbc/configuration/built-in-algorithm/keygen/ + type: SNOWFLAKE + # 分布式序列算法属性配置 + props: + # 工作机器唯一标识 + worker-id: 32 + tables: + # 逻辑表名称 + tb_jl_v1_backdata: + # 行表达式标识符可以使用 ${...} 或 $->{...}，但前者与 Spring 本身的属性文件占位符冲突，因此在 Spring 环境中使用行表达式标识符建议使用 $->{...} + actual-data-nodes: db$->{0..1}.tb_jl_v1_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: jl-v1-table-inline + # 逻辑表名称 + tb_jl_v2_backdata: + actual-data-nodes: db$->{0..1}.tb_jl_v2_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: jl-v2-table-inline + # 逻辑表名称 + tb_bd_new_backdata: + actual-data-nodes: db$->{0..1}.tb_bd_new_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: bd-table-inline + # 逻辑表名称 + tb_gdt_new_backdata: + actual-data-nodes: db$->{0..1}.tb_gdt_new_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: gdt-table-inline + # 逻辑表名称 + tb_ks_new_backdata: + actual-data-nodes: db$->{0..1}.tb_ks_new_backdata_${0..15} + key-generate-strategy: + column: id + key-generator-name: snowflake + # 分库策略 + database-strategy: + standard: + # 分片列名称 + sharding-column: id + # 分片算法名称 + sharding-algorithm-name: database-inline + # 分表策略 + table-strategy: + standard: + # 分片列名称 + sharding-column: tag + # 分片算法名称 + sharding-algorithm-name: ks-table-inline + + # 属性配置 + props: + # 展示修改以后的sql语句 + sql-show: true platform: authToken: nqJpVSf3UcrEcVIH url: https://baiyee.vip/api/taskImei/callback/add tf: source: - path: /home/www/ad-bak/source/ \ No newline at end of file + path: /home/www/ad-bak/source/ + + +#配置数据源 + + + diff --git a/src/test/java/com/baiyee/adcallback/TagTest.java b/src/test/java/com/baiyee/adcallback/TagTest.java new file mode 100644 index 0000000..ebc96b1 --- /dev/null +++ b/src/test/java/com/baiyee/adcallback/TagTest.java @@ -0,0 +1,79 @@ +package com.baiyee.adcallback; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.text.CharSequenceUtil; +import cn.hutool.core.text.csv.CsvData; +import cn.hutool.core.text.csv.CsvReader; +import cn.hutool.core.text.csv.CsvUtil; +import com.baiyee.adcallback.api.dto.PlateFormDTO; +import com.baiyee.adcallback.api.vo.KSNewMobileMonitorCallbackRequestVO; +import com.baiyee.adcallback.common.enums.DeliveryPlatformEnum; +import com.baiyee.adcallback.repository.TbEquipmentEntityRepository; +import com.baiyee.adcallback.repository.TbJLV1BackdataEntityRepository; +import com.baiyee.adcallback.repository.entity.TbEquipmentEntity; +import com.baiyee.adcallback.repository.entity.TbJLBackDataV1Entity; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import javax.annotation.Resource; +import java.io.File; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author Enzo + * @date : 2023/3/1 + */ + +@Slf4j +@RunWith(SpringRunner.class) +@SpringBootTest(classes = AdCallbackApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public class TagTest { + + @Resource + private TbJLV1BackdataEntityRepository jlv1BackdataEntityRepository; + + + @Test + public void getSetting() { + + List tbJLBackDataV1Entities = jlv1BackdataEntityRepository.queryBytag("2r310nar8x59"); + ArrayList strings = Lists.newArrayList("arc1iiemw9ti","r55t3fp2dio5"); + + List imei__ = jlv1BackdataEntityRepository.queryPlateFormDTO + (CharSequenceUtil.EMPTY, "__IMEI__", DateUtil.beginOfDay(DateTime.now()), DateUtil.endOfDay(DateTime.now()), strings); + + System.out.println(); + } + + @Test + public void filter() { + + + ArrayList objects = Lists.newArrayList(); + ArrayList a = Lists.newArrayList(); + List list = Lists.newArrayList("jl-aaa-das", "jl-aaa-das-v2", "jl-aaa-das"); + + for (String s1 : list) { + int v2 = s1.lastIndexOf("v2"); + if (v2 > 0) { + objects.add(s1); + continue; + } + a.add(s1); + } + System.out.println(); + } + + +}