From cd444b568463d14f88bda7aa1da9c8f710cfa177 Mon Sep 17 00:00:00 2001 From: bynt <13586541001@163.com> Date: Wed, 17 Jan 2024 10:59:50 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=BA=BF=E7=A8=8B=E6=B1=A0?= =?UTF-8?q?=E6=95=B0=E9=87=8F=20=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/properties/DeliveryProperties.java | 7 + .../modules/platform/domain/ModeRecord.java | 4 + .../repository/ModeRecordRepository.java | 4 +- .../platform/rest/ModeRecordController.java | 12 +- .../platform/service/ModeRecordService.java | 6 +- .../platform/service/dto/BuildTaskDTO.java | 22 +++ .../modules/platform/service/dto/ImeiDTO.java | 4 + .../platform/service/dto/ModelRecordDto.java | 3 + .../impl/DeliveryRecordServiceImpl.java | 7 +- .../service/impl/ModelRecordServiceImpl.java | 97 +++++++----- .../java/com/baiye/timed/DecryptTask.java | 144 ++++++++++-------- .../main/resources/config/application-dev.yml | 8 +- .../resources/config/application-prod.yml | 6 +- .../src/main/resources/config/application.yml | 2 +- .../src/main/resources/application-dev.yml | 4 +- .../src/main/resources/application-dev.yml | 4 +- .../src/main/resources/application-prod.yml | 4 +- .../src/main/resources/application-test.yml | 4 +- 18 files changed, 214 insertions(+), 128 deletions(-) create mode 100644 ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/BuildTaskDTO.java diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/config/properties/DeliveryProperties.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/config/properties/DeliveryProperties.java index 2915d860..7bbd2b6e 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/config/properties/DeliveryProperties.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/config/properties/DeliveryProperties.java @@ -54,4 +54,11 @@ public class DeliveryProperties { private String dmpDownPath; + @ApiModelProperty("建模下载地址") + private String modelPath; + + @ApiModelProperty("建模下载地址") + private String modelUrl; + + } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/domain/ModeRecord.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/domain/ModeRecord.java index ce5e6013..6efd9698 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/domain/ModeRecord.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/domain/ModeRecord.java @@ -79,4 +79,8 @@ public class ModeRecord extends BaseEntity implements Serializable { @Column(name = "tag_str") private String tagStr; + @ApiModelProperty("下载地址") + @Column(name = "down_str") + private String downUrl; + } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/repository/ModeRecordRepository.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/repository/ModeRecordRepository.java index cecb38e9..72e18580 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/repository/ModeRecordRepository.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/repository/ModeRecordRepository.java @@ -63,8 +63,8 @@ public interface ModeRecordRepository extends JpaRepository, J * @return */ @Modifying - @Query("UPDATE ModeRecord set recordStatus = ?1,buildTime = ?2,sendNum = ?3 where id = ?4") - int updateStatusAndTimeById(Integer num, DateTime date, Integer count, Long id); + @Query("UPDATE ModeRecord set recordStatus = ?1,buildTime = ?2,sendNum = ?3 where id in ?4") + int updateStatusAndTimeById(Integer num, DateTime date, Integer count, List id); /** diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/rest/ModeRecordController.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/rest/ModeRecordController.java index a4310229..b0e996c3 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/rest/ModeRecordController.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/rest/ModeRecordController.java @@ -15,11 +15,13 @@ */ package com.baiye.modules.platform.rest; +import cn.hutool.core.collection.CollUtil; import com.baiye.annotation.Inner; import com.baiye.annotation.Log; import com.baiye.constant.DefaultNumberConstants; import com.baiye.http.CommonResponse; import com.baiye.modules.platform.service.ModeRecordService; +import com.baiye.modules.platform.service.dto.BuildTaskDTO; import com.baiye.modules.platform.service.dto.ModeRecordQueryCriteria; import com.baiye.util.SecurityUtils; import io.swagger.annotations.Api; @@ -70,11 +72,11 @@ public class ModeRecordController { @ApiOperation("创建任务") - @GetMapping("/build") - public CommonResponse buildModel(@RequestParam("id") Long id, @RequestParam(value = "actName") String actName, - @RequestParam("actId") String actId) { - return Boolean.TRUE.equals(modeRecordService.sendDataByActNameAndActId(id, actName, actId)) ? - CommonResponse.createBySuccess() : CommonResponse.createByError(); + @PostMapping("/build") + public CommonResponse buildModel(@RequestBody BuildTaskDTO buildTaskDTO) { + return Boolean.TRUE.equals(modeRecordService.sendDataByActNameAndActId + (CollUtil.newArrayList(buildTaskDTO.getIds()), buildTaskDTO.getActName(), buildTaskDTO.getActId())) ? + CommonResponse.createBySuccess() : CommonResponse.createByError(); } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/ModeRecordService.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/ModeRecordService.java index a2d26f2f..2888da38 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/ModeRecordService.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/ModeRecordService.java @@ -4,6 +4,8 @@ import com.baiye.modules.platform.service.dto.ModeRecordQueryCriteria; import org.springframework.data.domain.Pageable; import org.springframework.web.multipart.MultipartFile; +import java.util.List; + /** * @author Enzo * @date : 2024/1/4 @@ -46,10 +48,10 @@ public interface ModeRecordService { /** * 创建任务 * - * @param id + * @param ids * @param actName * @param actId * @return */ - Boolean sendDataByActNameAndActId(Long id, String actName, String actId); + Boolean sendDataByActNameAndActId(List ids, String actName, String actId); } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/BuildTaskDTO.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/BuildTaskDTO.java new file mode 100644 index 00000000..21b86b89 --- /dev/null +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/BuildTaskDTO.java @@ -0,0 +1,22 @@ +package com.baiye.modules.platform.service.dto; + +import lombok.Data; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; + +/** + * @author Enzo + * @date : 2024/1/16 + */ +@Data +public class BuildTaskDTO { + @NotEmpty(message = "分发任务不能为空") + private Long[] ids; + + @NotBlank(message = "actName不能为空") + private String actName; + + @NotBlank(message = "actId不能为空") + private String actId; +} diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ImeiDTO.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ImeiDTO.java index 59b90a9b..c43f6e66 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ImeiDTO.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ImeiDTO.java @@ -25,4 +25,8 @@ public class ImeiDTO implements Serializable { @ApiModelProperty(value = "tag") private String tag; + + @ApiModelProperty(value = "time") + private String time; + } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ModelRecordDto.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ModelRecordDto.java index c3f83615..55dde850 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ModelRecordDto.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/dto/ModelRecordDto.java @@ -66,5 +66,8 @@ public class ModelRecordDto extends BaseDTO implements Serializable { @ApiModelProperty("层级") private Integer treeLevel; + @ApiModelProperty("下载地址") + private String downUrl; + private List treeList; } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/DeliveryRecordServiceImpl.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/DeliveryRecordServiceImpl.java index 0adfba08..55e3f1a9 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/DeliveryRecordServiceImpl.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/DeliveryRecordServiceImpl.java @@ -18,6 +18,7 @@ import com.baiye.modules.platform.service.dto.ImeiDTO; import com.google.common.collect.Lists; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import java.util.List; @@ -70,8 +71,10 @@ public class DeliveryRecordServiceImpl implements DeliveryRecordService { List list = Lists.newArrayList(); dtoList.forEach(imeiDTO -> { ExchangeStripsDTO.Data data = new ExchangeStripsDTO.Data(); - data.setData(Base64.encode(imeiDTO.getImei().concat(StrPool.COMMA).concat(imeiDTO.getTag()))); - list.add(data); + if (StringUtils.isNotBlank(imeiDTO.getImei()) && StringUtils.isNotBlank(imeiDTO.getTag())) { + data.setData(Base64.encode(imeiDTO.getImei().concat(StrPool.COMMA).concat(imeiDTO.getTag()))); + list.add(data); + } }); ExchangeStripsDTO build = ExchangeStripsDTO.builder().batchId(taskImeiName).reqId (RandomUtil.randomString(DefaultNumberConstants.TEN_NUMBER)).orgCode diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/ModelRecordServiceImpl.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/ModelRecordServiceImpl.java index 71f3504e..5f7ff198 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/ModelRecordServiceImpl.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/modules/platform/service/impl/ModelRecordServiceImpl.java @@ -2,12 +2,11 @@ package com.baiye.modules.platform.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.text.CharSequenceUtil; -import cn.hutool.core.text.csv.CsvData; -import cn.hutool.core.text.csv.CsvReader; -import cn.hutool.core.text.csv.CsvRow; -import cn.hutool.core.text.csv.CsvUtil; +import cn.hutool.core.text.StrPool; +import cn.hutool.core.text.csv.*; +import cn.hutool.core.util.CharsetUtil; +import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.ObjectUtil; -import cn.hutool.core.util.StrUtil; import cn.hutool.poi.excel.ExcelReader; import cn.hutool.poi.excel.ExcelUtil; import com.baiye.config.properties.DeliveryProperties; @@ -38,9 +37,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; /** @@ -94,9 +91,12 @@ public class ModelRecordServiceImpl implements ModeRecordService { } // 去除重复数据 List dtoArrayList = Lists.newArrayList(Sets.newHashSet(stringList)); + // 保存文件 + String downUrl = getDownUrl(stringList); + // 保存父类数据 Long recordId = saveRecord(taskName, uploadType, userId, (long) DefaultNumberConstants.ZERO_NUMBER, - DefaultNumberConstants.ONE_NUMBER, CharSequenceUtil.EMPTY, dtoArrayList); + DefaultNumberConstants.ONE_NUMBER, CharSequenceUtil.EMPTY, downUrl, dtoArrayList); // tag分组 Map> collect = dtoArrayList.stream(). collect(Collectors.groupingBy(ImeiDTO::getTag)); @@ -105,7 +105,7 @@ public class ModelRecordServiceImpl implements ModeRecordService { List imeiDTOList = entry.getValue(); // 保存子类数据 Long childrenId = saveRecord(taskName, uploadType, userId, recordId, - DefaultNumberConstants.TWO_NUMBER, tag, imeiDTOList); + DefaultNumberConstants.TWO_NUMBER, tag, null, imeiDTOList); if (uploadType < DefaultNumberConstants.TWO_NUMBER) { // 每次100000 数据 List> partition = Lists.partition @@ -128,6 +128,22 @@ public class ModelRecordServiceImpl implements ModeRecordService { return Boolean.TRUE; } + private String getDownUrl(List stringList) { + String uuid = IdUtil.fastSimpleUUID(); + String fileName = deliveryProperties.getModelUrl().concat(StrPool.SLASH).concat(uuid); + String csvPath = fileName.concat(StrPool.DOT).concat(FileConstant.CSV_FILE_SUB_NAME); + CsvWriter writer = CsvUtil.getWriter(csvPath, CharsetUtil.CHARSET_UTF_8); + String zipPath = fileName.concat(FileConstant.ZIP_FILE_SUB_NAME); + writer.writeBeans(stringList); + writer.close(); + // 设置压缩文件 + CompressUtil.decryptionCompression(zipPath, csvPath, null); + String filePath = zipPath.substring + (zipPath.lastIndexOf(StrPool.SLASH) + DefaultNumberConstants.ONE_NUMBER); + cn.hutool.core.io.FileUtil.del(csvPath); + return deliveryProperties.getModelPath().concat(filePath); + } + @Override public String findPath(Long downId) { ModeRecord downRecord = modeRecordRepository.findById(downId).orElseGet(ModeRecord::new); @@ -150,38 +166,41 @@ public class ModelRecordServiceImpl implements ModeRecordService { @Override @SneakyThrows @Transactional(rollbackFor = Exception.class) - public Boolean sendDataByActNameAndActId(Long id, String actName, String actId) { + public Boolean sendDataByActNameAndActId(List ids, String actName, String actId) { List stringList = Lists.newArrayList(); - ModeRecord downRecord = modeRecordRepository.findById(id).orElseGet(ModeRecord::new); - if (ObjectUtil.isNull(downRecord) || - downRecord.getRecordStatus() != DefaultNumberConstants.ONE_NUMBER) { - throw new BadRequestException("该批次暂不能建模!"); + for (Long id : ids) { + ModeRecord downRecord = modeRecordRepository.findById(id).orElseGet(ModeRecord::new); + if (ObjectUtil.isNull(downRecord) || + downRecord.getRecordStatus() != DefaultNumberConstants.ONE_NUMBER) { + throw new BadRequestException("该批次暂不能建模!"); + } + // 解压文件 + String unzipPath = CompressUtil.unzipFiles + (deliveryProperties.getFileUrl(), downRecord.getUploadPath(), deliveryProperties.getZipPassword()); + File unzipFile = new File(unzipPath); + File parseFile = Objects.requireNonNull + (unzipFile.listFiles())[DefaultNumberConstants.ZERO_NUMBER]; + CsvReader reader = CsvUtil.getReader(); + CsvData data = reader.read(parseFile); + // csv通配 + data.getRows().forEach(str -> stringList.add(str.size() > + DefaultNumberConstants.THREE_NUMBER ? str.get(DefaultNumberConstants.TWO_NUMBER) + : str.get(DefaultNumberConstants.ZERO_NUMBER))); + cn.hutool.core.io.FileUtil.del(unzipPath); + // 修改状态 + modeRecordRepository.updateStatusById(DefaultNumberConstants.TWO_NUMBER, id); } - // 解压文件 - String unzipPath = CompressUtil.unzipFiles - (deliveryProperties.getFileUrl(), downRecord.getUploadPath(), deliveryProperties.getZipPassword()); - File unzipFile = new File(unzipPath); - File parseFile = Objects.requireNonNull - (unzipFile.listFiles())[DefaultNumberConstants.ZERO_NUMBER]; - CsvReader reader = CsvUtil.getReader(); - CsvData data = reader.read(parseFile); - // csv通配 - data.getRows().forEach(str -> stringList.add(str.size() > - DefaultNumberConstants.THREE_NUMBER ? str.get(DefaultNumberConstants.TWO_NUMBER) - : str.get(DefaultNumberConstants.ZERO_NUMBER))); - cn.hutool.core.io.FileUtil.del(unzipPath); - // 修改状态 - modeRecordRepository.updateStatusById(DefaultNumberConstants.TWO_NUMBER, id); + decryptTask.doRunSendTask(stringList, ids, actId, actName); // 调用异步任务 - decryptTask.doRunSendTask(stringList, id, actId, actName); log.info("============= the request str list {} =============", stringList.size()); return Boolean.TRUE; } - private Long saveRecord(String taskName, Integer uploadType, Long userId, Long parentId, Integer levelNum, String tagStr, List dtoArrayList) { + private Long saveRecord(String taskName, Integer uploadType, Long userId, Long parentId, Integer levelNum, String tagStr, String downUrl, List dtoArrayList) { ModeRecord modeRecord = new ModeRecord(); modeRecord.setUserId(userId); modeRecord.setTagStr(tagStr); + modeRecord.setDownUrl(downUrl); modeRecord.setParentId(parentId); modeRecord.setTaskName(taskName); modeRecord.setTreeLevel(levelNum); @@ -201,24 +220,30 @@ public class ModelRecordServiceImpl implements ModeRecordService { || originalFilename.endsWith(FileConstant.XLSX_FILE_SUB_NAME)) { ExcelReader reader = ExcelUtil.getReader(upload); for (List objects : reader.read()) { + ImeiDTO dto = new ImeiDTO(); if (objects.size() > DefaultNumberConstants.ONE_NUMBER) { - ImeiDTO dto = new ImeiDTO(); dto.setTag(objects.get(DefaultNumberConstants.ONE_NUMBER).toString()); dto.setImei(objects.get(DefaultNumberConstants.ZERO_NUMBER).toString()); - dtoList.add(dto); } + if (objects.size() > DefaultNumberConstants.TWO_NUMBER) { + dto.setTime(objects.get(DefaultNumberConstants.TWO_NUMBER).toString()); + } + dtoList.add(dto); } } if (originalFilename.endsWith(FileConstant.CSV_FILE_SUB_NAME)) { CsvReader reader = CsvUtil.getReader(); CsvData read = reader.read(upload); for (CsvRow row : read.getRows()) { + ImeiDTO dto = new ImeiDTO(); if (row.size() > DefaultNumberConstants.ONE_NUMBER) { - ImeiDTO dto = new ImeiDTO(); dto.setTag(row.get(DefaultNumberConstants.ONE_NUMBER)); dto.setImei(row.get(DefaultNumberConstants.ZERO_NUMBER)); - dtoList.add(dto); } + if (row.size() > DefaultNumberConstants.TWO_NUMBER) { + dto.setTime(row.get(DefaultNumberConstants.TWO_NUMBER)); + } + dtoList.add(dto); } } } diff --git a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/timed/DecryptTask.java b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/timed/DecryptTask.java index df8a8c01..4098c08c 100644 --- a/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/timed/DecryptTask.java +++ b/ad-platform-manage/ad-platform-management/src/main/java/com/baiye/timed/DecryptTask.java @@ -1,5 +1,6 @@ package com.baiye.timed; +import cn.hutool.core.collection.CollUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.text.StrPool; @@ -81,67 +82,72 @@ public class DecryptTask { // 取出线程池 Executor executor = threadConfig.getExecutor(); List>> results = Lists.newArrayList(); - // 取出值 - List sendList = dtoArrayList.stream().map(ImeiDTO::getImei).collect(Collectors.toList()); - List> partition = Lists.partition(sendList, DefaultNumberConstants.TWO_HUNDRED); - int batchNum = partition.size(); - stopWatch.start(); - for (List list : partition) { - TimeUnit.MILLISECONDS.sleep(DefaultNumberConstants.ONE_THOUSAND); - // 异步线程池 - CompletableFuture> future = CompletableFuture.supplyAsync(() -> { - Gson gson = new Gson(); - // 转换 - String encryptStr = StringUtils.substringBeforeLast(Joiner.on(StrPool.COMMA).skipNulls().join(list), StrPool.COMMA); - // 构造请求 - DecryptionJsonRequest build = - DecryptionJsonRequest.builder().cellphoneMd5 - (encryptStr).requestId(RandomUtil.randomString(DefaultNumberConstants.TEN_NUMBER)).build(); - try { - // 调用HTTP请求发送数据 - DecryptionDTO decryptionDTO = JSONUtil.toBean(HttpUtil.post(decryptAddress, gson.toJson(build)), DecryptionDTO.class); - if (StringUtils.isNotBlank(decryptionDTO.getReason()) - && SecurityConstants.SUCCESS.equalsIgnoreCase(decryptionDTO.getReason()) && (StringUtils.isNotBlank(decryptionDTO.getCellPhone()))) { - List parseArray = JSONUtil.toList(decryptionDTO.getCellPhone(), String.class); - if (!parseArray.isEmpty()) { - parseArray = parseArray.stream().filter - (StringUtils::isNotBlank).collect(Collectors.toList()); - log.info("================ response size as {} ================", parseArray.size()); - return EncryptPhoneUtil.encryptPhoneList(parseArray); + // 取出值并剔除空值 + List sendList = dtoArrayList.stream().map(ImeiDTO::getImei).filter(StringUtils::isNotBlank).collect(Collectors.toList()); + if (CollUtil.isNotEmpty(sendList)) { + List> partition = Lists.partition(sendList, DefaultNumberConstants.TWO_HUNDRED); + int batchNum = partition.size(); + stopWatch.start(); + for (List list : partition) { + TimeUnit.MILLISECONDS.sleep(DefaultNumberConstants.ONE_THOUSAND); + // 异步线程池 + CompletableFuture> future = CompletableFuture.supplyAsync(() -> { + Gson gson = new Gson(); + // 转换 + String encryptStr = StringUtils.substringBeforeLast(Joiner.on(StrPool.COMMA).skipNulls().join(list), StrPool.COMMA); + // 构造请求 + DecryptionJsonRequest build = + DecryptionJsonRequest.builder().cellphoneMd5 + (encryptStr).requestId(RandomUtil.randomString(DefaultNumberConstants.TEN_NUMBER)).build(); + try { + // 调用HTTP请求发送数据 + DecryptionDTO decryptionDTO = JSONUtil.toBean(HttpUtil.post(decryptAddress, gson.toJson(build)), DecryptionDTO.class); + log.info("================ the decrypt result as {} ================",JSONUtil.toJsonStr(decryptionDTO.getResult())); + if (StringUtils.isNotBlank(decryptionDTO.getReason()) + && SecurityConstants.SUCCESS.equalsIgnoreCase(decryptionDTO.getReason()) && (StringUtils.isNotBlank(decryptionDTO.getCellPhone()))) { + List parseArray = JSONUtil.toList(decryptionDTO.getCellPhone(), String.class); + if (!parseArray.isEmpty()) { + parseArray = parseArray.stream().filter + (StringUtils::isNotBlank).collect(Collectors.toList()); + log.info("================ response size as {} ================", parseArray.size()); + return EncryptPhoneUtil.encryptPhoneList(parseArray); + } } + } catch (Exception e) { + log.error("============ the request exception {} ============", e.getMessage()); } - } catch (Exception e) { - log.error("============ the request exception {} ============", e.getMessage()); - } - return Lists.newArrayList(); - }, executor); - results.add(future); + return Lists.newArrayList(); + }, executor); + results.add(future); + } + // 线程结束取出结果集 + CompletableFuture allCompletableFuture = CompletableFuture.allOf + (results.toArray(new CompletableFuture[batchNum])); + List> result = allCompletableFuture.thenApply(e -> results.stream().map + (CompletableFuture::join).collect(Collectors.toList())).join(); + result.forEach(stringList::addAll); + stopWatch.stop(); + if (CollUtil.isNotEmpty(stringList)) { + String file = deliveryProperties.getFileUrl().concat(StrPool.SLASH).concat(uuid); + // 进行文件保存 + String csvPath = file.concat(StrPool.DOT).concat(FileConstant.CSV_FILE_SUB_NAME); + String zipPath = file.concat(FileConstant.ZIP_FILE_SUB_NAME); + log.info("============= the save path as {} ,num {} ===================", csvPath, stringList.size()); + FileZipUtil.writeCSV(csvPath, stringList); + // 设置压缩文件 + CompressUtil.decryptionCompression(zipPath, csvPath, null); + // 修改状态并更改 + modeRecordRepository.updateStatusNumById(DefaultNumberConstants.ONE_NUMBER, stringList.size(), recordId); + modeRecordRepository.updateStatusAndPathById(DefaultNumberConstants.ONE_NUMBER, zipPath, stringList.size(), childrenId); + FileUtil.del(csvPath); + } } - // 线程结束取出结果集 - CompletableFuture allCompletableFuture = CompletableFuture.allOf - (results.toArray(new CompletableFuture[batchNum])); - List> result = allCompletableFuture.thenApply(e -> results.stream().map - (CompletableFuture::join).collect(Collectors.toList())).join(); - result.forEach(stringList::addAll); - stopWatch.stop(); - String file = deliveryProperties.getFileUrl().concat(StrPool.SLASH).concat(uuid); - // 进行文件保存 - String csvPath = file.concat(StrPool.DOT).concat(FileConstant.CSV_FILE_SUB_NAME); - String zipPath = file.concat(FileConstant.ZIP_FILE_SUB_NAME); - log.info("============= the save path as {} ,num {} ===================", csvPath, stringList.size()); - FileZipUtil.writeCSV(csvPath, stringList); - // 设置压缩文件 - CompressUtil.decryptionCompression(zipPath, csvPath, null); - // 修改状态并更改 - modeRecordRepository.updateStatusAndPathById(DefaultNumberConstants.ONE_NUMBER, zipPath, stringList.size(), childrenId); - modeRecordRepository.updateStatusNumById(DefaultNumberConstants.ONE_NUMBER, stringList.size(), recordId); - FileUtil.del(csvPath); log.info("================== run task end time {} ==================", DateUtil.now()); } @Async(value = "WorkExecutor") @Transactional(rollbackFor = Exception.class) - public void doRunSendTask(List stringList, Long id, String actId, String actName) { + public void doRunSendTask(List stringList, List ids, String actId, String actName) { log.info("================== run task begin time {} ==================", DateUtil.now()); int count = DefaultNumberConstants.ZERO_NUMBER; List> partition = Lists.partition(stringList, DefaultNumberConstants.ONE_THOUSAND); @@ -150,21 +156,25 @@ public class DecryptTask { dto.setActId(actId); dto.setActName(actName); for (List list : partition) { - List sendClientList = Lists.newArrayList(); - list.forEach(str -> { - CallImportDTO.Source client = new CallImportDTO.Source(); - client.setCellphone(str); - sendClientList.add(client); - }); - dto.setClientList(sendClientList); - String post = HttpUtil.post(sendClientAddress, JSONUtil.toJsonStr(dto)); - if (post.contains(XalanConstants.FEATURE_TRUE)) { - count += list.size(); + try { + List sendClientList = Lists.newArrayList(); + list.forEach(str -> { + CallImportDTO.Source client = new CallImportDTO.Source(); + client.setCellphone(str); + sendClientList.add(client); + }); + dto.setClientList(sendClientList); + String post = HttpUtil.post(sendClientAddress, JSONUtil.toJsonStr(dto)); + if (post.contains(XalanConstants.FEATURE_TRUE)) { + count += list.size(); + } + log.info("============ the submit result as {} =========", JSONUtil.toJsonStr(post)); + if (count > DefaultNumberConstants.ZERO_NUMBER) { + modeRecordRepository.updateStatusAndTimeById(DefaultNumberConstants.THREE_NUMBER, DateUtil.date(), count, ids); + } + } catch (Exception exception) { + log.error("============ the request exception {} ============", exception.getMessage()); } - log.info("============ the submit result as {} =========", JSONUtil.toJsonStr(post)); - } - if (count > DefaultNumberConstants.ZERO_NUMBER) { - modeRecordRepository.updateStatusAndTimeById(DefaultNumberConstants.THREE_NUMBER, DateUtil.date(), count, id); } log.info("================== run task end time {} ==================", DateUtil.now()); } diff --git a/ad-platform-manage/ad-platform-management/src/main/resources/config/application-dev.yml b/ad-platform-manage/ad-platform-management/src/main/resources/config/application-dev.yml index 6604f08e..9ee88de3 100644 --- a/ad-platform-manage/ad-platform-management/src/main/resources/config/application-dev.yml +++ b/ad-platform-manage/ad-platform-management/src/main/resources/config/application-dev.yml @@ -27,11 +27,11 @@ spring: username: root password: y7z7noq2 # 初始连接数 - initial-size: 5 + initial-size: 10 # 最小连接数 min-idle: 15 # 最大连接数 - max-active: 30 + max-active: 300 # 超时时间(以秒数为单位) remove-abandoned-timeout: 180 # 获取连接超时时间 @@ -199,13 +199,15 @@ platform: zipPassword: DB2022hjk213v1... exchangePassword: DA2023xlb213v1. fileUrl: /home/eladmin/mail + modelUrl: /home/eladmin/model emailAddress: ensoze@outlook.com genderEmailAddress: ensoze2023@outlook.com emailPassword: baiye2022 customerId: QISX0xz4l6fR3YL2sUNSpzM2 toEmailAddress: lambda0821@outlook.com toGenderEmailAddress: lambda0821@outlook.com - dmpDownPath: http:///39.100.77.21:8001/dmp/down/ + modelPath: http://39.100.77.21:8013/model/down/ + dmpDownPath: http://39.100.77.21:8013/dmp/down/ privateKey: MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAJi+nz0zudS+4Sh+SmTnRfzgJOBIEwlLlJhcRgUbInvFx03zUgdzOUrGsvD2OTPPssnY8dv+Zgz2kUSEDYaWWKUs+VAklVOo0sOxzRMaUfwHRBN3Eq9OkqibDerYf6WUdv1k1BJsXSIdTlz8m6rtVbPF2hz8x/kYyJFzhM2IKTITAgMBAAECgYAgTZAXvWy7lXLAwZSyKkce57hkxllgSd+vKTSVt9tfGcDAt4jNkoy3R7ZoR2ppjq9dCMh9ohuq+ipWtya1I+6zC5sflk9HI/rf+5bq3JRJvxq3EJYe5DlSjQitLUMRP6PQorHnZZj/bdqKgRvrulI8XtK5Fv9Cd4jhkbSZtzgpYQJBAOJu0nu4qJCqIYLCmFWDpRzi9cu8/TFCBLVDH0xhNi28JL6G8xOfdzxsQa8ZlLOxPwn56VbS3+Korq34WCOAG5cCQQCssI4I7dshlA7kXeurVSFvui1YV/7ofFOxRs019+V88tfwNby5TAS9YjX7AuvGkobjpBBNEkE0JExf69m6VBzlAkB5te4HuLNKx1gp7CVr2c43n7tVHynNf1n+gKzjJmGz5ayuiOVBx/aUkPAhiZOHnx9uYlnNZJ4ZPGhgdNwTgPnTAkB61pQSMe/AMOtu8ogjNck1CoAa6W0/vsBhx/VNQGsTuEJ2ciMuw65TcLrpNKi2daBR6XBXAnczOebCDKix7AcpAkEArrU+bx6GwR+UbgoNAl1KwFLrV50pasK7Mlp0BkHM0cz4BGB53O5ng+TJHzen03OgC9I1W1WFAYHj03lb84qM/w== publicKey: MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCYvp89M7nUvuEofkpk50X84CTgSBMJS5SYXEYFGyJ7xcdN81IHczlKxrLw9jkzz7LJ2PHb/mYM9pFEhA2GllilLPlQJJVTqNLDsc0TGlH8B0QTdxKvTpKomw3q2H+llHb9ZNQSbF0iHU5c/Juq7VWzxdoc/Mf5GMiRc4TNiCkyEwIDAQAB diff --git a/ad-platform-manage/ad-platform-management/src/main/resources/config/application-prod.yml b/ad-platform-manage/ad-platform-management/src/main/resources/config/application-prod.yml index 5617203f..67e09d0a 100644 --- a/ad-platform-manage/ad-platform-management/src/main/resources/config/application-prod.yml +++ b/ad-platform-manage/ad-platform-management/src/main/resources/config/application-prod.yml @@ -21,11 +21,11 @@ spring: username: root password: yuyou@RDS2020 # 初始连接数 - initial-size: 5 + initial-size: 10 # 最小连接数 min-idle: 15 # 最大连接数 - max-active: 30 + max-active: 300 # 超时时间(以秒数为单位) remove-abandoned-timeout: 180 # 获取连接超时时间 @@ -200,6 +200,7 @@ platform: zipPassword: DB2022hjk213v1... exchangePassword: DA2023xlb213v1. fileUrl: /home/eladmin/mail + modelUrl: /home/eladmin/model emailAddress: ensoze@outlook.com genderEmailAddress: ensoze2023@outlook.com emailPassword: baiye2022 @@ -207,6 +208,7 @@ platform: toGenderEmailAddress: ggggod_2022@outlook.com customerId: QISX0xz4l6fR3YL2sUNSpzM2 dmpDownPath: https://baiyee.vip/dmp/down/ + modelPath: https://baiyee.vip/model/down/ privateKey: MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAJi+nz0zudS+4Sh+SmTnRfzgJOBIEwlLlJhcRgUbInvFx03zUgdzOUrGsvD2OTPPssnY8dv+Zgz2kUSEDYaWWKUs+VAklVOo0sOxzRMaUfwHRBN3Eq9OkqibDerYf6WUdv1k1BJsXSIdTlz8m6rtVbPF2hz8x/kYyJFzhM2IKTITAgMBAAECgYAgTZAXvWy7lXLAwZSyKkce57hkxllgSd+vKTSVt9tfGcDAt4jNkoy3R7ZoR2ppjq9dCMh9ohuq+ipWtya1I+6zC5sflk9HI/rf+5bq3JRJvxq3EJYe5DlSjQitLUMRP6PQorHnZZj/bdqKgRvrulI8XtK5Fv9Cd4jhkbSZtzgpYQJBAOJu0nu4qJCqIYLCmFWDpRzi9cu8/TFCBLVDH0xhNi28JL6G8xOfdzxsQa8ZlLOxPwn56VbS3+Korq34WCOAG5cCQQCssI4I7dshlA7kXeurVSFvui1YV/7ofFOxRs019+V88tfwNby5TAS9YjX7AuvGkobjpBBNEkE0JExf69m6VBzlAkB5te4HuLNKx1gp7CVr2c43n7tVHynNf1n+gKzjJmGz5ayuiOVBx/aUkPAhiZOHnx9uYlnNZJ4ZPGhgdNwTgPnTAkB61pQSMe/AMOtu8ogjNck1CoAa6W0/vsBhx/VNQGsTuEJ2ciMuw65TcLrpNKi2daBR6XBXAnczOebCDKix7AcpAkEArrU+bx6GwR+UbgoNAl1KwFLrV50pasK7Mlp0BkHM0cz4BGB53O5ng+TJHzen03OgC9I1W1WFAYHj03lb84qM/w== publicKey: MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCYvp89M7nUvuEofkpk50X84CTgSBMJS5SYXEYFGyJ7xcdN81IHczlKxrLw9jkzz7LJ2PHb/mYM9pFEhA2GllilLPlQJJVTqNLDsc0TGlH8B0QTdxKvTpKomw3q2H+llHb9ZNQSbF0iHU5c/Juq7VWzxdoc/Mf5GMiRc4TNiCkyEwIDAQAB diff --git a/ad-platform-manage/ad-platform-management/src/main/resources/config/application.yml b/ad-platform-manage/ad-platform-management/src/main/resources/config/application.yml index 06b5d335..205dd3df 100644 --- a/ad-platform-manage/ad-platform-management/src/main/resources/config/application.yml +++ b/ad-platform-manage/ad-platform-management/src/main/resources/config/application.yml @@ -65,7 +65,7 @@ task: # 核心线程池大小 core-pool-size: 10 # 最大线程数 - max-pool-size: 30 + max-pool-size: 200 # 活跃时间 keep-alive-seconds: 60 # 队列容量 diff --git a/ad-platform-manage/ad-platform-task/src/main/resources/application-dev.yml b/ad-platform-manage/ad-platform-task/src/main/resources/application-dev.yml index 84203f8a..b68353f9 100644 --- a/ad-platform-manage/ad-platform-task/src/main/resources/application-dev.yml +++ b/ad-platform-manage/ad-platform-task/src/main/resources/application-dev.yml @@ -23,11 +23,11 @@ spring: # username: root # password: 12345678 # 初始连接数 - initial-size: 5 + initial-size: 10 # 最小连接数 min-idle: 15 # 最大连接数 - max-active: 30 + max-active: 300 # 超时时间(以秒数为单位) remove-abandoned-timeout: 180 # 获取连接超时时间 diff --git a/ad-platform-services/ad-platform-source/src/main/resources/application-dev.yml b/ad-platform-services/ad-platform-source/src/main/resources/application-dev.yml index 4281b293..e7516b14 100644 --- a/ad-platform-services/ad-platform-source/src/main/resources/application-dev.yml +++ b/ad-platform-services/ad-platform-source/src/main/resources/application-dev.yml @@ -21,11 +21,11 @@ spring: username: root password: y7z7noq2 # 初始连接数 - initial-size: 5 + initial-size: 10 # 最小连接数 min-idle: 15 # 最大连接数 - max-active: 30 + max-active: 300 # 超时时间(以秒数为单位) remove-abandoned-timeout: 180 # 获取连接超时时间 diff --git a/ad-platform-services/ad-platform-source/src/main/resources/application-prod.yml b/ad-platform-services/ad-platform-source/src/main/resources/application-prod.yml index 8b84f364..cf49ac31 100644 --- a/ad-platform-services/ad-platform-source/src/main/resources/application-prod.yml +++ b/ad-platform-services/ad-platform-source/src/main/resources/application-prod.yml @@ -18,11 +18,11 @@ spring: username: root password: yuyou@RDS2020 # 初始连接数 - initial-size: 5 + initial-size: 10 # 最小连接数 min-idle: 15 # 最大连接数 - max-active: 30 + max-active: 300 # 超时时间(以秒数为单位) remove-abandoned-timeout: 180 # 获取连接超时时间 diff --git a/ad-platform-services/ad-platform-source/src/main/resources/application-test.yml b/ad-platform-services/ad-platform-source/src/main/resources/application-test.yml index b0eb5b46..6e7c5f25 100644 --- a/ad-platform-services/ad-platform-source/src/main/resources/application-test.yml +++ b/ad-platform-services/ad-platform-source/src/main/resources/application-test.yml @@ -18,11 +18,11 @@ spring: username: root password: y7z7noq2 # 初始连接数 - initial-size: 5 + initial-size: 10 # 最小连接数 min-idle: 15 # 最大连接数 - max-active: 30 + max-active: 300 # 超时时间(以秒数为单位) remove-abandoned-timeout: 180 # 获取连接超时时间