diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/BuildRecordController.java b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/BuildRecordController.java index 1be201f..798ea59 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/BuildRecordController.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/BuildRecordController.java @@ -164,7 +164,7 @@ public class BuildRecordController { SendBigDataDTO sendBigDataDTO = new SendBigDataDTO(); BeanUtil.copyProperties(buildRecordSendVO, sendBigDataDTO); // 调用发送课包任务,开始发送 - sendBigDataTask.doRunTask(buildRecordSendVO.getResource(), sendBigDataDTO); + sendBigDataTask.doRunTask(buildRecordSendVO.getResource(), buildRecordSendVO.getRemoteRecord() , sendBigDataDTO); return new ResponseEntity<>(CommonResponse.createBySuccess(), HttpStatus.OK); } diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/vo/BuildRecordSendVO.java b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/vo/BuildRecordSendVO.java index 3019f57..784e7f4 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/vo/BuildRecordSendVO.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/vo/BuildRecordSendVO.java @@ -5,6 +5,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import me.zhengjie.modules.buildrecord.domain.BuildRecord; +import me.zhengjie.modules.remoterec.domain.RemoteRecord; import java.io.Serializable; @@ -29,6 +30,11 @@ public class BuildRecordSendVO implements Serializable { */ private BuildRecord resource; + /** + * 远程上传的实体 + */ + private RemoteRecord remoteRecord; + /** * 需要发送的活动名称 */ diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java index b379b83..6222316 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java @@ -12,6 +12,8 @@ import me.zhengjie.modules.buildrecord.domain.BuildRecord; import me.zhengjie.modules.buildrecord.service.BuildRecordService; import me.zhengjie.modules.buildrecord.service.dto.BuildRecordDto; import me.zhengjie.modules.buildrecord.task.dto.SendBigDataDTO; +import me.zhengjie.modules.remoterec.domain.RemoteRecord; +import me.zhengjie.modules.remoterec.service.RemoteRecordService; import me.zhengjie.modules.tag.domain.Tag; import me.zhengjie.modules.tag.service.TagService; import me.zhengjie.modules.tag.service.dto.TagDto; @@ -43,6 +45,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static me.zhengjie.modules.common.consts.SendBigDataConst.NON_FINISH_SEND_STATATUS; +import static me.zhengjie.modules.remoterec.consts.RemoteRecConst.SUCCESS_TAG; @Component @Scope("prototype") @@ -84,17 +87,19 @@ public class SendBigDataTask { private TagService tagService; @Autowired private TaskRecordService taskRecordService; + @Autowired + private RemoteRecordService remoteRecordService; @Async(value = "SendBigDataTaskExecutor") - public void doRunTask(BuildRecord resource, SendBigDataDTO sendBigDataDTO){ + public void doRunTask(BuildRecord resource, RemoteRecord remoteRecord, SendBigDataDTO sendBigDataDTO){ Long satrtMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); log.info("====== [ task start running, task name is {} ] ======", "SendBigDataTask"); - runTask(resource, sendBigDataDTO); + runTask(resource, remoteRecord, sendBigDataDTO); Long endMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); log.info("====== [ task start end, task name is {},cost milliSecond is {} ] ======", "SendBigDataTask", ConvertUtil.secondToTime(endMilliSecond - satrtMilliSecond)); } - private void runTask(BuildRecord resource, SendBigDataDTO sendBigDataDTO) { + private void runTask(BuildRecord resource, RemoteRecord remoteRecord, SendBigDataDTO sendBigDataDTO) { // 根据发送任务的Id来读取发送号码表 Integer id = resource.getId(); @@ -128,7 +133,7 @@ public class SendBigDataTask { remoteCollect = getNotDuplicateRemoteContent(remoteCollect, resultFilePath); // 批量发送信息并且更新记录 AtomicLong atomicLong = batchSendRemote(remoteCollect, sendBigDataDTO, buildRecordDto); - finishSendThenUpdateRec(resource, buildRecordDto, atomicLong); + finishSendThenUpdateRemoteRec(remoteRecord, atomicLong); } // 乱序 (乱序的逻辑写在SQL语句中) @@ -164,6 +169,18 @@ public class SendBigDataTask { } } + private void finishSendThenUpdateRemoteRec(RemoteRecord remoteRecord, AtomicLong atomicLong) { + // 对发送后的状态进行更新 + RemoteRecord remote = new RemoteRecord(); + BeanUtil.copyProperties(remoteRecord, remote); + remote.setTag(SUCCESS_TAG); + remote.setWeight(10); + remote.setSuccessSendCount(atomicLong.get()); + remoteRecordService.update(remote); + } + + + private List getNotDuplicateRemoteContent(List remoteCollect, String resultFilePath) { if (StringUtils.isNotBlank(resultFilePath)){ Set fileLines = new HashSet<>(50_0000); diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/RemoteRecConst.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/RemoteRecConst.java index c14fd79..1181ddd 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/RemoteRecConst.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/RemoteRecConst.java @@ -34,4 +34,9 @@ public class RemoteRecConst { * 成功建立任务标识 */ public static final Integer SUCCESS_BUILD_TAG = 1; + + /** + * 临时文件后缀 + */ + public static String TEMP_FILE_END_STR = "bak"; } diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java index 535b6c0..728bbfc 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java @@ -6,7 +6,7 @@ package me.zhengjie.modules.remoterec.consts; public class SysConst { // 远程服务器的相关配置 - public static final String REMOTE_TRANS_HOST = "47.110.11.213"; + public static final String REMOTE_TRANS_HOST = "118.178.137.129"; public static final Integer REMOTE_TRANS_PORT = 22; @@ -14,6 +14,11 @@ public class SysConst { public static final String REMOTE_TRANS_SSH_PW = "yuyou@ECS2020"; + /** + * 远程SFTP保存文件的路径 + */ + public static final String REMOTE_TRANS_DIR_PATH = "/home/"; + // 测试内容临时定义 public static final String TEST_USER_NAME = "测试用户"; } diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/domain/RemoteRecord.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/domain/RemoteRecord.java index 566f836..8859f60 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/domain/RemoteRecord.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/domain/RemoteRecord.java @@ -61,7 +61,7 @@ public class RemoteRecord implements Serializable { @ApiModelProperty(value = "文件上传保存路径") private String localSavePath; - @Column(name = "weight",nullable = false) + @Column(name = "weight") @ApiModelProperty(value = "权重值") private Integer weight; @@ -73,6 +73,10 @@ public class RemoteRecord implements Serializable { @ApiModelProperty(value = "SFTP传输的文件名称") private String sftpFilePath; + @Column(name = "success_send_count") + @ApiModelProperty(value = "成功上传数量") + private Long successSendCount; + public void copy(RemoteRecord source){ BeanUtil.copyProperties(source,this, CopyOptions.create().setIgnoreNullValue(true)); } diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java index e8cb33e..085c6f5 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java @@ -95,7 +95,7 @@ public class RemoteRecordController { // 调用异步任务进行sftp文件下载 downloadSFTPFileTask.doRunTask(resources); // 更新相关的记录 - return new ResponseEntity<>(remoteRecordService.create(resources),HttpStatus.CREATED); + return new ResponseEntity<>(CommonResponse.createBySuccess(),HttpStatus.OK); } @PutMapping diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/service/dto/RemoteRecordDto.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/service/dto/RemoteRecordDto.java index b1c4473..8cc07c0 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/service/dto/RemoteRecordDto.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/service/dto/RemoteRecordDto.java @@ -51,4 +51,7 @@ public class RemoteRecordDto implements Serializable { /** 元洪城呢个上传任务的名称*/ private String uploadRemoteTaskName; + + /* 成功上穿的數量 **/ + private Long successSendCount; } \ No newline at end of file diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java index f51b08a..05cd757 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java @@ -1,4 +1,5 @@ package me.zhengjie.modules.remoterec.task; +import java.sql.Timestamp; import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.util.RandomUtil; @@ -41,8 +42,6 @@ public class DownloadSFTPFileTask { @Autowired private RemoteRecordService remoteRecordService; - @Autowired - private BuildRecordService buildRecordService; /** * 远程服务器地址 @@ -84,6 +83,7 @@ public class DownloadSFTPFileTask { RemoteRecordDto remoteRecordDto = remoteRecordService.create(resources); // 然后调用sftp进行取回 String sftpFilePath = resources.getSftpFilePath(); + if (StringUtils.isNotBlank(sftpFilePath)){ sftpToDownloadFile(sftpFilePath, resources); } @@ -91,24 +91,15 @@ public class DownloadSFTPFileTask { // 取回成功/失败,更新记录状态 RemoteRecord remoteRecord = new RemoteRecord(); BeanUtil.copyProperties(remoteRecordDto, remoteRecord); - resources.setTag(SUCCESS_TAG); - remoteRecordService.update(remoteRecord); - - // 更新待发送记录表相关的信息 - BuildRecord buildRecord = new BuildRecord(); + remoteRecord.setOperation(resources.getOperation()); + remoteRecord.setFileTransSuccessCount(resources.getFileTransSuccessCount()); + remoteRecord.setTag(0); + remoteRecord.setLocalSavePath(resources.getLocalSavePath()); + remoteRecord.setWeight(0); + remoteRecord.setUploadRemoteTaskName(resources.getUploadRemoteTaskName()); - buildRecord.setTaskName(resources.getUploadRemoteTaskName()); - buildRecord.setTimePeriod(7); // 默认保存时间为7天 - buildRecord.setIsBuild(SUCCESS_BUILD_TAG); - buildRecord.setOprationName(resources.getOperation()); - buildRecord.setLocalFilePath(resources.getLocalSavePath()); - buildRecord.setTotal(Long.valueOf(resources.getFileTransSuccessCount())); + remoteRecordService.update(remoteRecord); - // 先对任务表进行记录 - BuildRecordDto buildRecordDto = buildRecordService.create(buildRecord); - if (buildRecordDto == null) { - log.error("============ [create build rec is fail, please check it. ] ============"); - } // fixme 更新失败补充推送费用 } @@ -127,9 +118,11 @@ public class DownloadSFTPFileTask { } String localSavePath = buildFileWritePath(fullFileName); resources.setLocalSavePath(localSavePath); + log.info(" ====== [DownloadSFTPFileTask|sftpToDownloadFiles , ftpFilePath is {} ] ======", sftpFilePath); // SFTP 进行下载 sftp.download(sftpFilePath, FileUtil.file(localSavePath)); sftp.close(); + } /** diff --git a/eladmin-system/src/main/resources/config/application-dev.yml b/eladmin-system/src/main/resources/config/application-dev.yml index cd96f5c..3eae4a3 100644 --- a/eladmin-system/src/main/resources/config/application-dev.yml +++ b/eladmin-system/src/main/resources/config/application-dev.yml @@ -231,7 +231,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8000/api/temp/file/download' + address: 'http://localhost:8001/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-prod.yml b/eladmin-system/src/main/resources/config/application-prod.yml index c09f415..de01d4f 100644 --- a/eladmin-system/src/main/resources/config/application-prod.yml +++ b/eladmin-system/src/main/resources/config/application-prod.yml @@ -6,9 +6,9 @@ spring: eladmin: type: com.alibaba.druid.pool.DruidDataSource driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy - url: jdbc:log4jdbc:mysql://47.99.218.9:3306/eladmin?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false + url: jdbc:log4jdbc:mysql://118.178.137.129:3306/eladmin?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false username: root - password: Yuyou@2020 + password: root # 初始连接数 initial-size: 5 # 最小连接数 @@ -189,7 +189,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8000/api/temp/file/download' + address: 'http://118.178.137.129:8001/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-single.yml b/eladmin-system/src/main/resources/config/application-single.yml index dfb10f9..f22275e 100644 --- a/eladmin-system/src/main/resources/config/application-single.yml +++ b/eladmin-system/src/main/resources/config/application-single.yml @@ -169,7 +169,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8000/api/temp/file/download' + address: 'http://116.62.197.152:8001/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-stg.yml b/eladmin-system/src/main/resources/config/application-stg.yml index 9732ad3..b38ed79 100644 --- a/eladmin-system/src/main/resources/config/application-stg.yml +++ b/eladmin-system/src/main/resources/config/application-stg.yml @@ -231,7 +231,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8000/api/temp/file/download' + address: 'http://116.62.197.152:8001/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-test.yml b/eladmin-system/src/main/resources/config/application-test.yml index ad33ab4..f24327a 100644 --- a/eladmin-system/src/main/resources/config/application-test.yml +++ b/eladmin-system/src/main/resources/config/application-test.yml @@ -224,7 +224,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8000/api/temp/file/download' + address: 'http://116.62.197.152:8001/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application.yml b/eladmin-system/src/main/resources/config/application.yml index 2817347..2606f39 100644 --- a/eladmin-system/src/main/resources/config/application.yml +++ b/eladmin-system/src/main/resources/config/application.yml @@ -5,7 +5,7 @@ spring: freemarker: check-template-location: false profiles: - active: dev + active: prod jackson: time-zone: GMT+8 data: diff --git a/eladmin-system/src/main/resources/logback.xml b/eladmin-system/src/main/resources/logback.xml index 30ccc85..dcf1306 100644 --- a/eladmin-system/src/main/resources/logback.xml +++ b/eladmin-system/src/main/resources/logback.xml @@ -77,7 +77,7 @@ - - + + \ No newline at end of file diff --git a/eladmin-system/src/test/java/me/zhengjie/SpringJPATest.java b/eladmin-system/src/test/java/me/zhengjie/SpringJPATest.java index 78d0f4c..f74561e 100644 --- a/eladmin-system/src/test/java/me/zhengjie/SpringJPATest.java +++ b/eladmin-system/src/test/java/me/zhengjie/SpringJPATest.java @@ -1,5 +1,6 @@ package me.zhengjie; +import cn.hutool.core.bean.BeanUtil; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import me.zhengjie.modules.abmessage.domain.AbMessage; @@ -8,6 +9,8 @@ import me.zhengjie.modules.abmessage.service.dto.AbMessageQueryCriteria; import me.zhengjie.modules.edu.domain.Edu; import me.zhengjie.modules.edu.repository.EduRepository; import me.zhengjie.modules.mnt.service.dto.AppQueryCriteria; +import me.zhengjie.modules.remoterec.domain.RemoteRecord; +import me.zhengjie.modules.remoterec.service.RemoteRecordService; import me.zhengjie.modules.student.domain.Student; import me.zhengjie.modules.student.repository.StudentRepository; import me.zhengjie.modules.student.service.StudentService; @@ -31,6 +34,7 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -60,6 +64,9 @@ public class SpringJPATest { @Autowired private TagService tagService; + @Autowired + private RemoteRecordService remoteRecordService; + @Test public void testQueryBySlice() throws Exception{ @@ -308,4 +315,19 @@ public class SpringJPATest { System.out.println(tagService.countSendSum(taskId, 1)); } + + @Test + public void testUpdateRemoteRec(){ + RemoteRecord remoteRecord = new RemoteRecord(); + remoteRecord.setId(12); + remoteRecord.setUploadTime(new Timestamp(new java.util.Date().getTime())); + remoteRecord.setOperation("admin1"); + remoteRecord.setFileTransSuccessCount(1111); + remoteRecord.setTag(0); + remoteRecord.setLocalSavePath("2321312"); + remoteRecord.setWeight(0); + remoteRecord.setUploadRemoteTaskName("1111"); + + remoteRecordService.update(remoteRecord); + } }