BY-Team
/
plat1-eladmin
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

上传平台修复bug最终版

Browse Source
master
土豆兄弟 4 years ago
parent cecfb72eab
commit 58bb8e2315
17 changed files with 91 additions and 36 deletions
Show Stats Download Patch File Download Diff File

2
eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/BuildRecordController.java
Unescape Escape View File

@ -164,7 +164,7 @@ public class BuildRecordController {
SendBigDataDTO sendBigDataDTO = new SendBigDataDTO();
BeanUtil.copyProperties(buildRecordSendVO, sendBigDataDTO);
// 调用发送课包任务,开始发送
sendBigDataTask.doRunTask(buildRecordSendVO.getResource(), sendBigDataDTO);
sendBigDataTask.doRunTask(buildRecordSendVO.getResource(), buildRecordSendVO.getRemoteRecord() , sendBigDataDTO);
return new ResponseEntity<>(CommonResponse.createBySuccess(), HttpStatus.OK);
}

6
eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/rest/vo/BuildRecordSendVO.java
Unescape Escape View File

@ -5,6 +5,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import me.zhengjie.modules.buildrecord.domain.BuildRecord;
import me.zhengjie.modules.remoterec.domain.RemoteRecord;
import java.io.Serializable;
@ -29,6 +30,11 @@ public class BuildRecordSendVO implements Serializable {
*/
private BuildRecord resource;
/**
* 远程上传的实体
*/
private RemoteRecord remoteRecord;
/**
* 需要发送的活动名称
*/

25
eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java
Unescape Escape View File

@ -12,6 +12,8 @@ import me.zhengjie.modules.buildrecord.domain.BuildRecord;
import me.zhengjie.modules.buildrecord.service.BuildRecordService;
import me.zhengjie.modules.buildrecord.service.dto.BuildRecordDto;
import me.zhengjie.modules.buildrecord.task.dto.SendBigDataDTO;
import me.zhengjie.modules.remoterec.domain.RemoteRecord;
import me.zhengjie.modules.remoterec.service.RemoteRecordService;
import me.zhengjie.modules.tag.domain.Tag;
import me.zhengjie.modules.tag.service.TagService;
import me.zhengjie.modules.tag.service.dto.TagDto;
@ -43,6 +45,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import static me.zhengjie.modules.common.consts.SendBigDataConst.NON_FINISH_SEND_STATATUS;
import static me.zhengjie.modules.remoterec.consts.RemoteRecConst.SUCCESS_TAG;
@Component
@Scope("prototype")
@ -84,17 +87,19 @@ public class SendBigDataTask {
private TagService tagService;
@Autowired
private TaskRecordService taskRecordService;
@Autowired
private RemoteRecordService remoteRecordService;
@Async(value = "SendBigDataTaskExecutor")
public void doRunTask(BuildRecord resource, SendBigDataDTO sendBigDataDTO){
public void doRunTask(BuildRecord resource, RemoteRecord remoteRecord, SendBigDataDTO sendBigDataDTO){
Long satrtMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
log.info("====== [ task start running, task name is {} ] ======", "SendBigDataTask");
runTask(resource, sendBigDataDTO);
runTask(resource, remoteRecord, sendBigDataDTO);
Long endMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
log.info("====== [ task start end, task name is {},cost milliSecond is {} ] ======", "SendBigDataTask", ConvertUtil.secondToTime(endMilliSecond - satrtMilliSecond));
}
private void runTask(BuildRecord resource, SendBigDataDTO sendBigDataDTO) {
private void runTask(BuildRecord resource, RemoteRecord remoteRecord, SendBigDataDTO sendBigDataDTO) {
// 根据发送任务的Id来读取发送号码表
Integer id = resource.getId();
@ -128,7 +133,7 @@ public class SendBigDataTask {
remoteCollect = getNotDuplicateRemoteContent(remoteCollect, resultFilePath);
// 批量发送信息并且更新记录
AtomicLong atomicLong = batchSendRemote(remoteCollect, sendBigDataDTO, buildRecordDto);
finishSendThenUpdateRec(resource, buildRecordDto, atomicLong);
finishSendThenUpdateRemoteRec(remoteRecord, atomicLong);
}
// 乱序 （乱序的逻辑写在SQL语句中）
@ -164,6 +169,18 @@ public class SendBigDataTask {
}
}
private void finishSendThenUpdateRemoteRec(RemoteRecord remoteRecord, AtomicLong atomicLong) {
// 对发送后的状态进行更新
RemoteRecord remote = new RemoteRecord();
BeanUtil.copyProperties(remoteRecord, remote);
remote.setTag(SUCCESS_TAG);
remote.setWeight(10);
remote.setSuccessSendCount(atomicLong.get());
remoteRecordService.update(remote);
}
private List<String> getNotDuplicateRemoteContent(List<String> remoteCollect, String resultFilePath) {
if (StringUtils.isNotBlank(resultFilePath)){
Set<String> fileLines = new HashSet<>(50_0000);

5
eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/RemoteRecConst.java
Unescape Escape View File

@ -34,4 +34,9 @@ public class RemoteRecConst {
* 成功建立任务标识
*/
public static final Integer SUCCESS_BUILD_TAG = 1;
/**
* 临时文件后缀
*/
public static String TEMP_FILE_END_STR = "bak";
}

7
eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java
Unescape Escape View File

@ -6,7 +6,7 @@ package me.zhengjie.modules.remoterec.consts;
public class SysConst {
// 远程服务器的相关配置
public static final String REMOTE_TRANS_HOST = "47.110.11.213";
public static final String REMOTE_TRANS_HOST = "118.178.137.129";
public static final Integer REMOTE_TRANS_PORT = 22;
@ -14,6 +14,11 @@ public class SysConst {
public static final String REMOTE_TRANS_SSH_PW = "yuyou@ECS2020";
/**
* 远程SFTP保存文件的路径
*/
public static final String REMOTE_TRANS_DIR_PATH = "/home/";
// 测试内容临时定义
public static final String TEST_USER_NAME = "测试用户";
}

6
eladmin-system/src/main/java/me/zhengjie/modules/remoterec/domain/RemoteRecord.java
Unescape Escape View File

@ -61,7 +61,7 @@ public class RemoteRecord implements Serializable {
@ApiModelProperty(value = "文件上传保存路径")
private String localSavePath;
@Column(name = "weight",nullable = false)
@Column(name = "weight")
@ApiModelProperty(value = "权重值")
private Integer weight;
@ -73,6 +73,10 @@ public class RemoteRecord implements Serializable {
@ApiModelProperty(value = "SFTP传输的文件名称")
private String sftpFilePath;
@Column(name = "success_send_count")
@ApiModelProperty(value = "成功上传数量")
private Long successSendCount;
public void copy(RemoteRecord source){
BeanUtil.copyProperties(source,this, CopyOptions.create().setIgnoreNullValue(true));
}

2
eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java
Unescape Escape View File

@ -95,7 +95,7 @@ public class RemoteRecordController {
// 调用异步任务进行sftp文件下载
downloadSFTPFileTask.doRunTask(resources);
// 更新相关的记录
return new ResponseEntity<>(remoteRecordService.create(resources),HttpStatus.CREATED);
return new ResponseEntity<>(CommonResponse.createBySuccess(),HttpStatus.OK);
}
@PutMapping

3
eladmin-system/src/main/java/me/zhengjie/modules/remoterec/service/dto/RemoteRecordDto.java
Unescape Escape View File

@ -51,4 +51,7 @@ public class RemoteRecordDto implements Serializable {
/** 元洪城呢个上传任务的名称*/
private String uploadRemoteTaskName;
/* 成功上穿的數量 **/
private Long successSendCount;
}

29
eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java
Unescape Escape View File

@ -1,4 +1,5 @@
package me.zhengjie.modules.remoterec.task;
import java.sql.Timestamp;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.util.RandomUtil;
@ -41,8 +42,6 @@ public class DownloadSFTPFileTask {
@Autowired
private RemoteRecordService remoteRecordService;
@Autowired
private BuildRecordService buildRecordService;
/**
* 远程服务器地址
@ -84,6 +83,7 @@ public class DownloadSFTPFileTask {
RemoteRecordDto remoteRecordDto = remoteRecordService.create(resources);
// 然后调用sftp进行取回
String sftpFilePath = resources.getSftpFilePath();
if (StringUtils.isNotBlank(sftpFilePath)){
sftpToDownloadFile(sftpFilePath, resources);
}
@ -91,24 +91,15 @@ public class DownloadSFTPFileTask {
// 取回成功/失败,更新记录状态
RemoteRecord remoteRecord = new RemoteRecord();
BeanUtil.copyProperties(remoteRecordDto, remoteRecord);
resources.setTag(SUCCESS_TAG);
remoteRecordService.update(remoteRecord);
// 更新待发送记录表相关的信息
BuildRecord buildRecord = new BuildRecord();
remoteRecord.setOperation(resources.getOperation());
remoteRecord.setFileTransSuccessCount(resources.getFileTransSuccessCount());
remoteRecord.setTag(0);
remoteRecord.setLocalSavePath(resources.getLocalSavePath());
remoteRecord.setWeight(0);
remoteRecord.setUploadRemoteTaskName(resources.getUploadRemoteTaskName());
buildRecord.setTaskName(resources.getUploadRemoteTaskName());
buildRecord.setTimePeriod(7); // 默认保存时间为7天
buildRecord.setIsBuild(SUCCESS_BUILD_TAG);
buildRecord.setOprationName(resources.getOperation());
buildRecord.setLocalFilePath(resources.getLocalSavePath());
buildRecord.setTotal(Long.valueOf(resources.getFileTransSuccessCount()));
remoteRecordService.update(remoteRecord);
// 先对任务表进行记录
BuildRecordDto buildRecordDto = buildRecordService.create(buildRecord);
if (buildRecordDto == null) {
log.error("============ [create build rec is fail, please check it. ] ============");
}
// fixme 更新失败补充推送费用
}
@ -127,9 +118,11 @@ public class DownloadSFTPFileTask {
}
String localSavePath = buildFileWritePath(fullFileName);
resources.setLocalSavePath(localSavePath);
log.info(" ====== [DownloadSFTPFileTask|sftpToDownloadFiles , ftpFilePath is {} ] ======", sftpFilePath);
// SFTP 进行下载
sftp.download(sftpFilePath, FileUtil.file(localSavePath));
sftp.close();
}
/**

2
eladmin-system/src/main/resources/config/application-dev.yml
Unescape Escape View File

@ -231,7 +231,7 @@ tag:
sum: 10
remote:
link:
address: 'http://116.62.197.152:8000/api/temp/file/download'
address: 'http://localhost:8001/api/temp/file/download'
file-base-path-linux: /home/eladmin/file/temp/
file-base-path-windows: C:\eladmin\file\temp\
file-base-path-mac: ~/file/eladmin/temp/

6
eladmin-system/src/main/resources/config/application-prod.yml
Unescape Escape View File

@ -6,9 +6,9 @@ spring:
eladmin:
type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
url: jdbc:log4jdbc:mysql://47.99.218.9:3306/eladmin?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false
url: jdbc:log4jdbc:mysql://118.178.137.129:3306/eladmin?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false
username: root
password: Yuyou@2020
password: root
# 初始连接数
initial-size: 5
# 最小连接数
@ -189,7 +189,7 @@ tag:
sum: 10
remote:
link:
address: 'http://116.62.197.152:8000/api/temp/file/download'
address: 'http://118.178.137.129:8001/api/temp/file/download'
file-base-path-linux: /home/eladmin/file/temp/
file-base-path-windows: C:\eladmin\file\temp\
file-base-path-mac: ~/file/eladmin/temp/

2
eladmin-system/src/main/resources/config/application-single.yml
Unescape Escape View File

@ -169,7 +169,7 @@ tag:
sum: 10
remote:
link:
address: 'http://116.62.197.152:8000/api/temp/file/download'
address: 'http://116.62.197.152:8001/api/temp/file/download'
file-base-path-linux: /home/eladmin/file/temp/
file-base-path-windows: C:\eladmin\file\temp\
file-base-path-mac: ~/file/eladmin/temp/

2
eladmin-system/src/main/resources/config/application-stg.yml
Unescape Escape View File

@ -231,7 +231,7 @@ tag:
sum: 10
remote:
link:
address: 'http://116.62.197.152:8000/api/temp/file/download'
address: 'http://116.62.197.152:8001/api/temp/file/download'
file-base-path-linux: /home/eladmin/file/temp/
file-base-path-windows: C:\eladmin\file\temp\
file-base-path-mac: ~/file/eladmin/temp/

2
eladmin-system/src/main/resources/config/application-test.yml
Unescape Escape View File

@ -224,7 +224,7 @@ tag:
sum: 10
remote:
link:
address: 'http://116.62.197.152:8000/api/temp/file/download'
address: 'http://116.62.197.152:8001/api/temp/file/download'
file-base-path-linux: /home/eladmin/file/temp/
file-base-path-windows: C:\eladmin\file\temp\
file-base-path-mac: ~/file/eladmin/temp/

2
eladmin-system/src/main/resources/config/application.yml
Unescape Escape View File

@ -5,7 +5,7 @@ spring:
freemarker:
check-template-location: false
profiles:
active: dev
active: prod
jackson:
time-zone: GMT+8
data:

4
eladmin-system/src/main/resources/logback.xml
Unescape Escape View File

@ -77,7 +77,7 @@
<root level="INFO">
<!-- TODO prod 环境去掉std -->
<appender-ref ref="stdAppender"/>
<!--<appender-ref ref="fileAppender"/>-->
<!--<appender-ref ref="stdAppender"/>-->
<appender-ref ref="fileAppender"/>
</root>
</configuration>

22
eladmin-system/src/test/java/me/zhengjie/SpringJPATest.java
Unescape Escape View File

@ -1,5 +1,6 @@
package me.zhengjie;
import cn.hutool.core.bean.BeanUtil;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import me.zhengjie.modules.abmessage.domain.AbMessage;
@ -8,6 +9,8 @@ import me.zhengjie.modules.abmessage.service.dto.AbMessageQueryCriteria;
import me.zhengjie.modules.edu.domain.Edu;
import me.zhengjie.modules.edu.repository.EduRepository;
import me.zhengjie.modules.mnt.service.dto.AppQueryCriteria;
import me.zhengjie.modules.remoterec.domain.RemoteRecord;
import me.zhengjie.modules.remoterec.service.RemoteRecordService;
import me.zhengjie.modules.student.domain.Student;
import me.zhengjie.modules.student.repository.StudentRepository;
import me.zhengjie.modules.student.service.StudentService;
@ -31,6 +34,7 @@ import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -60,6 +64,9 @@ public class SpringJPATest {
@Autowired
private TagService tagService;
@Autowired
private RemoteRecordService remoteRecordService;
@Test
public void testQueryBySlice() throws Exception{
@ -308,4 +315,19 @@ public class SpringJPATest {
System.out.println(tagService.countSendSum(taskId, 1));
}
@Test
public void testUpdateRemoteRec(){
RemoteRecord remoteRecord = new RemoteRecord();
remoteRecord.setId(12);
remoteRecord.setUploadTime(new Timestamp(new java.util.Date().getTime()));
remoteRecord.setOperation("admin1");
remoteRecord.setFileTransSuccessCount(1111);
remoteRecord.setTag(0);
remoteRecord.setLocalSavePath("2321312");
remoteRecord.setWeight(0);
remoteRecord.setUploadRemoteTaskName("1111");
remoteRecordService.update(remoteRecord);
}
}

Write Preview
Loading…
Cancel
Save