BY-Team
/
plat2-eladmin
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

文件传输最终版

Browse Source
master
土豆兄弟 4 years ago
parent ea5f31acdb
commit e3906d1a72
5 changed files with 132 additions and 33 deletions
Show Stats Download Patch File Download Diff File

11
eladmin-system/src/main/java/me/zhengjie/modules/upload/consts/SysConst.java
Unescape Escape View File

@ -16,7 +16,7 @@ public class SysConst {
// Test相关的 - end // Test相关的 - end
public static final String REMOTE_TRANS_HOST = "47.110.11.213"; public static final String REMOTE_TRANS_HOST = "118.178.137.129";
public static final Integer REMOTE_TRANS_PORT = 22; public static final Integer REMOTE_TRANS_PORT = 22;
@ -30,13 +30,18 @@ public class SysConst {
public static final String REMOTE_TRANS_DIR_PATH = "/home/"; public static final String REMOTE_TRANS_DIR_PATH = "/home/";
// fixme 以后改成rpc调用的地址 // fixme 以后改成rpc调用的地址
public static final String REMOTE_UPDATE_ADDR = "http://116.62.197.152:8000/api/remoteRecord/remote/rec"; public static final String REMOTE_UPDATE_ADDR = "http://118.178.137.129:8001/api/remoteRecord/remote/rec";
// 测试内容临时定义 // 测试内容临时定义
public static final String TEST_USER_NAME = "测试用户"; public static final String TEST_USER_NAME = "测试用户";
// 系统开发,上线环境下进行关闭 - DEBUG开关 - 默认是打开的 // 系统开发,上线环境下进行关闭 - DEBUG开关 - 默认是打开的
public static Boolean sysDebug = true; public static Boolean sysDebug = false;
/**
* 临时文件后缀
*/
public static String TEMP_FILE_END_STR = "bak";
/** /**
* 使用Set方式注入 * 使用Set方式注入

64
eladmin-system/src/main/java/me/zhengjie/modules/upload/task/SaveToFileTask.java
Unescape Escape View File

@ -72,18 +72,19 @@ public class SaveToFileTask {
@Value(value = "${inter.address}") @Value(value = "${inter.address}")
private String encryptAddress; private String encryptAddress;
@Value(value = "inter.appid") @Value(value = "${inter.appid}")
private String encryptAppId; private String encryptAppId;
@Value(value = "inter.tk") @Value(value = "${inter.tk}")
private String encryptTK; private String encryptTK;
private Boolean booleanTag = true;
@Autowired @Autowired
private UploadFileService uploadFileService; private UploadFileService uploadFileService;
@Async(value = "SendBigDataTaskExecutor") @Async(value = "SendBigDataTaskExecutor")
public void doRunTask(UploadFileDto uploadFileDto) { public void doRunTask(UploadFileDto uploadFileDto) {
Long satrtMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); Long satrtMilliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
@ -119,14 +120,15 @@ public class SaveToFileTask {
// 更新状态为成功,更新解析成功的条数 // 更新状态为成功,更新解析成功的条数
UploadFile uploadFile = new UploadFile(); UploadFile uploadFile = new UploadFile();
if (finalTag){ if (finalTag) {
BeanUtils.copyProperties(uploadFileDto, uploadFile); BeanUtils.copyProperties(uploadFileDto, uploadFile);
uploadFile.setUploadTag(SUCCESS_TAG); uploadFile.setUploadTag(SUCCESS_TAG);
uploadFileService.update(uploadFile); uploadFileService.update(uploadFile);
}else { } else {
// 失败进行容错 // 失败进行容错
BeanUtils.copyProperties(uploadFileDto, uploadFile); BeanUtils.copyProperties(uploadFileDto, uploadFile);
uploadFile.setUploadTag(FAIL_TAG); uploadFile.setUploadTag(FAIL_TAG);
uploadFile.setFileTransSuccessCount(uploadFileDto.getFileCount());
uploadFileService.update(uploadFile); uploadFileService.update(uploadFile);
} }
} }
@ -140,19 +142,23 @@ public class SaveToFileTask {
List<String> fileAllLinesList = FileUtil.readLines(filePath, "utf-8"); List<String> fileAllLinesList = FileUtil.readLines(filePath, "utf-8");
if (CollectionUtil.isNotEmpty(fileAllLinesList)) { if (CollectionUtil.isNotEmpty(fileAllLinesList)) {
// 分批调用接口进行加密 // 分批调用接口进行加密
batchSendToEncrypt(fileAllLinesList); batchSendToEncrypt(filePath, fileAllLinesList);
}
// 加入一个全局控制开关
if (!booleanTag){
return Boolean.FALSE;
} }
// 把临时存储的文件进行删除 // 把临时存储的文件进行删除
boolean delFileTag = delTempSaveFile(filePath); boolean delFileTag = delTempSaveFile(filePath);
// 文件传输给2号库服务器 - 走sftp协议 - 支持断点续传 // 文件传输给2号库服务器 - 走sftp协议 - 支持断点续传
transFileToOtherServer(filePath); transFileToOtherServer(filePath + TEMP_FILE_END_STR);
// 调用远程接口完成一条记录更新 // 调用远程接口完成一条记录更新
boolean sendUpdatePostReqTag = sendUpdatePostReq(filePath, uploadFileDto); boolean sendUpdatePostReqTag = sendUpdatePostReq(filePath+ TEMP_FILE_END_STR, uploadFileDto);
// fixme 这里要修改之前的平台给一个更新接口,然后这边可以用rpc调用,也可以用http,也可以考虑直接消息中间件进行解耦 // fixme 这里要修改之前的平台给一个更新接口,然后这边可以用rpc调用,也可以用http,也可以考虑直接消息中间件进行解耦
if (delFileTag && sendUpdatePostReqTag){ if (delFileTag && sendUpdatePostReqTag) {
return Boolean.TRUE; return Boolean.TRUE;
} }
return Boolean.FALSE; return Boolean.FALSE;
@ -165,6 +171,8 @@ public class SaveToFileTask {
Sftp sftp = JschUtil.createSftp(session); Sftp sftp = JschUtil.createSftp(session);
//fixme sftp.put("C:\\Users\\Administrator\\Desktop\\233.txt", "/home", Sftp.Mode.RESUME); //fixme sftp.put("C:\\Users\\Administrator\\Desktop\\233.txt", "/home", Sftp.Mode.RESUME);
log.info("========= [SaveToFileTask|transFileToOtherServer, trans address is {} ] =========", filePath);
sftp.put(filePath, REMOTE_TRANS_DIR_PATH, Sftp.Mode.RESUME); sftp.put(filePath, REMOTE_TRANS_DIR_PATH, Sftp.Mode.RESUME);
sftp.close(); sftp.close();
@ -172,16 +180,16 @@ public class SaveToFileTask {
private boolean delTempSaveFile(String tempFilesPath) { private boolean delTempSaveFile(String tempFilesPath) {
boolean del = FileUtil.del(tempFilesPath); boolean del = FileUtil.del(tempFilesPath);
if (del){ if (del) {
log.info("======== [success del file, file path is {} ] ========", tempFilesPath); log.info("======== [success del file, file path is {} ] ========", tempFilesPath);
return Boolean.TRUE; return Boolean.TRUE;
}else { } else {
log.error("======== [fail del file, file path is {} ] ========", tempFilesPath); log.error("======== [fail del file, file path is {} ] ========", tempFilesPath);
return Boolean.FALSE; return Boolean.FALSE;
} }
} }
private void batchSendToEncrypt(List<String> fileAllLinesList) { private void batchSendToEncrypt(String filePath, List<String> fileAllLinesList) {
List<List<String>> partition = Lists.partition(fileAllLinesList, SEND_ENCRYPT_LIMIT); List<List<String>> partition = Lists.partition(fileAllLinesList, SEND_ENCRYPT_LIMIT);
partition.forEach( partition.forEach(
@ -202,15 +210,18 @@ public class SaveToFileTask {
// 组装成JSON // 组装成JSON
String readSendJsonStr = JSON.toJSONString(sendEncryptJsonContent); String readSendJsonStr = JSON.toJSONString(sendEncryptJsonContent);
log.info("SaveToFileTask|batchSendToEncrypt ready send json is : {}", readSendJsonStr); log.info("SaveToFileTask|batchSendToEncrypt ready send json is : {}", readSendJsonStr);
int count = 1; int count = 0;
while (count <= 3) { while (count < 3) {
// 调用HTTP请求发送数据 // 调用HTTP请求发送数据
HttpResponse httpResponse = sendPostReq(readSendJsonStr); HttpResponse httpResponse = sendPostReq(readSendJsonStr);
if (httpResponse.isOk() && httpResponse.body().contains("success")) { if (httpResponse.isOk() && httpResponse.body().contains("success")) {
log.info("========== [SaveToFileTask|batchSendToEncrypt request success, response is {} ] ==========", httpResponse.body()); log.info("========== [SaveToFileTask|batchSendToEncrypt request success, response is {} ] ==========", httpResponse.body());
String responseStr = httpResponse.body(); String responseStr = httpResponse.body();
// 解析返回的结果,并写回本地 // 解析返回的结果,并写回本地
parseResponseStr(responseStr); if (!parseResponseStr(filePath, responseStr)) {
log.error("========== [ SaveToFileTask|batchSendToEncrypt parse fail ] ==========");
booleanTag = false;
}
break; break;
} else { } else {
count++; count++;
@ -223,7 +234,7 @@ public class SaveToFileTask {
log.error("========== [SaveToFileTask|batchSendToEncrypt request fail, response is {} ] ==========", httpResponse.body()); log.error("========== [SaveToFileTask|batchSendToEncrypt request fail, response is {} ] ==========", httpResponse.body());
} }
} }
if (count > 3) { if (count >= 3) {
log.error("========== [SaveToFileTask|batchSendToEncrypt update send status fail, url is {} ] ==========", encryptAddress); log.error("========== [SaveToFileTask|batchSendToEncrypt update send status fail, url is {} ] ==========", encryptAddress);
} }
@ -231,21 +242,22 @@ public class SaveToFileTask {
); );
} }
private void parseResponseStr(String responseStr) { private boolean parseResponseStr(String filePath, String responseStr) {
ResponseEncryptJsonContent responseContent = JSONArray.parseObject(responseStr, ResponseEncryptJsonContent.class); ResponseEncryptJsonContent responseContent = JSONArray.parseObject(responseStr, ResponseEncryptJsonContent.class);
// 处理需要的加密号串 // 处理需要的加密号串
String tels = responseContent.getTels(); String tels = responseContent.getTels();
String[] splitResTels = StringUtils.split(tels, SPLIT_TAG); String[] splitResTels = StringUtils.split(Base64.decodeStr(tels), SPLIT_TAG);
if (splitResTels != null && splitResTels.length > 0) { if (splitResTels != null && splitResTels.length > 0) {
List<String> stringList = Arrays.asList(splitResTels); List<String> stringList = Arrays.asList(splitResTels);
// TODO: 2021/1/5 0005 这里保存文件的路径进行定义下 // TODO: 2021/1/5 0005 这里保存文件的路径进行定义下
String path = ""; String responseFilePath = filePath + TEMP_FILE_END_STR;
// 写入指定路径 // 写入指定路径
writeToFile(stringList, path); writeToFile(stringList, responseFilePath);
return Boolean.TRUE;
} }
return Boolean.FALSE;
} }
private void writeToFile(List<String> collect, String fullPath) { private void writeToFile(List<String> collect, String fullPath) {
@ -291,7 +303,6 @@ public class SaveToFileTask {
* 调用HTTP请求发送更新记录的Post请求 * 调用HTTP请求发送更新记录的Post请求
* *
* @param path 单个存储在本地的路径地址 * @param path 单个存储在本地的路径地址
*
* @return 返回请求结果 * @return 返回请求结果
*/ */
private boolean sendUpdatePostReq(String path, UploadFileDto uploadFileDto) { private boolean sendUpdatePostReq(String path, UploadFileDto uploadFileDto) {
@ -300,7 +311,8 @@ public class SaveToFileTask {
BeanUtil.copyProperties(uploadFileDto, sendRemoteUpdateJsonContent); BeanUtil.copyProperties(uploadFileDto, sendRemoteUpdateJsonContent);
String fileFullName = FileUtil.getName(path); String fileFullName = FileUtil.getName(path);
if (StringUtils.isNotBlank(fileFullName)){ if (StringUtils.isNotBlank(fileFullName)) {
// FIXME: 2021/1/12 0012
sendRemoteUpdateJsonContent.setSftpFilePath(REMOTE_TRANS_DIR_PATH + fileFullName); sendRemoteUpdateJsonContent.setSftpFilePath(REMOTE_TRANS_DIR_PATH + fileFullName);
} }
sendRemoteUpdateJsonContent.setFileTransSuccessCount(uploadFileDto.getFileCount()); sendRemoteUpdateJsonContent.setFileTransSuccessCount(uploadFileDto.getFileCount());
@ -317,11 +329,11 @@ public class SaveToFileTask {
.execute(); .execute();
// 解析回送请求 // 解析回送请求
if (httpResponse.isOk() && httpResponse.body().contains("SUCCESS")){ if (httpResponse.isOk() && httpResponse.body().contains("SUCCESS")) {
log.info("====== [success send upload record request! ]======"); log.info("====== [success send upload record request! ]======");
return Boolean.TRUE; return Boolean.TRUE;
}else { } else {
log.error("====== [ SaveToFileTask|sendUpdatePostReq , send upload record request Fail! ]======"); log.error("====== [ SaveToFileTask|sendUpdatePostReq , send upload record request Fail! , reason is {} ]======", httpResponse.body());
} }
return Boolean.FALSE; return Boolean.FALSE;

4
eladmin-system/src/main/resources/config/application.yml
Unescape Escape View File

@ -5,7 +5,7 @@ spring:
freemarker: freemarker:
check-template-location: false check-template-location: false
profiles: profiles:
active: dev active: prod
jackson: jackson:
time-zone: GMT+8 time-zone: GMT+8
data: data:
@ -64,4 +64,4 @@ remote:
inter: inter:
address: https://bd.hzdaba.cn:8085/v3/Accounts/yuyoukeji/BigData/EncryptTel address: https://bd.hzdaba.cn:8085/v3/Accounts/yuyoukeji/BigData/EncryptTel
appid: yuyoukeji appid: yuyoukeji
tk: 60da2e99ddb4d56d5ace6ff379fb754a tk: e3fb7090b4babe44a5113601487af433

4
eladmin-system/src/main/resources/logback.xml
Unescape Escape View File

@ -77,7 +77,7 @@
<root level="INFO"> <root level="INFO">
<!-- TODO prod 环境去掉std --> <!-- TODO prod 环境去掉std -->
<appender-ref ref="stdAppender"/> <!--<appender-ref ref="stdAppender"/>-->
<!--<appender-ref ref="fileAppender"/>--> <appender-ref ref="fileAppender"/>
</root> </root>
</configuration> </configuration>

82
eladmin-system/src/test/java/me/zhengjie/TestEncryptInter.java
Unescape Escape View File

@ -0,0 +1,82 @@
package me.zhengjie;
import cn.hutool.core.codec.Base64;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import me.zhengjie.modules.upload.task.model.SendEncryptJsonContent;
import me.zhengjie.utils.DateUtil;
import me.zhengjie.utils.FileUtil;
import me.zhengjie.utils.StringUtils;
import org.junit.Test;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.util.List;
@Slf4j
public class TestEncryptInter {
/**
* 时间格式
*/
private final static String FORMATE_TIMESTAMP = "yyyyMMddHHmmss";
private String address = "https://bd.hzdaba.cn:8085/v3/Accounts/yuyoukeji/BigData/EncryptTel";
private String appid = "yuyoukeji";
private String tk = "e3fb7090b4babe44a5113601487af433";
@Test
public void testEncryptInter(){
List<String> stringList = FileUtil.readLines("C:\\Users\\Administrator\\Desktop\\9wf21.txt", Charset.defaultCharset());
SendEncryptJsonContent sendEncryptJsonContent = new SendEncryptJsonContent();
//fixme 还有一个过期时间参数为选填参数,暂时不做设置
String tels = StringUtils.listPrintWithSpecialSplit(stringList, null);
sendEncryptJsonContent.setTels(Base64.encode(tels));
sendEncryptJsonContent.setReqId(RandomUtil.randomString(10));
sendEncryptJsonContent.setAppId(appid);
// 配置 sign
sendEncryptJsonContent.setSig(makeSign());
// 组装成JSON
String readSendJsonStr = JSON.toJSONString(sendEncryptJsonContent);
System.out.println(readSendJsonStr);
// 调用HTTP请求发送数据
HttpResponse httpResponse = sendPostReq(readSendJsonStr);
System.out.println(httpResponse.body());
}
private String makeSign() {
String signBuilder = appid +
tk +
DateUtil.localDateTimeFormat(LocalDateTime.now(), FORMATE_TIMESTAMP);
System.out.println(signBuilder);
return SecureUtil.md5(signBuilder).toUpperCase();
}
private HttpResponse sendPostReq(String json) {
HttpResponse execute = HttpRequest
.post(address)
.header("Content-Type", "application/json;charset=utf-8")
.body(json)
.execute();
return execute;
}
}
Write Preview
Loading…
Cancel
Save