From a6f8f023144a243650f7ac283c7717dcdefc3ae3 Mon Sep 17 00:00:00 2001 From: qyx <565485304@qq.com> Date: Mon, 18 Jan 2021 17:07:32 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=B9=B3=E5=8F=B0=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dbug=E6=9C=80=E7=BB=88=E7=89=884?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../buildrecord/task/SendBigDataTask.java | 28 ++++++++-- .../modules/remoterec/consts/SysConst.java | 2 +- .../rest/RemoteRecordController.java | 1 + .../remoterec/task/DownloadSFTPFileTask.java | 23 ++++---- .../main/resources/config/application-dev.yml | 2 +- .../resources/config/application-prod.yml | 2 +- .../resources/config/application-single.yml | 2 +- .../main/resources/config/application-stg.yml | 2 +- .../resources/config/application-test.yml | 13 ++++- .../src/main/resources/config/application.yml | 2 +- .../src/test/java/me/zhengjie/StreamTest.java | 55 ++++++++++++++----- 11 files changed, 95 insertions(+), 37 deletions(-) diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java index 78955d0..98ad2c1 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java @@ -6,6 +6,7 @@ import cn.hutool.core.util.RandomUtil; import cn.hutool.http.HttpResponse; import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; import me.zhengjie.common.json.PushDBJsonContent; import me.zhengjie.modules.buildrecord.domain.BuildRecord; @@ -199,12 +200,29 @@ public class SendBigDataTask { } catch (IOException e) { log.error("================== {read file error , please check is , file path is : {} } ================================", resultFilePath, e); } - // 过滤的集合 - Set filterCollect = remoteCollect.stream().parallel().filter(fileLines::contains).collect(Collectors.toSet()); - if (CollectionUtil.isNotEmpty(filterCollect)){ - log.info("================== [Filter collect is ready, collect size is {} ] ================================",filterCollect.size()); - remoteCollect = remoteCollect.stream().filter(one -> !filterCollect.contains(one)).collect(Collectors.toList()); + // 根据集合的大小不同采取不提,不同的方案 + if (remoteCollect.size() > fileLines.size()){ + // 重复的set + Set filterCollect = remoteCollect.stream().parallel().filter(fileLines::contains).collect(Collectors.toSet()); + if (CollectionUtil.isNotEmpty(filterCollect)){ + log.info("================== [Filter collect is ready, collect size is {} ] ================================",filterCollect.size()); + // 把重复的进行去除 + remoteCollect = remoteCollect.stream().filter(one -> !filterCollect.contains(one)).collect(Collectors.toList()); + } } + + if (remoteCollect.size() <= fileLines.size()){ + List fileLinesList = Lists.newArrayList(fileLines); + Set remoteCollectSet = Sets.newHashSet(remoteCollect); + // 重复的set + Set filterCollect = fileLinesList.stream().parallel().filter(remoteCollectSet::contains).collect(Collectors.toSet()); + if (CollectionUtil.isNotEmpty(filterCollect)){ + // 把重复的进行去除 + log.info("================== [Filter collect is ready, collect size is {} ] ================================",filterCollect.size()); + remoteCollect = remoteCollect.stream().filter(one -> !filterCollect.contains(one)).collect(Collectors.toList()); + } + } + } return remoteCollect; } diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java index e4af824..df116fa 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java @@ -6,7 +6,7 @@ package me.zhengjie.modules.remoterec.consts; public class SysConst { // 远程服务器的相关配置 - public static final String REMOTE_TRANS_HOST = "116.62.197.152"; + public static final String REMOTE_TRANS_HOST = "116.62.197.152"; // 118.178.137.129 116.62.197.152 public static final Integer REMOTE_TRANS_PORT = 22; diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java index 085c6f5..6b2ebe2 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java @@ -64,6 +64,7 @@ public class RemoteRecordController { @Log("查询远程上传记录") @ApiOperation("查询远程上传记录") @PreAuthorize("@el.check('remoteRecord:list')") + @AnonymousAccess public ResponseEntity query(RemoteRecordQueryCriteria criteria, Pageable pageable){ return new ResponseEntity<>(remoteRecordService.queryAll(criteria,pageable),HttpStatus.OK); } diff --git a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java index 5dbd1ef..0197321 100644 --- a/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java +++ b/eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java @@ -7,6 +7,7 @@ import java.sql.Timestamp; import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.RandomUtil; +import cn.hutool.extra.ftp.Ftp; import cn.hutool.extra.ssh.JschUtil; import cn.hutool.extra.ssh.Sftp; import cn.hutool.system.OsInfo; @@ -84,6 +85,7 @@ public class DownloadSFTPFileTask { } private void runTask(RemoteRecord resources) { + resources.setOperation(resources.getOperation() + FILE_NAME_SPLIT + resources.getUploadRemoteTaskName()); // 更新一条记录,然后状态为待取回状态 RemoteRecordDto remoteRecordDto = remoteRecordService.create(resources); // 然后调用sftp进行取回 @@ -116,13 +118,13 @@ public class DownloadSFTPFileTask { remoteRecordService.update(remoteRecord); +// sftp.close(); // fixme 更新失败补充推送费用 } private String sftpToDownloadFile(String sftpFilePath, RemoteRecord resources) { - // 准备sftp配置 - Session session = JschUtil.getSession(REMOTE_TRANS_HOST, REMOTE_TRANS_PORT, REMOTE_TRANS_SSH_USER, REMOTE_TRANS_SSH_PW); + Session session = JschUtil.createSession(REMOTE_TRANS_HOST, REMOTE_TRANS_PORT, REMOTE_TRANS_SSH_USER, REMOTE_TRANS_SSH_PW); Sftp sftp = JschUtil.createSftp(session); // 构建输出 @@ -136,11 +138,11 @@ public class DownloadSFTPFileTask { resources.setLocalSavePath(localSavePath); log.info(" ====== [DownloadSFTPFileTask|sftpToDownloadFiles , ftpFilePath is {} ] ======", sftpFilePath); // SFTP 进行下载 - sftp.download(sftpFilePath, FileUtil.file(localSavePath)); - sftp.close(); + sftp.get(sftpFilePath, localSavePath); + sftp.close(); +// sftp.download(sftpFilePath, FileUtil.file(localSavePath)); return localSavePath; - } /** @@ -155,23 +157,24 @@ public class DownloadSFTPFileTask { String dirPath; // 生成一个随机文件夹目录,方便整理和打包 - String filePath = RandomUtil.randomString(6) + FILE_NAME_SPLIT + baseStr; - if (osInfo.isWindows()) { dirPath = remoteLinkFileBasePathWindows + timeFormate + File.separator; + log.info("=========== [dirPath is {} ] =========== ", dirPath); FileUtil.mkdir(new File(dirPath)); // 构建存储文件 - return dirPath + filePath; + return dirPath + baseStr; } else if (osInfo.isLinux()) { dirPath = remoteLinkFileBasePathLinux + timeFormate + File.separator; + log.info("=========== [dirPath is {} ] =========== ", dirPath); FileUtil.mkdir(new File(dirPath)); // 构建存储文件 - return dirPath + filePath; + return dirPath + baseStr; } else if (osInfo.isMac()) { dirPath = remoteLinkFileBasePathMac + timeFormate + File.separator; + log.info("=========== [dirPath is {} ] =========== ", dirPath); FileUtil.mkdir(new File(dirPath)); // 构建存储文件 - return dirPath + filePath; + return dirPath + baseStr; } else { return ""; } diff --git a/eladmin-system/src/main/resources/config/application-dev.yml b/eladmin-system/src/main/resources/config/application-dev.yml index 3eae4a3..b7ec013 100644 --- a/eladmin-system/src/main/resources/config/application-dev.yml +++ b/eladmin-system/src/main/resources/config/application-dev.yml @@ -231,7 +231,7 @@ tag: sum: 10 remote: link: - address: 'http://localhost:8001/api/temp/file/download' + address: 'http://localhost:8000/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-prod.yml b/eladmin-system/src/main/resources/config/application-prod.yml index de01d4f..99ef6bf 100644 --- a/eladmin-system/src/main/resources/config/application-prod.yml +++ b/eladmin-system/src/main/resources/config/application-prod.yml @@ -189,7 +189,7 @@ tag: sum: 10 remote: link: - address: 'http://118.178.137.129:8001/api/temp/file/download' + address: 'http://118.178.137.129:8000/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-single.yml b/eladmin-system/src/main/resources/config/application-single.yml index f22275e..dfb10f9 100644 --- a/eladmin-system/src/main/resources/config/application-single.yml +++ b/eladmin-system/src/main/resources/config/application-single.yml @@ -169,7 +169,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8001/api/temp/file/download' + address: 'http://116.62.197.152:8000/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-stg.yml b/eladmin-system/src/main/resources/config/application-stg.yml index b38ed79..9732ad3 100644 --- a/eladmin-system/src/main/resources/config/application-stg.yml +++ b/eladmin-system/src/main/resources/config/application-stg.yml @@ -231,7 +231,7 @@ tag: sum: 10 remote: link: - address: 'http://116.62.197.152:8001/api/temp/file/download' + address: 'http://116.62.197.152:8000/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file diff --git a/eladmin-system/src/main/resources/config/application-test.yml b/eladmin-system/src/main/resources/config/application-test.yml index f24327a..8b7f207 100644 --- a/eladmin-system/src/main/resources/config/application-test.yml +++ b/eladmin-system/src/main/resources/config/application-test.yml @@ -208,6 +208,13 @@ SendBigData: maxPoolSize: 16 queueCapacity: 3 ThreadNamePrefix: 'SendBigDataTaskExecutor-' +remoteRec: + task: + thread_pool: + corePoolSize: 2 + maxPoolSize: 16 + queueCapacity: 3 + ThreadNamePrefix: 'RemoteRecTaskExecutor-' # 增加日志相关的配置 logging: level: @@ -217,14 +224,14 @@ logging: org.springframework.web: error org.hibernate.SQL: debug org.hibernate.engine.QueryParameters: debug - org.hibernate.engine.query.HQLQueryPlan: debug + org.hibernate.engine.query.HQLQueryPlan: DEBUG org.hibernate.type.descriptor.sql.BasicBinder: trace tag: split-table: sum: 10 remote: link: - address: 'http://116.62.197.152:8001/api/temp/file/download' + address: 'http://116.62.197.152:8000/api/temp/file/download' file-base-path-linux: /home/eladmin/file/temp/ file-base-path-windows: C:\eladmin\file\temp\ - file-base-path-mac: ~/file/eladmin/temp/ \ No newline at end of file + file-base-path-mac: ~/file/eladmin/temp/ diff --git a/eladmin-system/src/main/resources/config/application.yml b/eladmin-system/src/main/resources/config/application.yml index 8a59dec..6a02dab 100644 --- a/eladmin-system/src/main/resources/config/application.yml +++ b/eladmin-system/src/main/resources/config/application.yml @@ -1,5 +1,5 @@ server: - port: 8001 + port: 8000 spring: freemarker: diff --git a/eladmin-system/src/test/java/me/zhengjie/StreamTest.java b/eladmin-system/src/test/java/me/zhengjie/StreamTest.java index 8a89676..77d0f7b 100644 --- a/eladmin-system/src/test/java/me/zhengjie/StreamTest.java +++ b/eladmin-system/src/test/java/me/zhengjie/StreamTest.java @@ -13,26 +13,35 @@ import java.util.stream.Collectors; public class StreamTest { - private static Set uids; - +// private static Set uids; private static List base; + private static Set equals; + static { - uids = new HashSet<>(50_0000); - uids.add("111111111111111111111111"); - uids.add("111111111111111111111112"); - for (int i = 0; i < 500000; i++) { - uids.add(RandomUtil.randomString(24)); - } - uids.add("111111111111111111111113"); +// uids = new HashSet<>(50_0000); +// uids.add("111111111111111111111111"); +// uids.add("111111111111111111111112"); +// for (int i = 0; i < 500000; i++) { +// uids.add(RandomUtil.randomString(24)); +// } +// uids.add("111111111111111111111113"); - base = new ArrayList<>(2000_0000); + base = new ArrayList<>(1_0000); base.add("111111111111111111111111"); base.add("111111111111111111111112"); - for (int i = 0; i < 2000_0000; i++) { + for (int i = 0; i < 20_0000; i++) { base.add(RandomUtil.randomString(24)); } base.add("111111111111111111111113"); + + equals = new HashSet<>(1_0000); + equals.add("111111111111111111111111"); + equals.add("111111111111111111111112"); + for (int i = 0; i < 20_0000; i++) { + equals.add(RandomUtil.randomString(24)); + } + equals.add("111111111111111111111113"); } /** @@ -43,13 +52,33 @@ public class StreamTest { */ @Test public void testFilter(){ +// Long start = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); +// System.out.println(" start , " + start); +//// Set collect = uids.stream().parallel().filter(base::contains).collect(Collectors.toSet()); +// Set collect = base.stream().parallel().filter(uids::contains).collect(Collectors.toSet()); // 2119 334 +// +// System.out.println("uids size :" + uids.size() + ", base size: " + base.size() + ", 重复"+ collect.size()); +// Long end = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); +// System.out.println(" end, "+ end +", cost , " + (end - start)); + } + + /** + * 测试去重写法 - c测试想等的情况 + * 测试结论: + * 1. 一定要在大集合里面去重小集合 + * 2. 数据结构大集合采用 List 小集合采用 Set + */ + @Test + public void testFilter2(){ Long start = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); System.out.println(" start , " + start); // Set collect = uids.stream().parallel().filter(base::contains).collect(Collectors.toSet()); - Set collect = base.stream().parallel().filter(uids::contains).collect(Collectors.toSet()); // 2119 334 + Set collect = base.stream().parallel().filter(equals::contains).collect(Collectors.toSet()); // 2119 334 - System.out.println("uids size :" + uids.size() + ", base size: " + base.size() + ", 重复"+ collect.size()); + System.out.println("equals size :" + equals.size() + ", base size: " + base.size() + ", 重复"+ collect.size()); Long end = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli(); System.out.println(" end, "+ end +", cost , " + (end - start)); } + + }