上传平台修复bug最终版4

eladmin-system/src/main/java/me/zhengjie/modules/buildrecord/task/SendBigDataTask.java

@@ -6,6 +6,7 @@ import cn.hutool.core.util.RandomUtil;
 import cn.hutool.http.HttpResponse;
 import com.alibaba.fastjson.JSON;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import lombok.extern.slf4j.Slf4j;
 import me.zhengjie.common.json.PushDBJsonContent;
 import me.zhengjie.modules.buildrecord.domain.BuildRecord;
@@ -199,12 +200,29 @@ public class SendBigDataTask {
             } catch (IOException e) {
                 log.error("================== {read file error , please check is , file path is : {} } ================================", resultFilePath, e);
             }
-            // 过滤的集合
+            // 根据集合的大小不同采取不提,不同的方案
-            Set<String> filterCollect = remoteCollect.stream().parallel().filter(fileLines::contains).collect(Collectors.toSet());
+            if (remoteCollect.size() > fileLines.size()){
-            if (CollectionUtil.isNotEmpty(filterCollect)){
+                // 重复的set
-                log.info("================== [Filter collect is ready, collect size is {} ] ================================",filterCollect.size());
+                Set<String> filterCollect = remoteCollect.stream().parallel().filter(fileLines::contains).collect(Collectors.toSet());
-                remoteCollect = remoteCollect.stream().filter(one -> !filterCollect.contains(one)).collect(Collectors.toList());
+                if (CollectionUtil.isNotEmpty(filterCollect)){
+                    log.info("================== [Filter collect is ready, collect size is {} ] ================================",filterCollect.size());
+                    // 把重复的进行去除
+                    remoteCollect = remoteCollect.stream().filter(one -> !filterCollect.contains(one)).collect(Collectors.toList());
+                }
             }
+
+            if (remoteCollect.size() <= fileLines.size()){
+                List<String> fileLinesList = Lists.newArrayList(fileLines);
+                Set<String> remoteCollectSet = Sets.newHashSet(remoteCollect);
+                //  重复的set
+                Set<String> filterCollect = fileLinesList.stream().parallel().filter(remoteCollectSet::contains).collect(Collectors.toSet());
+                if (CollectionUtil.isNotEmpty(filterCollect)){
+                    // 把重复的进行去除
+                    log.info("================== [Filter collect is ready, collect size is {} ] ================================",filterCollect.size());
+                    remoteCollect = remoteCollect.stream().filter(one -> !filterCollect.contains(one)).collect(Collectors.toList());
+                }
+            }
+
         }
         return remoteCollect;
     }

eladmin-system/src/main/java/me/zhengjie/modules/remoterec/consts/SysConst.java

@@ -6,7 +6,7 @@ package me.zhengjie.modules.remoterec.consts;
 public class SysConst {
 
     // 远程服务器的相关配置
-    public static final String REMOTE_TRANS_HOST = "116.62.197.152";
+    public static final String REMOTE_TRANS_HOST = "116.62.197.152"; // 118.178.137.129 116.62.197.152
 
     public static final Integer REMOTE_TRANS_PORT = 22;
 

eladmin-system/src/main/java/me/zhengjie/modules/remoterec/rest/RemoteRecordController.java

@@ -64,6 +64,7 @@ public class RemoteRecordController {
     @Log("查询远程上传记录")
     @ApiOperation("查询远程上传记录")
     @PreAuthorize("@el.check('remoteRecord:list')")
+    @AnonymousAccess
     public ResponseEntity<Object> query(RemoteRecordQueryCriteria criteria, Pageable pageable){
         return new ResponseEntity<>(remoteRecordService.queryAll(criteria,pageable),HttpStatus.OK);
     }

eladmin-system/src/main/java/me/zhengjie/modules/remoterec/task/DownloadSFTPFileTask.java

@@ -7,6 +7,7 @@ import java.sql.Timestamp;
 import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollectionUtil;
 import cn.hutool.core.util.RandomUtil;
+import cn.hutool.extra.ftp.Ftp;
 import cn.hutool.extra.ssh.JschUtil;
 import cn.hutool.extra.ssh.Sftp;
 import cn.hutool.system.OsInfo;
@@ -84,6 +85,7 @@ public class DownloadSFTPFileTask {
     }
 
     private void runTask(RemoteRecord resources) {
+        resources.setOperation(resources.getOperation() + FILE_NAME_SPLIT + resources.getUploadRemoteTaskName());
         // 更新一条记录,然后状态为待取回状态
         RemoteRecordDto remoteRecordDto = remoteRecordService.create(resources);
         // 然后调用sftp进行取回
@@ -116,13 +118,13 @@ public class DownloadSFTPFileTask {
 
         remoteRecordService.update(remoteRecord);
 
+//        sftp.close();
         // fixme 更新失败补充推送费用
     }
 
     private String sftpToDownloadFile(String sftpFilePath, RemoteRecord resources) {
-
         // 准备sftp配置
-        Session session = JschUtil.getSession(REMOTE_TRANS_HOST, REMOTE_TRANS_PORT, REMOTE_TRANS_SSH_USER, REMOTE_TRANS_SSH_PW);
+        Session session = JschUtil.createSession(REMOTE_TRANS_HOST, REMOTE_TRANS_PORT, REMOTE_TRANS_SSH_USER, REMOTE_TRANS_SSH_PW);
         Sftp sftp = JschUtil.createSftp(session);
 
         // 构建输出
@@ -136,11 +138,11 @@ public class DownloadSFTPFileTask {
         resources.setLocalSavePath(localSavePath);
         log.info(" ====== [DownloadSFTPFileTask|sftpToDownloadFiles , ftpFilePath is {} ] ======", sftpFilePath);
         // SFTP 进行下载
-        sftp.download(sftpFilePath, FileUtil.file(localSavePath));
+        sftp.get(sftpFilePath, localSavePath);
-        sftp.close();
 
+        sftp.close();
+//        sftp.download(sftpFilePath, FileUtil.file(localSavePath));
         return localSavePath;
-
     }
 
     /**
@@ -155,23 +157,24 @@ public class DownloadSFTPFileTask {
         String dirPath;
         // 生成一个随机文件夹目录,方便整理和打包
 
-        String filePath = RandomUtil.randomString(6) + FILE_NAME_SPLIT + baseStr;
-
         if (osInfo.isWindows()) {
             dirPath = remoteLinkFileBasePathWindows + timeFormate + File.separator;
+            log.info("=========== [dirPath is {} ] =========== ", dirPath);
             FileUtil.mkdir(new File(dirPath));
             // 构建存储文件
-            return dirPath + filePath;
+            return dirPath + baseStr;
         } else if (osInfo.isLinux()) {
             dirPath = remoteLinkFileBasePathLinux + timeFormate + File.separator;
+            log.info("=========== [dirPath is {} ] =========== ", dirPath);
             FileUtil.mkdir(new File(dirPath));
             // 构建存储文件
-            return dirPath + filePath;
+            return dirPath + baseStr;
         } else if (osInfo.isMac()) {
             dirPath = remoteLinkFileBasePathMac + timeFormate + File.separator;
+            log.info("=========== [dirPath is {} ] =========== ", dirPath);
             FileUtil.mkdir(new File(dirPath));
             // 构建存储文件
-            return dirPath + filePath;
+            return dirPath + baseStr;
         } else {
             return "";
         }

eladmin-system/src/main/resources/config/application-dev.yml

@@ -231,7 +231,7 @@ tag:
     sum: 10
 remote:
   link:
-    address: 'http://localhost:8001/api/temp/file/download'
+    address: 'http://localhost:8000/api/temp/file/download'
     file-base-path-linux: /home/eladmin/file/temp/
     file-base-path-windows: C:\eladmin\file\temp\
     file-base-path-mac: ~/file/eladmin/temp/

eladmin-system/src/main/resources/config/application-prod.yml

@@ -189,7 +189,7 @@ tag:
     sum: 10
 remote:
   link:
-    address: 'http://118.178.137.129:8001/api/temp/file/download'
+    address: 'http://118.178.137.129:8000/api/temp/file/download'
     file-base-path-linux: /home/eladmin/file/temp/
     file-base-path-windows: C:\eladmin\file\temp\
     file-base-path-mac: ~/file/eladmin/temp/

eladmin-system/src/main/resources/config/application-single.yml

@@ -169,7 +169,7 @@ tag:
     sum: 10
 remote:
   link:
-    address: 'http://116.62.197.152:8001/api/temp/file/download'
+    address: 'http://116.62.197.152:8000/api/temp/file/download'
     file-base-path-linux: /home/eladmin/file/temp/
     file-base-path-windows: C:\eladmin\file\temp\
     file-base-path-mac: ~/file/eladmin/temp/

eladmin-system/src/main/resources/config/application-stg.yml

@@ -231,7 +231,7 @@ tag:
     sum: 10
 remote:
   link:
-    address: 'http://116.62.197.152:8001/api/temp/file/download'
+    address: 'http://116.62.197.152:8000/api/temp/file/download'
     file-base-path-linux: /home/eladmin/file/temp/
     file-base-path-windows: C:\eladmin\file\temp\
     file-base-path-mac: ~/file/eladmin/temp/

eladmin-system/src/main/resources/config/application-test.yml

@@ -208,6 +208,13 @@ SendBigData:
       maxPoolSize: 16
       queueCapacity: 3
       ThreadNamePrefix: 'SendBigDataTaskExecutor-'
+remoteRec:
+  task:
+    thread_pool:
+      corePoolSize: 2
+      maxPoolSize: 16
+      queueCapacity: 3
+      ThreadNamePrefix: 'RemoteRecTaskExecutor-'
 # 增加日志相关的配置
 logging:
   level:
@@ -217,14 +224,14 @@ logging:
     org.springframework.web: error
     org.hibernate.SQL: debug
     org.hibernate.engine.QueryParameters: debug
-    org.hibernate.engine.query.HQLQueryPlan: debug
+    org.hibernate.engine.query.HQLQueryPlan: DEBUG
     org.hibernate.type.descriptor.sql.BasicBinder: trace
 tag:
   split-table:
     sum: 10
 remote:
   link:
-    address: 'http://116.62.197.152:8001/api/temp/file/download'
+    address: 'http://116.62.197.152:8000/api/temp/file/download'
     file-base-path-linux: /home/eladmin/file/temp/
     file-base-path-windows: C:\eladmin\file\temp\
     file-base-path-mac: ~/file/eladmin/temp/

eladmin-system/src/main/resources/config/application.yml

@@ -1,5 +1,5 @@
 server:
-  port: 8001
+  port: 8000
 
 spring:
   freemarker:

eladmin-system/src/test/java/me/zhengjie/StreamTest.java

@@ -13,26 +13,35 @@ import java.util.stream.Collectors;
 
 public class StreamTest {
 
-    private static Set<String> uids;
+//    private static Set<String> uids;
-
     private static List<String> base;
 
+    private static Set<String> equals;
+
     static {
-        uids = new HashSet<>(50_0000);
+//        uids = new HashSet<>(50_0000);
-        uids.add("111111111111111111111111");
+//        uids.add("111111111111111111111111");
-        uids.add("111111111111111111111112");
+//        uids.add("111111111111111111111112");
-        for (int i = 0; i < 500000; i++) {
+//        for (int i = 0; i < 500000; i++) {
-            uids.add(RandomUtil.randomString(24));
+//            uids.add(RandomUtil.randomString(24));
-        }
+//        }
-        uids.add("111111111111111111111113");
+//        uids.add("111111111111111111111113");
 
-        base = new ArrayList<>(2000_0000);
+        base = new ArrayList<>(1_0000);
         base.add("111111111111111111111111");
         base.add("111111111111111111111112");
-        for (int i = 0; i < 2000_0000; i++) {
+        for (int i = 0; i < 20_0000; i++) {
             base.add(RandomUtil.randomString(24));
         }
         base.add("111111111111111111111113");
+
+        equals = new HashSet<>(1_0000);
+        equals.add("111111111111111111111111");
+        equals.add("111111111111111111111112");
+        for (int i = 0; i < 20_0000; i++) {
+            equals.add(RandomUtil.randomString(24));
+        }
+        equals.add("111111111111111111111113");
     }
 
     /**
@@ -43,13 +52,33 @@ public class StreamTest {
      */
     @Test
     public void testFilter(){
+//        Long start = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
+//        System.out.println(" start , " + start);
+////        Set<String> collect = uids.stream().parallel().filter(base::contains).collect(Collectors.toSet());
+//        Set<String> collect = base.stream().parallel().filter(uids::contains).collect(Collectors.toSet()); // 2119 334
+//
+//        System.out.println("uids size :" + uids.size() + ", base size: " + base.size() + ", 重复"+ collect.size());
+//        Long end = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
+//        System.out.println(" end, "+ end +", cost , " + (end - start));
+    }
+
+    /**
+     * 测试去重写法 - c测试想等的情况
+     * 测试结论:
+     *  1. 一定要在大集合里面去重小集合
+     *  2. 数据结构大集合采用 List 小集合采用 Set
+     */
+    @Test
+    public void testFilter2(){
         Long start = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
         System.out.println(" start , " + start);
 //        Set<String> collect = uids.stream().parallel().filter(base::contains).collect(Collectors.toSet());
-        Set<String> collect = base.stream().parallel().filter(uids::contains).collect(Collectors.toSet()); // 2119 334
+        Set<String> collect = base.stream().parallel().filter(equals::contains).collect(Collectors.toSet()); // 2119 334
 
-        System.out.println("uids size :" + uids.size() + ", base size: " + base.size() + ", 重复"+ collect.size());
+        System.out.println("equals size :" + equals.size() + ", base size: " + base.size() + ", 重复"+ collect.size());
         Long end = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();
         System.out.println(" end, "+ end +", cost , " + (end - start));
     }
+
+
 }