diff --git a/database/mycat/README.md b/database/mycat/README.md new file mode 100644 index 0000000..bec8c61 --- /dev/null +++ b/database/mycat/README.md @@ -0,0 +1,35 @@ +# MyCat指南 + +## 1. Mycat相关的基本概念 + +* Mycat的概念 + * MyCAT相当于MySQL SERVER层 + * MySQL相当于MyCAT的存储层 + * MyCAT中不存储数据,所有的数据存储在MySQL中 + * MyCAT对于研发人员来说是透明的 + * MyCAT对使用的SQL有一些限制 + * MyCAT是数据库中间层 + * MyCAT可以实现对后端数据库的分库分表和读写分离 + * MyCAT对前端应用隐藏了后端数据库的存储逻辑 +* 什么是数据中间层 + * 链接数据和应用的一层设计，MyCAT就是用来实现数据中间层的数据中间件 +* MyCAT的主要作用 + * 作为分布式数据库中间层使用 + * 实现后端数据库读写分离及负载均衡 + * 对业务数据库进行垂直/水平切分 + * 控制数据库连接的数量 +* Mycat的基本元素 + * 逻辑库 + * 应用层直接看成数据库即可，实际可能是由多个数据库组成的逻辑库 + * 逻辑库中不会保存数据，只是一个对外开放数据库的概念 + * 逻辑表 + * 水平切分后的订单表，结构相同，对外可以作为一张大表来使用 + * 逻辑表也是概念的东西，实际并不用来进行存储数据 + * 逻辑表的种类: 分片表和非分片表；全局表，在所有分片中都存在的表；ER关系表，按ER关系进行分片的表 +* MyCAT安装 + * 下载解压MyCAT + * 安装JAVA运行环境 + * 新建MyCAT运行系统账号 + * 配置系统的环境变量 + * 修改Mycat启动参数,启动MyCAT + * diff --git a/database/mycat/pic/MyCAT作为分布式数据库中间层的使用.png b/database/mycat/pic/MyCAT作为分布式数据库中间层的使用.png new file mode 100644 index 0000000..87dd5b5 Binary files /dev/null and b/database/mycat/pic/MyCAT作为分布式数据库中间层的使用.png differ diff --git a/database/mycat/pic/Mycat架构图.png b/database/mycat/pic/Mycat架构图.png new file mode 100644 index 0000000..7326d15 Binary files /dev/null and b/database/mycat/pic/Mycat架构图.png differ diff --git a/dev-protocol-tools/pom.xml b/dev-protocol-tools/pom.xml index 484e5b3..6efb5d5 100644 --- a/dev-protocol-tools/pom.xml +++ b/dev-protocol-tools/pom.xml @@ -47,6 +47,12 @@ easyexcel 3.0.5 + + org.jetbrains + annotations + RELEASE + compile + \ No newline at end of file diff --git a/dev-protocol-tools/src/main/java/com/baiye/number/NumberUtil.java b/dev-protocol-tools/src/main/java/com/baiye/number/NumberUtil.java new file mode 100644 index 0000000..d1b7509 --- /dev/null +++ b/dev-protocol-tools/src/main/java/com/baiye/number/NumberUtil.java @@ -0,0 +1,82 @@ +package com.baiye.number; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; + +/** + * 号码生成工具类 + * + * @author q + * @date 2022/06/15 + */ +public class NumberUtil { + + + /** + * 创建流水号函数 + * + * @param type 类型 线上/线下 1位 + * @param organizationId 零售店/仓库 6位 + * @param spgId 品类编号 5位 + * @param date 购物日期 6位 + * @return {@link String} + */ + public static String createOrderCode(String type, String organizationId, String spgId, String date) { + return type + + organizationId + + spgId + + date + + produceRandomNumber(); + } + + static StringBuffer produceRandomNumber() { + StringBuffer buffer = new StringBuffer(); + ThreadLocalRandom.current().ints(0, 9).limit(10).forEach( + buffer::append + ); + return buffer; + } + + + /** + * 测试 - + *

+ * fixme 这边把 只能通过线程池方式创建线程的警告进行忽略 + */ + @SuppressWarnings("all") + public static void main(String[] args) { + + int sum = 10000; + + CountDownLatch countDownLatch = new CountDownLatch(sum); + Set set = new HashSet<>(sum); + + for (int i = 0; i < sum; i++) { + new Thread(() -> { + String num = createOrderCode("S", "000012", "11002", "2022615"); + System.out.println( + Thread.currentThread().getName() + + " - " + + num); + set.add(num); + countDownLatch.countDown(); + }).start(); + } + + try { + countDownLatch.await(); + // 结果为 定义的 sum值表示 生成的都是唯一的Id,且数目对的 + System.out.println(set.size()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + + + } + +} diff --git a/longpolling/demo/demo1/README.md b/longpolling/demo/demo1/README.md new file mode 100644 index 0000000..9d1c02d --- /dev/null +++ b/longpolling/demo/demo1/README.md @@ -0,0 +1,3 @@ +# Disruptor与Netty实现百万级长连接接入 + +## diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-client/.gitignore b/longpolling/demo/demo1/dev-protocol-disruptor-netty-client/.gitignore new file mode 100644 index 0000000..d4beec9 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-client/.gitignore @@ -0,0 +1,46 @@ +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen +### Maven template +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +### Java template +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-client/pom.xml b/longpolling/demo/demo1/dev-protocol-disruptor-netty-client/pom.xml new file mode 100644 index 0000000..26bb31e --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-client/pom.xml @@ -0,0 +1,20 @@ + + + + dev-protocol + org.example + 1.0-SNAPSHOT + ../../../../pom.xml + + 4.0.0 + + dev-protocol-disruptor-netty-client + + + 8 + 8 + + + \ No newline at end of file diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/.gitignore b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/.gitignore new file mode 100644 index 0000000..d4beec9 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/.gitignore @@ -0,0 +1,46 @@ +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen +### Maven template +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +### Java template +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/pom.xml b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/pom.xml new file mode 100644 index 0000000..e986b9c --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/pom.xml @@ -0,0 +1,72 @@ + + + + dev-protocol + org.example + 1.0-SNAPSHOT + ../../../../pom.xml + + 4.0.0 + + dev-protocol-disruptor-netty-common + + + 8 + 8 + + + + + org.projectlombok + lombok + + + io.netty + netty-all + 4.1.12.Final + + + + org.jboss.marshalling + jboss-marshalling + 1.3.0.CR9 + + + org.jboss.marshalling + jboss-marshalling-serial + 1.3.0.CR9 + + + + com.google.protobuf + protobuf-java + 2.5.0 + + + + com.lmax + disruptor + 3.3.2 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + + \ No newline at end of file diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/src/main/java/com/baiye/codec/MarshallingCodeCFactory.java b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/src/main/java/com/baiye/codec/MarshallingCodeCFactory.java new file mode 100644 index 0000000..9dd88eb --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/src/main/java/com/baiye/codec/MarshallingCodeCFactory.java @@ -0,0 +1,49 @@ +package com.baiye.codec; + +import io.netty.handler.codec.marshalling.DefaultMarshallerProvider; +import io.netty.handler.codec.marshalling.DefaultUnmarshallerProvider; +import io.netty.handler.codec.marshalling.MarshallerProvider; +import io.netty.handler.codec.marshalling.MarshallingDecoder; +import io.netty.handler.codec.marshalling.MarshallingEncoder; +import io.netty.handler.codec.marshalling.UnmarshallerProvider; + +import org.jboss.marshalling.MarshallerFactory; +import org.jboss.marshalling.Marshalling; +import org.jboss.marshalling.MarshallingConfiguration; + +/** + * Marshalling工厂 - 进行自定义编码 + */ +public final class MarshallingCodeCFactory { + + /** + * 创建Jboss Marshalling解码器MarshallingDecoder + * @return MarshallingDecoder + */ + public static MarshallingDecoder buildMarshallingDecoder() { + //首先通过Marshalling工具类的精通方法获取Marshalling实例对象 参数serial标识创建的是java序列化工厂对象。 + final MarshallerFactory marshallerFactory = Marshalling.getProvidedMarshallerFactory("serial"); + //创建了MarshallingConfiguration对象，配置了版本号为5 + final MarshallingConfiguration configuration = new MarshallingConfiguration(); + configuration.setVersion(5); + //根据marshallerFactory和configuration创建provider + UnmarshallerProvider provider = new DefaultUnmarshallerProvider(marshallerFactory, configuration); + //构建Netty的MarshallingDecoder对象，俩个参数分别为provider和单个消息序列化后的最大长度 + MarshallingDecoder decoder = new MarshallingDecoder(provider, 1024 * 1024 * 1); + return decoder; + } + + /** + * 创建Jboss Marshalling编码器MarshallingEncoder + * @return MarshallingEncoder + */ + public static MarshallingEncoder buildMarshallingEncoder() { + final MarshallerFactory marshallerFactory = Marshalling.getProvidedMarshallerFactory("serial"); + final MarshallingConfiguration configuration = new MarshallingConfiguration(); + configuration.setVersion(5); + MarshallerProvider provider = new DefaultMarshallerProvider(marshallerFactory, configuration); + //构建Netty的MarshallingEncoder对象，MarshallingEncoder用于实现序列化接口的POJO对象序列化为二进制数组 + MarshallingEncoder encoder = new MarshallingEncoder(provider); + return encoder; + } +} diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/src/main/java/com/baiye/dto/TranslatorData.java b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/src/main/java/com/baiye/dto/TranslatorData.java new file mode 100644 index 0000000..ae95f37 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-common/src/main/java/com/baiye/dto/TranslatorData.java @@ -0,0 +1,38 @@ +package com.baiye.dto; + +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.io.Serializable; + +/** + * 传输数据对象 + * + * @author q + * @date 2022/07/11 + */ +@Getter +@Setter +@ToString +public class TranslatorData implements Serializable { + + + /** + * id + */ + private String id; + + /** + * 消息的名字 + */ + private String name; + + /** + * 消息内容 + */ + private String message; + + +} diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/.gitignore b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/.gitignore new file mode 100644 index 0000000..d4beec9 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/.gitignore @@ -0,0 +1,46 @@ +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen +### Maven template +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +### Java template +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/pom.xml b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/pom.xml new file mode 100644 index 0000000..1714e43 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/pom.xml @@ -0,0 +1,58 @@ + + + + dev-protocol + org.example + 1.0-SNAPSHOT + ../../../../pom.xml + + 4.0.0 + + dev-protocol-disruptor-netty-server + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.example + dev-protocol-disruptor-netty-common + 1.0-SNAPSHOT + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + + + \ No newline at end of file diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/NettyServerApplication.java b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/NettyServerApplication.java new file mode 100644 index 0000000..3117ca0 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/NettyServerApplication.java @@ -0,0 +1,4 @@ +package com.baiye; + +public class NettyServerApplication { +} diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/server/NettyServer.java b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/server/NettyServer.java new file mode 100644 index 0000000..46114a0 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/server/NettyServer.java @@ -0,0 +1,78 @@ +package com.baiye.server; + +import com.baiye.codec.MarshallingCodeCFactory; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.channel.AdaptiveRecvByteBufAllocator; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; + +/** + * Netty Server 端 + * + * @author q + * @date 2022/07/11 + */ +public class NettyServer { + + + /** + * Netty 两个线程租的底层原理 + * 图示: + * + * —————— ——————————————————————————————————————————— + * | |------------ip/port,发起第一次握手----------------|-> —————— Server[Online] | + * | | | | | | + * | |<----------ack包,发起第而次握手-------------------|- |SYNC | | + * | | | —————— | + * | | | sync + accept = backlog | + * | Client | | —————— | + * | |--第三次握手成功，把SYNC中的链接放到Accept队列中----->| |Accept| | + * | | | —————— | + * | | | serverSocket.accept() | + * —————— ——————————————————————————————————————————— + * + * + */ + public NettyServer() { + // 1. 创建两个工作线程组: 1个用于接收网络请求的, 另一个用于实际处理业务的线程组 + EventLoopGroup bossGroup = new NioEventLoopGroup(); + + EventLoopGroup workGroup = new NioEventLoopGroup(); + + // 创建辅助类 - 帮助构建Netty模型 + ServerBootstrap serverBootstrap = new ServerBootstrap() + .group(bossGroup, workGroup) + .channel(NioServerSocketChannel.class) + // fixme 设置 backlog大小 + .option(ChannelOption.SO_BACKLOG, 1024) + // fixme 表示缓存区动态调配(自适应) 数据包相差不大的时候比较合适 + .option(ChannelOption.RCVBUF_ALLOCATOR, AdaptiveRecvByteBufAllocator.DEFAULT) + // 缓冲区 池化操作 + .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + // 记录日志 + .handler(new LoggingHandler(LogLevel.INFO)) + // 数据接收过来给哪个方法进行回调 -> 接收数据进行异步处理 + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel socketChannel) throws Exception { + // fixme 在管道上加一些拦截器进行处理 + + // 进行设置编解码 -> 对Java对象转为的二进制数据进行编解码 + socketChannel.pipeline().addLast(MarshallingCodeCFactory.buildMarshallingDecoder()); + socketChannel.pipeline().addLast(MarshallingCodeCFactory.buildMarshallingEncoder()); + // 信息处理 + socketChannel.pipeline().addLast(new ServerHandler()); + } + }); + + + + } +} diff --git a/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/server/ServerHandler.java b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/server/ServerHandler.java new file mode 100644 index 0000000..e91e8a1 --- /dev/null +++ b/longpolling/demo/demo1/dev-protocol-disruptor-netty-server/src/main/java/com/baiye/server/ServerHandler.java @@ -0,0 +1,32 @@ +package com.baiye.server; + +import com.baiye.dto.TranslatorData; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; + +/** + * 继承其接口的子类,防止去重写那么多的接口 + * + * @author q + * @date 2022/07/11 + */ +public class ServerHandler extends ChannelInboundHandlerAdapter { + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + + // 接收到数据 数据已经进行了编解码, 所以直接进行转换 + TranslatorData request = (TranslatorData) msg; + System.out.println("Server 端数据 ： " + request.toString()); + + // 设计响应回送数据 + TranslatorData response = new TranslatorData(); + response.setId("resp: " + request.getId()); + response.setName("resp: " + request.getName()); + response.setMessage("resp: " + request.getMessage()); + + // 写数据并刷新 + ctx.writeAndFlush(response); + + } +} diff --git a/pom.xml b/pom.xml index c6941d4..29b12bc 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,9 @@ dev-protocol-pay dev-protocol-tools dev-protocol-drools + longpolling/demo/demo1/dev-protocol-disruptor-netty-client + longpolling/demo/demo1/dev-protocol-disruptor-netty-server + longpolling/demo/demo1/dev-protocol-disruptor-netty-common