【软件部署】用docker部署Apache Kafka 集群架构的isolated模式

国内服务器1周前发布 beixibaobao
6 0 0

说明

与Kafka官网配置的差异

  • 新增数据持久化到数据卷
  • 指定容器名，规避更换目录导致名字变化问题
  • 指定专用网络，使容器可通过服务名或容器名直接访问，规避IP限制。

无SSL配置，集群，isolated

模式差异

模式 核心定义 关键配置 节点类型
Combined (合并) 模式 单个节点同时承担 Controller (控制器)和Broker (数据节点) 双重角色 process.roles=broker,controller 合并节点 (Combined Node)
Isolated (分离) 模式 控制器与 Broker 角色完全分离，各自运行在独立节点上 控制器节点: process.roles=controllerBroker 节点: process.roles=broker 专用控制器节点 (Dedicated Controller)纯 Broker 节点 (Pure Broker)

适配场景

模式 最佳适用场景 不推荐场景
Combined 模式 1. 开发 / 测试环境2. 小型 POC (概念验证)3. 资源受限的边缘环境4. 学习 KRaft 架构的实验环境 1. 生产环境2. 大规模集群 (>10 节点)3. 高可用要求的关键业务4. 安全敏感场景
Isolated 模式 1. 生产环境2. 大规模集群 (>20 节点)3. 关键业务系统 (金融、电商等)4. 对稳定性和安全性要求高的场景 1. 资源极度受限的环境2. 简单测试或演示环境

配置

创建文件compose.yaml

# 无SSL
services:
  init-kafka-perms:
    image: busybox:latest
    command: sh -c "chown -R 1000:1000 /controller-1 /controller-2 /controller-3 /kafka1 /kafka2 /kafka3"
    volumes:
      - controller-1:/controller-1
      - controller-2:/controller-2
      - controller-3:/controller-3
      - kafka1-logs:/kafka1
      - kafka2-logs:/kafka2
      - kafka3-logs:/kafka3
    restart: "no"
    container_name: kafka-perms-fix
    networks:
      - kafka
  controller-1:
    hostname: controller-1
    container_name: kafka-controller-1
    image: apache/kafka:4.2.0
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: 'controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LISTENERS: 'CONTROLLER://0.0.0.0:9093'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      init-kafka-perms:
        condition: service_completed_successfully
    networks:
      - kafka
    volumes:
      - controller-1:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 9093 || exit 1
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
  controller-2:
    hostname: controller-2
    container_name: kafka-controller-2
    image: apache/kafka:4.2.0
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_PROCESS_ROLES: 'controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LISTENERS: 'CONTROLLER://0.0.0.0:9093'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      init-kafka-perms:
        condition: service_completed_successfully
    networks:
      - kafka
    volumes:
      - controller-2:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 9093 || exit 1
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
  controller-3:
    hostname: controller-3
    container_name: kafka-controller-3
    image: apache/kafka:4.2.0
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 3
      KAFKA_PROCESS_ROLES: 'controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LISTENERS: 'CONTROLLER://0.0.0.0:9093'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      init-kafka-perms:
        condition: service_completed_successfully
    networks:
      - kafka
    volumes:
      - controller-3:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 9093 || exit 1
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
  kafka-1:
    image: apache/kafka:4.2.0
    ports:
      - 29092:9092
    hostname: kafka-1
    container_name: kafka-1
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 4
      KAFKA_PROCESS_ROLES: 'broker'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093'
      KAFKA_LISTENERS: 'PLAINTEXT://:19092,PLAINTEXT_HOST://:9092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-1:19092,PLAINTEXT_HOST://localhost:29092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      controller-1: { condition: service_healthy }
      controller-2: { condition: service_healthy }
      controller-3: { condition: service_healthy }
    networks:
      - kafka
    volumes:
      - kafka1-logs:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 9092 || exit 1
      interval: 60s
      timeout: 5s
      retries: 2
      start_period: 30s
  kafka-2:
    image: apache/kafka:4.2.0
    ports:
      - 39092:9092
    hostname: kafka-2
    container_name: kafka-2
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 5
      KAFKA_PROCESS_ROLES: 'broker'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093'
      KAFKA_LISTENERS: 'PLAINTEXT://:19092,PLAINTEXT_HOST://:9092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-2:19092,PLAINTEXT_HOST://localhost:39092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      controller-1: { condition: service_healthy }
      controller-2: { condition: service_healthy }
      controller-3: { condition: service_healthy }
    networks:
      - kafka
    volumes:
      - kafka2-logs:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 9092 || exit 1
      interval: 60s
      timeout: 5s
      retries: 2
      start_period: 30s
  kafka-3:
    image: apache/kafka:4.2.0
    ports:
      - 49092:9092
    hostname: kafka-3
    container_name: kafka-3
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 6
      KAFKA_PROCESS_ROLES: 'broker'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093'
      KAFKA_LISTENERS: 'PLAINTEXT://:19092,PLAINTEXT_HOST://:9092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-3:19092,PLAINTEXT_HOST://localhost:49092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      controller-1: { condition: service_healthy }
      controller-2: { condition: service_healthy }
      controller-3: { condition: service_healthy }
    networks:
      - kafka
    volumes:
      - kafka3-logs:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 9092 || exit 1
      interval: 60s
      timeout: 5s
      retries: 2
      start_period: 30s
volumes:
  controller-1:
    name: kafka-controller-1
  controller-2:
    name: kafka-controller-2
  controller-3:
    name: kafka-controller-3
  kafka1-logs:
    name: kafka1-logs
  kafka2-logs:
    name: kafka2-logs
  kafka3-logs:
    name: kafka3-logs
networks:
  kafka:
    name: kafka

部署

docker compose up -d

测试
查询容器健康状态和容器日志无明显报错信息。
功能未测试，待更新。

© 版权声明
文章版权归作者所有，未经允许请勿转载。

相关文章

标签云