Skip to content

chore: test against kafka 3.7 and remove old versions of kafka from CI #1381

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
May 6, 2025
138 changes: 15 additions & 123 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,82 +68,11 @@ jobs:
working_directory: ./sasl/aws_msk_iam
command: go test -race -cover ./...

kafka-011:
working_directory: *working_directory
environment:
KAFKA_VERSION: "0.11.0"
docker:
- image: circleci/golang
- image: bitnami/zookeeper:latest
ports:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:0.11.0-1-r1
ports:
- 9092:9092
- 9093:9093
environment:
KAFKA_BROKER_ID: 1
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_ADVERTISED_HOST_NAME: 'localhost'
KAFKA_ADVERTISED_PORT: '9092'
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
KAFKA_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_server_jaas.conf"
ALLOW_PLAINTEXT_LISTENER: "yes"
entrypoint:
- "/bin/bash"
- "-c"
# 0.11.0 image is not honoring some configs required in server.properties
- echo -e '\nsasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\nmessage.max.bytes=200000000\nauto.create.topics.enable=true\nport=9092' >> /opt/bitnami/kafka/config/server.properties; echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/bitnami/kafka/config/kafka_server_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config 'SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]' --entity-type users --entity-name adminscram; exec /app-entrypoint.sh /run.sh
steps: *steps

kafka-231:
working_directory: *working_directory
environment:
KAFKA_VERSION: "2.3.1"
docker:
- image: circleci/golang
- image: bitnami/zookeeper:latest
ports:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:2.3.1-ol-7-r61
ports:
- 9092:9092
- 9093:9093
environment:
KAFKA_CFG_BROKER_ID: 1
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'
KAFKA_CFG_ADVERTISED_PORT: '9092'
KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'
KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'
KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
KAFKA_INTER_BROKER_USER: adminplain
KAFKA_INTER_BROKER_PASSWORD: admin-secret
KAFKA_BROKER_USER: adminplain
KAFKA_BROKER_PASSWORD: admin-secret
ALLOW_PLAINTEXT_LISTENER: yes
entrypoint:
- "/bin/bash"
- "-c"
- /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
steps: *steps

kafka-241:
# NOTE: Bitnami does not have suport for kafka version 2.7.1. Hence we use 2.7.0
kafka-270:
working_directory: *working_directory
environment:
KAFKA_VERSION: "2.4.1"
KAFKA_VERSION: "2.7.0"

# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
Expand All @@ -162,7 +91,7 @@ jobs:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:2.4.1
- image: bitnami/kafka:2.7.0
ports:
- 9092:9092
- 9093:9093
Expand All @@ -187,10 +116,10 @@ jobs:
- echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
steps: *steps

kafka-260:
kafka-281:
working_directory: *working_directory
environment:
KAFKA_VERSION: "2.6.0"
KAFKA_VERSION: "2.8.1"

# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
Expand All @@ -209,19 +138,18 @@ jobs:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:2.6.0
- image: bitnami/kafka:2.8.1
ports:
- 9092:9092
- 9093:9093
environment: *environment
entrypoint: *entrypoint
steps: *steps

# NOTE: Bitnami does not have suport for kafka version 2.7.1. Hence we use 2.7.0
kafka-270:
kafka-370:
working_directory: *working_directory
environment:
KAFKA_VERSION: "2.7.0"
KAFKA_VERSION: "3.7.0"

# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
Expand All @@ -240,41 +168,13 @@ jobs:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:2.7.0
ports:
- 9092:9092
- 9093:9093
environment: *environment
entrypoint: *entrypoint
steps: *steps

kafka-281:
working_directory: *working_directory
environment:
KAFKA_VERSION: "2.8.1"

# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
# --- FAIL: TestConn/nettest/PingPong (7.40s)
# conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request
# conntest.go:118: mismatching value: got 77, want 78
# conntest.go:118: mismatching value: got 78, want 79
# ...
#
# TODO: Figure out why these are happening and fix them (they don't appear to be new).
KAFKA_SKIP_NETTEST: "1"
docker:
- image: circleci/golang
- image: bitnami/zookeeper:latest
- image: bitnami/kafka:3.7.0
ports:
- 2181:2181
- 9092:9092
- 9093:9093
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:2.8.1
ports:
- 9092:9092
- 9093:9093
environment: *environment
<<: *environment
KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
entrypoint: *entrypoint
steps: *steps

Expand All @@ -284,14 +184,6 @@ workflows:
jobs:
- lint
- kafka-010
- kafka-011
# - kafka-101 # Bitnami image fails for SCRAM. refer kafka-do/docker_compose_versions/README.md
# - kafka-111 # Bitnami image fails for SCRAM. refer kafka-do/docker_compose_versions/README.md
# - kafka-201 # Bitnami image fails for SCRAM. refer kafka-do/docker_compose_versions/README.md
# - kafka-211 # Bitnami image fails for SCRAM. refer kafka-do/docker_compose_versions/README.md
# - kafka-222 # Bitnami v222 not found. v221 Bitnami image fails for SCRAM. refer kafka-do/docker_compose_versions/README.md
- kafka-231
- kafka-241
- kafka-260
- kafka-270
- kafka-281
- kafka-370
15 changes: 10 additions & 5 deletions addoffsetstotxn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package kafka
import (
"context"
"log"
"net"
"os"
"strconv"
"testing"
"time"

Expand All @@ -16,6 +14,13 @@ func TestClientAddOffsetsToTxn(t *testing.T) {
if !ktesting.KafkaIsAtLeast("0.11.0") {
t.Skip("Skipping test because kafka version is not high enough.")
}

// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support
// work is revisited.
if ktesting.KafkaIsAtLeast("3.0.0") {
t.Skip("Skipping test because it fails on Kafka version 3.0.0 or higher.")
}

topic := makeTopic()
transactionalID := makeTransactionalID()
client, shutdown := newLocalClient()
Expand Down Expand Up @@ -77,9 +82,9 @@ func TestClientAddOffsetsToTxn(t *testing.T) {
t.Fatal(err)
}

transactionCoordinator := TCP(net.JoinHostPort(respc.Coordinator.Host, strconv.Itoa(int(respc.Coordinator.Port))))
client, shutdown = newClient(transactionCoordinator)
defer shutdown()
if respc.Error != nil {
t.Fatal(err)
}

ipResp, err := client.InitProducerID(ctx, &InitProducerIDRequest{
TransactionalID: transactionalID,
Expand Down
7 changes: 7 additions & 0 deletions addpartitionstotxn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ func TestClientAddPartitionsToTxn(t *testing.T) {
if !ktesting.KafkaIsAtLeast("0.11.0") {
t.Skip("Skipping test because kafka version is not high enough.")
}

// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support
// work is revisited.
if ktesting.KafkaIsAtLeast("3.0.0") {
t.Skip("Skipping test because it fails on Kafka version 3.0.0 or higher.")
}

topic1 := makeTopic()
topic2 := makeTopic()

Expand Down
11 changes: 4 additions & 7 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ services:
ALLOW_ANONYMOUS_LOGIN: yes
kafka:
container_name: kafka
image: bitnami/kafka:2.3.1-ol-7-r61
image: bitnami/kafka:3.7.0
restart: on-failure:3
links:
- zookeeper
Expand All @@ -29,14 +29,11 @@ services:
KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'
KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'
KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
KAFKA_INTER_BROKER_USER: adminplain
KAFKA_INTER_BROKER_PASSWORD: admin-secret
KAFKA_BROKER_USER: adminplain
KAFKA_BROKER_PASSWORD: admin-secret
KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf"
ALLOW_PLAINTEXT_LISTENER: yes
entrypoint:
- "/bin/bash"
- "-c"
- /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
- echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
36 changes: 0 additions & 36 deletions docker_compose_versions/docker-compose-011.yml

This file was deleted.

42 changes: 0 additions & 42 deletions docker_compose_versions/docker-compose-231.yml

This file was deleted.

39 changes: 0 additions & 39 deletions docker_compose_versions/docker-compose-241.yml

This file was deleted.

Loading