Skip to content

Commit 5a02d63

Browse files
committed
Merge pull request #193 from dpkp/test_apache_binary_release
Use Kafka Binaries for Integration Tests (Issue #176)
2 parents c37dc89 + 813a0bd commit 5a02d63

18 files changed

+475
-64
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,6 @@ build
55
dist
66
MANIFEST
77
env
8+
servers/*/kafka-bin
9+
.coverage
10+
.noseids

.gitmodules

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +0,0 @@
1-
[submodule "servers/0.8.0/kafka-src"]
2-
path = servers/0.8.0/kafka-src
3-
url = https://github.com/apache/kafka.git
4-
[submodule "servers/0.8.1/kafka-src"]
5-
path = servers/0.8.1/kafka-src
6-
url = https://github.com/apache/kafka.git

.travis.yml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,13 @@ python:
55
- 2.7
66
- pypy
77

8+
env:
9+
-
10+
- KAFKA_VERSION=0.8.0
11+
- KAFKA_VERSION=0.8.1
12+
- KAFKA_VERSION=0.8.1.1
13+
814
before_install:
9-
- git submodule update --init --recursive
1015
- sudo apt-get install libsnappy-dev
1116
- ./build_integration.sh
1217

@@ -19,5 +24,3 @@ install:
1924

2025
script:
2126
- tox -e `./travis_selector.sh $TRAVIS_PYTHON_VERSION`
22-
- KAFKA_VERSION=0.8.0 tox -e `./travis_selector.sh $TRAVIS_PYTHON_VERSION`
23-
- KAFKA_VERSION=0.8.1 tox -e `./travis_selector.sh $TRAVIS_PYTHON_VERSION`

README.md

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -190,32 +190,33 @@ pip install python-snappy
190190
tox
191191
```
192192

193+
## Run a single unit test
194+
```shell
195+
tox -e py27 -- -v --with-id 102
196+
```
197+
193198
## Run the integration tests
194199

195200
The integration tests will actually start up real local Zookeeper
196201
instance and Kafka brokers, and send messages in using the client.
197202

198-
Note that you may want to add this to your global gitignore:
203+
First, get the kafka binaries for integration testing:
199204
```shell
200-
.gradle/
201-
clients/build/
202-
contrib/build/
203-
contrib/hadoop-consumer/build/
204-
contrib/hadoop-producer/build/
205-
core/build/
206-
core/data/
207-
examples/build/
208-
perf/build/
205+
./build_integration.sh
209206
```
210-
211-
First, check out and the Kafka source:
207+
By default, the build_integration.sh script will download binary
208+
distributions for all supported kafka versions.
209+
To test against the latest source build, set KAFKA_VERSION=trunk
210+
and optionally set SCALA_VERSION (defaults to 2.8.0, but 2.10.1 is recommended)
212211
```shell
213-
git submodule update --init
214-
./build_integration.sh
212+
SCALA_VERSION=2.10.1 KAFKA_VERSION=trunk ./build_integration.sh
215213
```
216214

217215
Then run the tests against supported Kafka versions:
218216
```shell
219217
KAFKA_VERSION=0.8.0 tox
220218
KAFKA_VERSION=0.8.1 tox
219+
KAFKA_VERSION=0.8.1.1 tox
220+
KAFKA_VERSION=trunk tox
221221
```
222+

build_integration.sh

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,61 @@
11
#!/bin/bash
22

3-
git submodule update --init
4-
(cd servers/0.8.0/kafka-src && ./sbt update package assembly-package-dependency)
5-
(cd servers/0.8.1/kafka-src && ./gradlew jar)
3+
# Versions available for testing via binary distributions
4+
OFFICIAL_RELEASES="0.8.0 0.8.1 0.8.1.1"
5+
6+
# Useful configuration vars, with sensible defaults
7+
if [ -z "$SCALA_VERSION" ]; then
8+
SCALA_VERSION=2.8.0
9+
fi
10+
11+
# On travis CI, empty KAFKA_VERSION means skip integration tests
12+
# so we dont try to get binaries
13+
# Otherwise it means test all official releases, so we get all of them!
14+
if [ -z "$KAFKA_VERSION" -a -z "$TRAVIS" ]; then
15+
KAFKA_VERSION=$OFFICIAL_RELEASES
16+
fi
17+
18+
# By default look for binary releases at archive.apache.org
19+
if [ -z "$DIST_BASE_URL" ]; then
20+
DIST_BASE_URL="https://archive.apache.org/dist/kafka/"
21+
fi
22+
23+
# When testing against source builds, use this git repo
24+
if [ -z "$KAFKA_SRC_GIT" ]; then
25+
KAFKA_SRC_GIT="https://github.com/apache/kafka.git"
26+
fi
27+
28+
pushd servers
29+
mkdir -p dist
30+
pushd dist
31+
for kafka in $KAFKA_VERSION; do
32+
if [ "$kafka" == "trunk" ]; then
33+
if [ ! -d "$kafka" ]; then
34+
git clone $KAFKA_SRC_GIT $kafka
35+
fi
36+
pushd $kafka
37+
git pull
38+
./gradlew -PscalaVersion=$SCALA_VERSION -Pversion=$kafka releaseTarGz -x signArchives
39+
popd
40+
# Not sure how to construct the .tgz name accurately, so use a wildcard (ugh)
41+
tar xzvf $kafka/core/build/distributions/kafka_*.tgz -C ../$kafka/
42+
rm $kafka/core/build/distributions/kafka_*.tgz
43+
mv ../$kafka/kafka_* ../$kafka/kafka-bin
44+
else
45+
echo "-------------------------------------"
46+
echo "Checking kafka binaries for ${kafka}"
47+
echo
48+
wget -N https://archive.apache.org/dist/kafka/$kafka/kafka_${SCALA_VERSION}-${kafka}.tgz || wget -N https://archive.apache.org/dist/kafka/$kafka/kafka_${SCALA_VERSION}-${kafka}.tar.gz
49+
echo
50+
if [ ! -d "../$kafka/kafka-bin" ]; then
51+
echo "Extracting kafka binaries for ${kafka}"
52+
tar xzvf kafka_${SCALA_VERSION}-${kafka}.t* -C ../$kafka/
53+
mv ../$kafka/kafka_${SCALA_VERSION}-${kafka} ../$kafka/kafka-bin
54+
else
55+
echo "$kafka/kafka-bin directory already exists -- skipping tgz extraction"
56+
fi
57+
fi
58+
echo
59+
done
60+
popd
61+
popd

servers/0.8.0/kafka-src

Lines changed: 0 additions & 1 deletion
This file was deleted.
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# see kafka.server.KafkaConfig for additional details and defaults
16+
17+
############################# Server Basics #############################
18+
19+
# The id of the broker. This must be set to a unique integer for each broker.
20+
broker.id={broker_id}
21+
22+
############################# Socket Server Settings #############################
23+
24+
# The port the socket server listens on
25+
port={port}
26+
27+
# Hostname the broker will bind to. If not set, the server will bind to all interfaces
28+
host.name={host}
29+
30+
# Hostname the broker will advertise to producers and consumers. If not set, it uses the
31+
# value for "host.name" if configured. Otherwise, it will use the value returned from
32+
# java.net.InetAddress.getCanonicalHostName().
33+
#advertised.host.name=<hostname routable by clients>
34+
35+
# The port to publish to ZooKeeper for clients to use. If this is not set,
36+
# it will publish the same port that the broker binds to.
37+
#advertised.port=<port accessible by clients>
38+
39+
# The number of threads handling network requests
40+
num.network.threads=2
41+
42+
# The number of threads doing disk I/O
43+
num.io.threads=8
44+
45+
# The send buffer (SO_SNDBUF) used by the socket server
46+
socket.send.buffer.bytes=1048576
47+
48+
# The receive buffer (SO_RCVBUF) used by the socket server
49+
socket.receive.buffer.bytes=1048576
50+
51+
# The maximum size of a request that the socket server will accept (protection against OOM)
52+
socket.request.max.bytes=104857600
53+
54+
55+
############################# Log Basics #############################
56+
57+
# A comma seperated list of directories under which to store log files
58+
log.dirs={tmp_dir}/data
59+
60+
# The default number of log partitions per topic. More partitions allow greater
61+
# parallelism for consumption, but this will also result in more files across
62+
# the brokers.
63+
num.partitions={partitions}
64+
default.replication.factor={replicas}
65+
66+
############################# Log Flush Policy #############################
67+
68+
# Messages are immediately written to the filesystem but by default we only fsync() to sync
69+
# the OS cache lazily. The following configurations control the flush of data to disk.
70+
# There are a few important trade-offs here:
71+
# 1. Durability: Unflushed data may be lost if you are not using replication.
72+
# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
73+
# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
74+
# The settings below allow one to configure the flush policy to flush data after a period of time or
75+
# every N messages (or both). This can be done globally and overridden on a per-topic basis.
76+
77+
# The number of messages to accept before forcing a flush of data to disk
78+
#log.flush.interval.messages=10000
79+
80+
# The maximum amount of time a message can sit in a log before we force a flush
81+
#log.flush.interval.ms=1000
82+
83+
############################# Log Retention Policy #############################
84+
85+
# The following configurations control the disposal of log segments. The policy can
86+
# be set to delete segments after a period of time, or after a given size has accumulated.
87+
# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
88+
# from the end of the log.
89+
90+
# The minimum age of a log file to be eligible for deletion
91+
log.retention.hours=168
92+
93+
# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
94+
# segments don't drop below log.retention.bytes.
95+
#log.retention.bytes=1073741824
96+
97+
# The maximum size of a log segment file. When this size is reached a new log segment will be created.
98+
log.segment.bytes=536870912
99+
100+
# The interval at which log segments are checked to see if they can be deleted according
101+
# to the retention policies
102+
log.retention.check.interval.ms=60000
103+
104+
# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires.
105+
# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
106+
log.cleaner.enable=false
107+
108+
############################# Zookeeper #############################
109+
110+
# Zookeeper connection string (see zookeeper docs for details).
111+
# This is a comma separated host:port pairs, each corresponding to a zk
112+
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
113+
# You can also append an optional chroot string to the urls to specify the
114+
# root directory for all kafka znodes.
115+
zookeeper.connect={zk_host}:{zk_port}/{zk_chroot}
116+
117+
# Timeout in ms for connecting to zookeeper
118+
zookeeper.connection.timeout.ms=1000000
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
log4j.rootLogger=INFO, stdout
17+
18+
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
19+
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
20+
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
21+
22+
log4j.logger.kafka=DEBUG, stdout
23+
log4j.logger.org.I0Itec.zkclient.ZkClient=INFO, stdout
24+
log4j.logger.org.apache.zookeeper=INFO, stdout
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# the directory where the snapshot is stored.
16+
dataDir={tmp_dir}
17+
# the port at which the clients will connect
18+
clientPort={port}
19+
clientPortAddress={host}
20+
# disable the per-ip limit on the number of connections since this is a non-production config
21+
maxClientCnxns=0

servers/0.8.1/kafka-src

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)