From a84cdcb1442b99cc9f20ed9ba5f16ac9a52dde88 Mon Sep 17 00:00:00 2001 From: Jeff Widman Date: Fri, 26 Oct 2018 22:37:01 -0700 Subject: [PATCH 01/18] Add python 3.7 support Add Python 3.7 to the tests. Note that Travis requires a workaround for now. Document 3.7 support on PyPi. --- .travis.yml | 13 +++++++++++++ Makefile | 6 +++--- docs/compatibility.rst | 2 +- setup.py | 1 + tox.ini | 6 +++--- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd6018b5a..917de1bd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,21 @@ python: - 3.4 - 3.5 - 3.6 + # - 3.7 # TODO uncomment once Travis supports 3.7 w/o the `matrix` workaround used below - pypy +# TODO: workaround pulled from: +# https://github.com/travis-ci/travis-ci/issues/9815#issue-336465122 +# It only runs a single 3.7 test job against one version of Kafka but +# that's good enough for now since we test all the other permutations, +# especially since this workaround should be removed in the near future. +# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true + env: - KAFKA_VERSION=0.8.2.2 - KAFKA_VERSION=0.9.0.1 diff --git a/Makefile b/Makefile index 7dfd305e6..b4dcbffc9 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,8 @@ servers/$(KAFKA_VERSION)/kafka-bin: build-integration: servers/$(KAFKA_VERSION)/kafka-bin # Test and produce coverage using tox. This is the same as is run on Travis -test36: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py36 -- $(FLAGS) +test37: build-integration + KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py37 -- $(FLAGS) test27: build-integration KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py27 -- $(FLAGS) @@ -56,4 +56,4 @@ doc: make -C docs html @echo "open file://`pwd`/docs/_build/html/index.html" -.PHONY: all test36 test27 test-local cov-local clean doc +.PHONY: all test37 test27 test-local cov-local clean doc diff --git a/docs/compatibility.rst b/docs/compatibility.rst index 85b2c59db..ce222ee63 100644 --- a/docs/compatibility.rst +++ b/docs/compatibility.rst @@ -12,6 +12,6 @@ through 0.8.0 . kafka-python is not compatible with the 0.8.2-beta release. Because the kafka server protocol is backwards compatible, kafka-python is expected to work with newer broker releases as well (2.0+). -kafka-python is tested on python 2.7, 3.4, 3.5, 3.6 and pypy. +kafka-python is tested on python 2.7, 3.4, 3.5, 3.6, 3.7, and pypy. Builds and tests via Travis-CI. See https://travis-ci.org/dpkp/kafka-python diff --git a/setup.py b/setup.py index 9dcdb8eaa..779adb92b 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ def run(cls): "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules", ] diff --git a/tox.ini b/tox.ini index 1da88f304..48a143eea 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{26,27,34,35,36,py}, docs +envlist = py{26,27,34,35,36,37,py}, docs [pytest] testpaths = kafka test @@ -11,8 +11,8 @@ log_format = %(created)f %(filename)-23s %(threadName)s %(message)s deps = pytest<4.0 pytest-cov - py{27,34,35,36,py}: pylint - py{27,34,35,36,py}: pytest-pylint + py{27,34,35,36,37,py}: pylint + py{27,34,35,36,37,py}: pytest-pylint pytest-mock mock python-snappy From 672941dbeb508be5bd02c3e168acd04f166d66f0 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 10:34:37 -0800 Subject: [PATCH 02/18] Use xenial dist for travis builds --- .travis.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 917de1bd8..616371083 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) +sudo: required # required for Python >= 3.7 (travis-ci/travis-ci#9069) + language: python python: @@ -5,21 +8,9 @@ python: - 3.4 - 3.5 - 3.6 - # - 3.7 # TODO uncomment once Travis supports 3.7 w/o the `matrix` workaround used below + - 3.7 - pypy -# TODO: workaround pulled from: -# https://github.com/travis-ci/travis-ci/issues/9815#issue-336465122 -# It only runs a single 3.7 test job against one version of Kafka but -# that's good enough for now since we test all the other permutations, -# especially since this workaround should be removed in the near future. -# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true - env: - KAFKA_VERSION=0.8.2.2 - KAFKA_VERSION=0.9.0.1 From 59a9e6271fe6eec9ac740fd1e9ac1b205bfb8f34 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 11:10:31 -0800 Subject: [PATCH 03/18] Pin travis tests to openjdk8 --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 616371083..8f7a12834 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,12 @@ sudo: required # required for Python >= 3.7 (travis-ci/travis-ci#9069) language: python +jdk: + - openjdk8 + python: - 2.7 - 3.4 - - 3.5 - - 3.6 - 3.7 - pypy From 9a7844023123e6832e50bf05bb104b88454a6d72 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 12:09:01 -0800 Subject: [PATCH 04/18] setting jdk on non-java project does not work... also drop sudo and improve tox variable handling --- .travis.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f7a12834..1450ac35b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,7 @@ -dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) -sudo: required # required for Python >= 3.7 (travis-ci/travis-ci#9069) +dist: xenial language: python -jdk: - - openjdk8 - python: - 2.7 - 3.4 @@ -23,6 +19,7 @@ addons: apt: packages: - libsnappy-dev + - oracle-java8-set-default cache: directories: @@ -37,7 +34,7 @@ install: - pip install . script: - - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` + - tox -e py${TRAVIS_PYTHON_VERSION#py} after_success: - coveralls From 15babe0ee0498db9609ecf2b5ee7e40aba025757 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 12:13:18 -0800 Subject: [PATCH 05/18] xenial needs explicit 3rd party apt source --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1450ac35b..adcb3c4a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ env: addons: apt: + sources: + - ppa:webupd8team/java packages: - libsnappy-dev - oracle-java8-set-default From b79a0fd4745b5f4cee5af26db5c735ec256a7df9 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 12:19:50 -0800 Subject: [PATCH 06/18] Try openjdk8? --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index adcb3c4a2..647c49636 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,9 @@ env: addons: apt: - sources: - - ppa:webupd8team/java packages: - libsnappy-dev - - oracle-java8-set-default + - openjdk8 cache: directories: From d20ee30bec777f58b1a7c168a75e948b70c98f5f Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 17:50:00 -0800 Subject: [PATCH 07/18] Perhaps no xenial? --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 647c49636..e5071e4d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,3 @@ -dist: xenial - language: python python: @@ -19,7 +17,6 @@ addons: apt: packages: - libsnappy-dev - - openjdk8 cache: directories: From 38c7d086bdbf4d6f54d4c980d27e4b6a277f4b21 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Sun, 13 Jan 2019 18:47:21 -0800 Subject: [PATCH 08/18] Revert TRAVIS_PYTHON_VERSION change --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e5071e4d2..a5b7b133c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ install: - pip install . script: - - tox -e py${TRAVIS_PYTHON_VERSION#py} + - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` after_success: - coveralls From 0a73b186c1ffadf8d54a38da96cdc3c02a9ef322 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 19:35:46 -0700 Subject: [PATCH 09/18] Try travis_java_install script from pywrangler --- .travis.yml | 1 + travis_java_install.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 travis_java_install.sh diff --git a/.travis.yml b/.travis.yml index a5b7b133c..0a87e7e46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ cache: - servers/ before_install: + - source travis_java_install.sh - ./build_integration.sh install: diff --git a/travis_java_install.sh b/travis_java_install.sh new file mode 100644 index 000000000..cc930a16b --- /dev/null +++ b/travis_java_install.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# borrowed from: https://github.com/mansenfranzen/pywrangler/blob/master/tests/travis_java_install.sh + +# Kafka requires Java 8 in order to work properly. However, TravisCI's Ubuntu +# 16.04 ships with Java 11 and Java can't be set with `jdk` when python is +# selected as language. Ubuntu 14.04 does not work due to missing python 3.7 +# support on TravisCI which does have Java 8 as default. + +# show current JAVA_HOME and java version +echo "Current JAVA_HOME: $JAVA_HOME" +echo "Current java -version:" +java -version + +# install Java 8 +sudo add-apt-repository -y ppa:openjdk-r/ppa +sudo apt-get -qq update +sudo apt-get install -y openjdk-8-jdk --no-install-recommends +sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 + +# change JAVA_HOME to Java 8 +export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 From 82912d88a86f8d9636e130bc5d40f686818e8517 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 19:56:23 -0700 Subject: [PATCH 10/18] Use xenial --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0a87e7e46..7e00782d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python +dist: xenial + python: - 2.7 - 3.4 From 59f1e1251569ac06a81183137a42566fa16c043a Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 20:19:58 -0700 Subject: [PATCH 11/18] Try install openjdk-8-jdk via travis.yml apt packages --- .travis.yml | 1 + travis_java_install.sh | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e00782d1..06c964fb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ addons: apt: packages: - libsnappy-dev + - openjdk-8-jdk cache: directories: diff --git a/travis_java_install.sh b/travis_java_install.sh index cc930a16b..3c53f9ec4 100644 --- a/travis_java_install.sh +++ b/travis_java_install.sh @@ -13,10 +13,19 @@ echo "Current java -version:" java -version # install Java 8 -sudo add-apt-repository -y ppa:openjdk-r/ppa -sudo apt-get -qq update -sudo apt-get install -y openjdk-8-jdk --no-install-recommends +#sudo add-apt-repository -y ppa:openjdk-r/ppa +#sudo apt-get -qq update +#sudo apt-get install -y openjdk-8-jdk --no-install-recommends + +echo "Updating java alternatives..." sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 +echo "New java -version" +java -version + +echo "Updating JAVA_HOME" # change JAVA_HOME to Java 8 export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + +echo "New new java -version" +java -version From 331ed0ece88640b391daf95820878107d2340681 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 20:27:34 -0700 Subject: [PATCH 12/18] list available java alternatives b/c this isnt working... --- travis_java_install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/travis_java_install.sh b/travis_java_install.sh index 3c53f9ec4..c4c3e67ee 100644 --- a/travis_java_install.sh +++ b/travis_java_install.sh @@ -18,8 +18,13 @@ java -version #sudo apt-get install -y openjdk-8-jdk --no-install-recommends echo "Updating java alternatives..." +sudo update-java-alternatives -l +sudo update-java-alternatives --help sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 +echo "update-alternatives --config java" +sudo update-alternatives --config java + echo "New java -version" java -version From 872b0fd69b241c40b4ad8a6d1ca9f56deaa316e5 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 20:35:28 -0700 Subject: [PATCH 13/18] Remove update-alternatives --config java; fix JAVA_HOME --- travis_java_install.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/travis_java_install.sh b/travis_java_install.sh index c4c3e67ee..ed278512e 100644 --- a/travis_java_install.sh +++ b/travis_java_install.sh @@ -22,15 +22,14 @@ sudo update-java-alternatives -l sudo update-java-alternatives --help sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 -echo "update-alternatives --config java" -sudo update-alternatives --config java - echo "New java -version" +which java java -version +echo $JAVA_HOME echo "Updating JAVA_HOME" # change JAVA_HOME to Java 8 -export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 echo "New new java -version" java -version From 7dbc583a5ffa97ebac9a7295349828fb94f9621c Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 20:54:40 -0700 Subject: [PATCH 14/18] Fix pypy travis install for xenial --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 06c964fb1..54a49d473 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - 2.7 - 3.4 - 3.7 - - pypy + - pypy2.7 env: - KAFKA_VERSION=0.8.2.2 @@ -35,7 +35,7 @@ install: - pip install . script: - - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` + - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy2.7" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` after_success: - coveralls From 15ba47495945bd3ec62b2ea7f88808447141f8eb Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 20:55:53 -0700 Subject: [PATCH 15/18] Maybe fixup PATH? --- travis_java_install.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/travis_java_install.sh b/travis_java_install.sh index ed278512e..774766142 100644 --- a/travis_java_install.sh +++ b/travis_java_install.sh @@ -33,3 +33,12 @@ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 echo "New new java -version" java -version + +echo "Updating PATH" +echo $PATH +export PATH=$JAVA_HOME:$PATH +echo $PATH + +echo "New new new java -version" +which java +java -version From ec7464d9da65f01faaf9e84f79d42834957e3979 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 21:03:30 -0700 Subject: [PATCH 16/18] replace java entry in PATH --- travis_java_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis_java_install.sh b/travis_java_install.sh index 774766142..aac45eeb1 100644 --- a/travis_java_install.sh +++ b/travis_java_install.sh @@ -36,7 +36,7 @@ java -version echo "Updating PATH" echo $PATH -export PATH=$JAVA_HOME:$PATH +export PATH=${PATH/\/usr\/local\/lib\/jvm\/openjdk11\/bin/$JAVA_HOME\/bin} echo $PATH echo "New new new java -version" From a83c4d07284f913a25d8a9538a332cec69a881ec Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 21:08:24 -0700 Subject: [PATCH 17/18] Maybe pypy2.7-6.0 ? --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54a49d473..cdb93396d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - 2.7 - 3.4 - 3.7 - - pypy2.7 + - pypy2.7-6.0 env: - KAFKA_VERSION=0.8.2.2 @@ -35,7 +35,7 @@ install: - pip install . script: - - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy2.7" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` + - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy2.7-6.0" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` after_success: - coveralls From b972a10fef2670ac4e46c92b03692eb031679ec0 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Tue, 12 Mar 2019 21:14:36 -0700 Subject: [PATCH 18/18] cleanup java install script --- travis_java_install.sh | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/travis_java_install.sh b/travis_java_install.sh index aac45eeb1..f662ce274 100644 --- a/travis_java_install.sh +++ b/travis_java_install.sh @@ -10,35 +10,16 @@ # show current JAVA_HOME and java version echo "Current JAVA_HOME: $JAVA_HOME" echo "Current java -version:" -java -version - -# install Java 8 -#sudo add-apt-repository -y ppa:openjdk-r/ppa -#sudo apt-get -qq update -#sudo apt-get install -y openjdk-8-jdk --no-install-recommends - -echo "Updating java alternatives..." -sudo update-java-alternatives -l -sudo update-java-alternatives --help -sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 - -echo "New java -version" which java java -version -echo $JAVA_HOME echo "Updating JAVA_HOME" # change JAVA_HOME to Java 8 export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 -echo "New new java -version" -java -version - echo "Updating PATH" -echo $PATH export PATH=${PATH/\/usr\/local\/lib\/jvm\/openjdk11\/bin/$JAVA_HOME\/bin} -echo $PATH -echo "New new new java -version" +echo "New java -version" which java java -version