publishing (apache-spark-on-k8s#25)

Author: robert3005

.gitignore

@@ -67,6 +67,7 @@ streaming-tests.log
 target/
 unit-tests.log
 work/
+pr-deps/
 
 # For Hive
 TempStatsStore/

build/mvn

@@ -36,15 +36,8 @@ install_app() {
   local binary="${_DIR}/$3"
 
   # setup `curl` and `wget` silent options if we're running on Jenkins
-  local curl_opts="-L"
-  local wget_opts=""
-  if [ -n "$AMPLAB_JENKINS" ]; then
-    curl_opts="-s ${curl_opts}"
-    wget_opts="--quiet ${wget_opts}"
-  else
-    curl_opts="--progress-bar ${curl_opts}"
-    wget_opts="--progress=bar:force ${wget_opts}"
-  fi
+  local curl_opts="-s -L"
+  local wget_opts="--quiet"
 
   if [ -z "$3" -o ! -f "$binary" ]; then
     # check if we already have the tarball
@@ -85,7 +78,10 @@ install_mvn() {
       "apache-maven-${MVN_VERSION}-bin.tar.gz" \
       "apache-maven-${MVN_VERSION}/bin/mvn"
 
-    MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn"
+    export M2_HOME="${_DIR}/apache-maven-${MVN_VERSION}"
+    export M2="$M2_HOME/bin"
+    export PATH=$M2:$PATH
+    MVN_BIN="$M2/mvn"
   fi
 }
 
@@ -158,4 +154,4 @@ export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
 echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
 
 # Last, call the `mvn` command as usual
-${MVN_BIN} -DzincPort=${ZINC_PORT} "$@"
+${MVN_BIN} -DzincPort=${ZINC_PORT} -e "$@"

circle.yml

@@ -0,0 +1,58 @@
+general:
+  artifacts:
+    - target
+
+machine:
+  pre:
+    - sudo pkill -u postgres
+  java:
+    version: oraclejdk8
+  environment:
+    TERM: dumb
+    HADOOP_PROFILE: "hadoop2.7"
+  post:
+    - sudo pkill mongod || true
+    - sudo pkill mysqld || true
+    - sudo pkill postgres || true
+
+checkout:
+  post:
+    - "[[ ! -s \"$(git rev-parse --git-dir)/shallow\" ]] || git fetch --unshallow"
+    - echo "user=$BINTRAY_USER" > .credentials
+    - echo "password=$BINTRAY_PASSWORD" >> .credentials
+    - echo "realm=Bintray API Realm" >> .credentials
+    - echo "host=api.bintray.com" >> .credentials
+
+dependencies:
+  override:
+    - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive update
+
+test:
+  override:
+    - /bin/true
+    #- ./dev/run-tests -p 1
+
+deployment:
+  release:
+    tag: /[0-9]+(?:\.[0-9]+)+-palantir[0-9]+/
+    commands:
+	  - ? |
+			set -euo pipefail
+			FLAGS="-Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos"
+			case $CIRCLE_NODE_INDEX in
+			0)
+			  ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive publish
+			  ;;
+			1)
+			  ./dev/make-distribution.sh --name without-hadoop --tgz "-Psparkr -Phadoop-provided -Pyarn -Pmesos" \
+				  2>&1 >  binary-release-without-hadoop.log
+			  ;;
+			2)
+			  ./dev/make-distribution.sh --name hadoop2.7 --tgz "-Phadoop2.7 $FLAGS" \
+				  2>&1 >  binary-release-hadoop2.7.log
+			  ;;
+			esac
+		:
+		  parallel: true
+		  timeout: 1200
+

dev/.rat-excludes

@@ -102,3 +102,5 @@ org.apache.spark.scheduler.ExternalClusterManager
 .Rbuildignore
 org.apache.spark.deploy.yarn.security.ServiceCredentialProvider
 spark-warehouse
+circle.yml
+.credentials

dev/check-license

@@ -40,7 +40,7 @@ acquire_rat_jar () {
   fi
 
   unzip -tq "$JAR" &> /dev/null
-  if [ $? -ne 0 ]; then 
+  if [ $? -ne 0 ]; then
     # We failed to download
     rm "$JAR"
     printf "Our attempt to download rat locally to ${JAR} failed. Please install rat manually.\n"
@@ -67,7 +67,7 @@ mkdir -p "$FWDIR"/lib
     exit 1
 }
 
-mkdir target
+mkdir -p target
 $java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > target/rat-results.txt
 
 if [ $? -ne 0 ]; then
@@ -77,10 +77,10 @@ fi
 
 ERRORS="$(cat target/rat-results.txt | grep -e "??")"
 
-if test ! -z "$ERRORS"; then 
+if test ! -z "$ERRORS"; then
     echo "Could not find Apache license headers in the following files:"
     echo "$ERRORS"
     exit 1
-else 
+else
     echo -e "RAT checks passed."
 fi

dev/test-dependencies.sh

@@ -32,10 +32,6 @@ export LC_ALL=C
 HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pyarn -Phive"
 MVN="build/mvn"
 HADOOP_PROFILES=(
-    hadoop-2.2
-    hadoop-2.3
-    hadoop-2.4
-    hadoop-2.6
     hadoop-2.7
 )
 

project/SparkBuild.scala

@@ -23,12 +23,15 @@ import scala.util.Properties
 import scala.collection.JavaConverters._
 import scala.collection.mutable.Stack
 
+import bintray.BintrayKeys
+import bintray.BintrayPlugin._
 import sbt._
 import sbt.Classpaths.publishTask
 import sbt.Keys._
 import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
 import com.simplytyped.Antlr4Plugin._
 import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
+import com.typesafe.sbt.SbtGit._
 import com.typesafe.tools.mima.plugin.MimaKeys
 import org.scalastyle.sbt.ScalastylePlugin._
 import org.scalastyle.sbt.Tasks
@@ -233,7 +236,7 @@ object SparkBuild extends PomBuild {
     }
   )
 
-  lazy val sharedSettings = sparkGenjavadocSettings ++
+  lazy val sharedSettings = bintraySettings ++ versionWithGit ++ sparkGenjavadocSettings ++
       (if (sys.env.contains("NOLINT_ON_COMPILE")) Nil else enableScalaStyle) ++ Seq(
     exportJars in Compile := true,
     exportJars in Test := false,
@@ -243,6 +246,29 @@ object SparkBuild extends PomBuild {
     incOptions := incOptions.value.withNameHashing(true),
     publishMavenStyle := true,
     unidocGenjavadocVersion := "0.10",
+    git.useGitDescribe := true,
+    useJGit,
+    version := {
+      val uncommittedSuffix =
+        git.makeUncommittedSignifierSuffix(git.gitUncommittedChanges.value, git.uncommittedSignifier.value)
+      val releaseVersion =
+        git.releaseVersion(git.gitCurrentTags.value, git.gitTagToVersionNumber.value, uncommittedSuffix)
+      val describedVersion =
+        git.flaggedOptional(git.useGitDescribe.value, git.describeVersion(git.gitDescribedVersion.value, uncommittedSuffix))
+      val commitVersion = git.formattedShaVersion.value
+      //Now we fall through the potential version numbers...
+      git.makeVersion(Seq(
+        releaseVersion,
+        describedVersion,
+        commitVersion
+      )) get
+    },
+    BintrayKeys.bintrayCredentialsFile := new File(".credentials"),
+    licenses += ("Apache 2.0", url("https://www.apache.org/licenses/LICENSE-2.0")),
+    BintrayKeys.bintrayOrganization := Some("palantir"),
+    BintrayKeys.bintrayRepository := "releases",
+    BintrayKeys.bintrayVcsUrl := Some("https://github.com/palantir/parquet-mr"),
+    concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
 
     // Override SBT's default resolvers:
     resolvers := Seq(

project/plugins.sbt

@@ -1,3 +1,7 @@
+addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5")
+
+addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0")
+
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")
 
 addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0")