From dbf26be7d579c1806e5862adaf19553e61e1d9c0 Mon Sep 17 00:00:00 2001 From: jkbradley Date: Wed, 17 May 2017 17:21:11 -0700 Subject: [PATCH 1/7] Update release number for 0.5 release (#199) * updated release numbers to 0.5.0 * Update travis branch to track latest Spark 2.1 release version (#198) * Update travis branch to track latest Spark 2.1 release version * update Makefile and build.sbt default --- .travis.yml | 4 ++-- Makefile | 8 ++++---- README.md | 12 +++++++++--- build.sbt | 4 ++-- docs/_config.yml | 2 +- docs/quick-start.md | 4 ++-- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e4c08ff8..73da185c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,10 @@ env: matrix: - PYSPARK_PYTHON=python2 SCALA_VERSION=2.10.6 SPARK_VERSION=1.6.3 SPARK_BUILD="spark-1.6.3-bin-hadoop2.6" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz" - PYSPARK_PYTHON=python2 SCALA_VERSION=2.11.8 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz" - - PYSPARK_PYTHON=python2 SCALA_VERSION=2.11.8 SPARK_VERSION=2.1.0 SPARK_BUILD="spark-2.1.0-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz" + - PYSPARK_PYTHON=python2 SCALA_VERSION=2.11.8 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz" - PYSPARK_PYTHON=python3 SCALA_VERSION=2.10.6 SPARK_VERSION=1.6.3 SPARK_BUILD="spark-1.6.3-bin-hadoop2.6" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz" - PYSPARK_PYTHON=python3 SCALA_VERSION=2.11.8 SPARK_VERSION=2.0.2 SPARK_BUILD="spark-2.0.2-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz" - - PYSPARK_PYTHON=python3 SCALA_VERSION=2.11.8 SPARK_VERSION=2.1.0 SPARK_BUILD="spark-2.1.0-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz" + - PYSPARK_PYTHON=python3 SCALA_VERSION=2.11.8 SPARK_VERSION=2.1.1 SPARK_BUILD="spark-2.1.1-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz" before_install: - ./bin/download_travis_dependencies.sh diff --git a/Makefile b/Makefile index 39f905b78..dd2c8c067 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -all: 2.1.0s2.10 1.6.3 2.0.2 2.1.0 +all: 2.1.1s2.10 1.6.3 2.0.2 2.1.1 clean: rm -rf target/graphframes_*.zip -1.6.3 2.0.2 2.1.0: +1.6.3 2.0.2 2.1.1: build/sbt -Dspark.version=$@ spDist -2.1.0s2.10: - build/sbt -Dspark.version=2.1.0 -Dscala.version=2.10.6 spDist assembly test +2.1.1s2.10: + build/sbt -Dspark.version=2.1.1 -Dscala.version=2.10.6 spDist assembly test diff --git a/README.md b/README.md index 2aaf1b410..344a72e60 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,16 @@ We welcome open source contributions as well! ## Releases: - 0.1.0 initial release -- 0.2.0 release for Spark 2.0 (work of @felixcheung) -- 0.3.0 +- 0.2.0 release + - Spark 2.0 support (work of @felixcheung) +- 0.3.0 release - DataFrame-based connected components implementation - added support for Python 3 - removed support for Spark 1.4 and 1.5 -- 0.4.0 release for Spark 2.1 +- 0.4.0 release + - Spark 2.1 support - Fix for checkpointing issue in DataFrame-based connected components implementation (issue 160) +- 0.5.0 release + - Major bug fix: Indexing non-Integer vertex IDs, which is used by algorithms which call GraphX + under the hood, including PageRank, ConnectedComponents, and others. + - aggregateMessages for Python API diff --git a/build.sbt b/build.sbt index 39e9eb25e..d24905feb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ // Your sbt build file. Guides on how to write one can be found at // http://www.scala-sbt.org/0.13/docs/index.html -val sparkVer = sys.props.getOrElse("spark.version", "2.1.0") +val sparkVer = sys.props.getOrElse("spark.version", "2.1.1") val sparkBranch = sparkVer.substring(0, 3) val defaultScalaVer = sparkBranch match { case "1.6" => "2.10.6" @@ -23,7 +23,7 @@ scalaVersion := scalaVer spName := "graphframes/graphframes" // Don't forget to set the version -version := s"0.4.0-SNAPSHOT-spark$sparkBranch" +version := s"0.5.0-spark$sparkBranch" // All Spark Packages need a license licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")) diff --git a/docs/_config.yml b/docs/_config.yml index 5a13e1e5d..6bc87f28d 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -14,7 +14,7 @@ include: # These allow the documentation to be updated with newer releases # of Spark, Scala, and Mesos. -GRAPHFRAMES_VERSION: 0.4.0 +GRAPHFRAMES_VERSION: 0.5.0 #SCALA_BINARY_VERSION: "2.10" #SCALA_VERSION: "2.10.4" #MESOS_VERSION: 0.21.0 diff --git a/docs/quick-start.md b/docs/quick-start.md index 271a57263..187547193 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -30,7 +30,7 @@ We use the `--packages` argument to download the graphframes package and any dep
{% highlight bash %} -$ ./bin/spark-shell --packages graphframes:graphframes:0.4.0-spark2.0-s_2.11 +$ ./bin/spark-shell --packages graphframes:graphframes:0.5.0-spark2.1-s_2.11 {% endhighlight %}
@@ -38,7 +38,7 @@ $ ./bin/spark-shell --packages graphframes:graphframes:0.4.0-spark2.0-s_2.11
{% highlight bash %} -$ ./bin/pyspark --packages graphframes:graphframes:0.4.0-spark2.0-s_2.11 +$ ./bin/pyspark --packages graphframes:graphframes:0.5.0-spark2.1-s_2.11 {% endhighlight %}
From 68d81cd0886d62b84eb3a49348e1829f4e370d2f Mon Sep 17 00:00:00 2001 From: cgosse Date: Wed, 11 Oct 2017 10:26:49 -0400 Subject: [PATCH 2/7] added a basic setup.py --- python/setup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 9dad5462e..f9ac07cd5 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,2 +1,6 @@ -# Your python setup file. An example can be found at: -# https://github.com/pypa/sampleproject/blob/master/setup.py +from setuptools import setup + +setup(name='graphframes', + version='0.5.0+metis', + packages=['graphframes', 'graphframes.lib'] +) From 94ca2bb7d501e456ff18e05b633f824034c81814 Mon Sep 17 00:00:00 2001 From: cgosse Date: Wed, 11 Oct 2017 10:32:37 -0400 Subject: [PATCH 3/7] removing custom metis version, it is just the plain version after all --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index f9ac07cd5..168b0fcbd 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,6 @@ from setuptools import setup setup(name='graphframes', - version='0.5.0+metis', + version='0.5.0', packages=['graphframes', 'graphframes.lib'] ) From 12885ef044b9dbde7d696d9aaa3c21e8c1bc51eb Mon Sep 17 00:00:00 2001 From: cgosse Date: Tue, 28 Nov 2017 12:41:44 -0500 Subject: [PATCH 4/7] moving to our own made up version 0.5.9 that is current master before 0.6.0 is released --- .gitignore | 1 + build.sbt | 10 +++++----- project/build.properties | 2 +- python/.gitignore | 1 + python/setup.py | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index dacb8dba4..b74bd29f3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ lib_managed/ src_managed/ project/boot/ project/plugins/project/ +project/project # intellij .idea/ diff --git a/build.sbt b/build.sbt index 8bb8d106c..8bbfa3ead 100644 --- a/build.sbt +++ b/build.sbt @@ -4,9 +4,9 @@ val sparkVer = sys.props.getOrElse("spark.version", "2.2.0") val sparkBranch = sparkVer.substring(0, 3) val defaultScalaVer = sparkBranch match { - case "2.0" => "2.11.8" - case "2.1" => "2.11.8" - case "2.2" => "2.11.8" + case "2.0" => "2.11.11" + case "2.1" => "2.11.11" + case "2.2" => "2.11.11" case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.") } val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer) @@ -20,11 +20,11 @@ sparkVersion := sparkVer scalaVersion := scalaVer name := "graphframes" - +organization := "graphframes" spName := "graphframes/graphframes" // Don't forget to set the version -version := s"0.6.0-SNAPSHOT-spark$sparkBranch" +version := s"0.5.9-spark$sparkBranch" // All Spark Packages need a license licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0")) diff --git a/project/build.properties b/project/build.properties index ebcf48fea..49829924c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ // This file should only contain the version of sbt to use. -sbt.version=0.13.6 +sbt.version=0.13.13 diff --git a/python/.gitignore b/python/.gitignore index 81410ca55..f6d838fc4 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -2,3 +2,4 @@ docs/_build/ build/ dist/ +graphframes.egg-info diff --git a/python/setup.py b/python/setup.py index 168b0fcbd..157ddfee4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,6 @@ from setuptools import setup setup(name='graphframes', - version='0.5.0', + version='0.5.9', packages=['graphframes', 'graphframes.lib'] ) From d7ad0b4d0e88e5e986344edbe69294c09b8026c7 Mon Sep 17 00:00:00 2001 From: cgosse Date: Tue, 28 Nov 2017 12:48:48 -0500 Subject: [PATCH 5/7] adding suggested info for pypi --- python/README.md | 3 +++ python/setup.py | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 python/README.md diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..6fcc66d82 --- /dev/null +++ b/python/README.md @@ -0,0 +1,3 @@ +# Graphframes +This is a dummy readme file. +The upstream graphframes package does not include install info for the python side of things, so this code accomplishes just that. diff --git a/python/setup.py b/python/setup.py index 157ddfee4..f89348adf 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,9 @@ from setuptools import setup setup(name='graphframes', + maintainer='Metismachine', + maintainer_email='info@metismachine.com', + url='http://metismachine.com', version='0.5.9', packages=['graphframes', 'graphframes.lib'] ) From 435325cb1e9995562da36afb9e5df92528648429 Mon Sep 17 00:00:00 2001 From: cgosse Date: Thu, 30 Nov 2017 10:14:18 -0500 Subject: [PATCH 6/7] added s3 artifact publish --- build.sbt | 33 +++++++++++++++++++++++++++++++++ project/plugins.sbt | 3 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8bbfa3ead..083446c15 100644 --- a/build.sbt +++ b/build.sbt @@ -78,3 +78,36 @@ concurrentRestrictions in Global := Seq( autoAPIMappings := true coverageHighlighting := false + +val metisArtifactRepoBaseURL = "s3://s3-us-east-1.amazonaws.com/metis-artifacts/" +// set up publishing to our private repo on S3 +def myPublishTo = Command.command("mpublish") { state => + val extracted = Project.extract(state) + Project.runTask( + publish in Compile, + extracted.append(List(publishTo := Some(Resolver.file("file", new File(System.getProperty("user.home") + + "/.m2/repository")))), state), + true + ) + Project.runTask( + publish in Compile, + extracted.append(List(publishTo := { + if (isSnapshot.value) + Some("snapshots" at metisArtifactRepoBaseURL + "snapshots") + else + Some("releases" at metisArtifactRepoBaseURL + "releases") + }), state), + true + ) + state +} + +commands += myPublishTo + +publishTo := { + if (isSnapshot.value) + Some("snapshots" at metisArtifactRepoBaseURL + "snapshots") + else + Some("releases" at metisArtifactRepoBaseURL + "releases") +} + diff --git a/project/plugins.sbt b/project/plugins.sbt index c4170fb39..a97c65f6c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,9 @@ // You may use this file to add plugin dependencies for sbt. resolvers += "Spark Packages repo" at "https://dl.bintray.com/spark-packages/maven/" - +addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.13.0") addSbtPlugin("org.spark-packages" %% "sbt-spark-package" % "0.2.6") // scalacOptions in (Compile,doc) := Seq("-groups", "-implicits") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") + From c2d66a9719dc2d0e39d47a3c3b78b153a286b34e Mon Sep 17 00:00:00 2001 From: cgosse Date: Wed, 6 Dec 2017 11:10:03 -0500 Subject: [PATCH 7/7] reverting to scala 2.11.8 to match the spark build --- build.sbt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 083446c15..d77cbf86e 100644 --- a/build.sbt +++ b/build.sbt @@ -4,9 +4,9 @@ val sparkVer = sys.props.getOrElse("spark.version", "2.2.0") val sparkBranch = sparkVer.substring(0, 3) val defaultScalaVer = sparkBranch match { - case "2.0" => "2.11.11" - case "2.1" => "2.11.11" - case "2.2" => "2.11.11" + case "2.0" => "2.11.8" + case "2.1" => "2.11.8" + case "2.2" => "2.11.8" case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.") } val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer) @@ -24,7 +24,7 @@ organization := "graphframes" spName := "graphframes/graphframes" // Don't forget to set the version -version := s"0.5.9-spark$sparkBranch" +version := s"0.5.10-spark$sparkBranch" // All Spark Packages need a license licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0"))