First, you need Java 8 JDK. ref: This is the recommended way.<\/p>\n ref: in .zshrc<\/p>\n ref: ref: ref: There are two deploy modes for Spark Standalone. In client mode, the driver is launched in the same process as the client that submits the application. In cluster mode, however, the driver is launched from one of the Worker.<\/p>\n ref: First, you need Java 8 JDK.<\/p>\n","protected":false},"author":1,"featured_media":375,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[97,112,4],"tags":[108,2,109],"class_list":["post-374","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-about-ai","category-about-big-data","category-about-python","tag-apache-spark","tag-python","tag-scala"],"_links":{"self":[{"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/posts\/374","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/comments?post=374"}],"version-history":[{"count":0,"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/posts\/374\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/media\/375"}],"wp:attachment":[{"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/media?parent=374"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/categories?post=374"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/vinta.ws\/code\/wp-json\/wp\/v2\/tags?post=374"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
\nhttp:\/\/www.oracle.com\/technetwork\/java\/javase\/downloads\/index.html<\/a><\/p>\n$ java -version\njava version \"1.8.0_131\"<\/code><\/pre>\nHomebrew Version<\/h3>\n
$ brew update\n$ brew install maven apache-spark<\/code><\/pre>\nPre-built Version<\/h3>\n
$ mkdir -p \/usr\/local\/share\/apache-spark && \n cd \/usr\/local\/share\/apache-spark && \n wget https:\/\/www.apache.org\/dyn\/closer.lua\/spark\/spark-2.2.0\/spark-2.2.0-bin-hadoop2.7.tgz && \n tar -xvzf spark-2.2.0-bin-hadoop2.7.tgz<\/code><\/pre>\n
\nhttp:\/\/spark.apache.org\/downloads.html<\/a><\/p>\nBuild Version<\/h3>\n
$ brew install scala@2.11\n$ export PATH=\"\/usr\/local\/opt\/scala@2.11\/bin:$PATH\"\n$ scala -version\nScala code runner version 2.11.8 -- Copyright 2002-2016, LAMP\/EPFL\n\n$ mkdir -p \/usr\/local\/share\/apache-spark && \n cd \/usr\/local\/share\/apache-spark && \n wget https:\/\/d3kbcqa49mib13.cloudfront.net\/spark-2.2.0.tgz && \n tar -xvzf spark-2.2.0.tgz && \n cd spark-2.2.0\n\n$ .\/build\/mvn -Pnetlib-lgpl -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.3 -DskipTests -T 4C package\n# or\n$ .\/build\/mvn -Pnetlib-lgpl -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.3 -DskipTests clean package<\/code><\/pre>\n$ spark-shell --packages \"com.github.fommil.netlib:all:1.1.2\"\nscala> import com.github.fommil.netlib.BLAS\nimport com.github.fommil.netlib.BLAS\nscala> BLAS.getInstance().getClass().getName()\nres1: String = com.github.fommil.netlib.NativeSystemBLAS<\/code><\/pre>\n
\nhttp:\/\/spark.apache.org\/downloads.html<\/a>
\nhttp:\/\/spark.apache.org\/docs\/latest\/building-spark.html<\/a>
\nhttp:\/\/spark.apache.org\/docs\/latest\/ml-guide.html#dependencies<\/a><\/p>\nConfigurations<\/h2>\n
if which java > \/dev\/null; then\n export JAVA_HOME=\"$(\/usr\/libexec\/java_home -v 1.8)\"\n export PATH=\"$JAVA_HOME\/bin:$PATH\"\nfi\n\nexport PATH=\"\/usr\/local\/opt\/scala@2.11\/bin:$PATH\"\n\n# homebrew version\nexport SPARK_HOME=\"\/usr\/local\/Cellar\/apache-spark\/2.2.0\/libexec\"\nexport PYTHONPATH=\"$SPARK_HOME\/python:$SPARK_HOME\/python\/lib\/py4j-0.10.4-src.zip:$PYTHONPATH\"\nexport PYSPARK_DRIVER_PYTHON=\"ipython\"\n\n# pre-built version\nexport SPARK_HOME=\"\/usr\/local\/share\/apache-spark\/spark-2.2.0-bin-hadoop2.7\"\nexport PATH=\"$SPARK_HOME\/bin:$PATH\"\nexport PYTHONPATH=\"$SPARK_HOME\/python:$SPARK_HOME\/python\/lib\/py4j-0.10.4-src.zip:$PYTHONPATH\"\n\n# build version\nexport SPARK_HOME=\"\/usr\/local\/share\/apache-spark\/spark-2.2.0\"\nexport PATH=\"$SPARK_HOME\/bin:$PATH\"\nexport PYTHONPATH=\"$SPARK_HOME\/python:$SPARK_HOME\/python\/lib\/py4j-0.10.4-src.zip:$PYTHONPATH\"<\/code><\/pre>\n
\nhttps:\/\/spark.apache.org\/docs\/latest\/programming-guide.html<\/a>
\nhttps:\/\/spark.apache.org\/docs\/latest\/configuration.html<\/a><\/p>\n$ cd $SPARK_HOME\n\n$ cp conf\/spark-defaults.conf.template conf\/spark-defaults.conf\nspark.driver.memory 4g\nspark.executor.memory 4g\nspark.jars.packages com.github.fommil.netlib:all:1.1.2,mysql:mysql-connector-java:5.1.41\nspark.serializer org.apache.spark.serializer.KryoSerializer\n\n$ cp conf\/spark-env.sh.template conf\/spark-env.sh\nexport PYTHONHASHSEED=42\n\n$ cp conf\/log4j.properties.template conf\/log4j.properties<\/code><\/pre>\n
\nhttps:\/\/spark.apache.org\/docs\/latest\/configuration.html<\/a><\/p>\nCommands<\/h2>\n
Local Mode<\/h3>\n
$ spark-shell\n\n$ export PYSPARK_DRIVER_PYTHON=\"jupyter\" && \nexport PYSPARK_DRIVER_PYTHON_OPTS=\"notebook --ip 0.0.0.0\" && \npyspark \n--packages \"com.github.fommil.netlib:all:1.1.2,mysql:mysql-connector-java:5.1.41\" \n--driver-memory 4g \n--executor-memory 4g \n--master \"local[*]\"\n\n$ spark-shell \n--packages \"com.github.fommil.netlib:all:1.1.2,mysql:mysql-connector-java:5.1.41\"\n--master \"local-cluster[3, 1, 4096]\"\n\n# Spark Application UI on the driver\n$ open http:\/\/localhost:4040\/<\/code><\/pre>\n
\nhttps:\/\/spark.apache.org\/docs\/latest\/programming-guide.html<\/a><\/p>\nStandalone mode<\/h3>\n
$ .\/sbin\/start-master.sh -h localhost\n$ .\/sbin\/start-slave.sh spark:\/\/localhost:7077\n\n# Spark Web UI on the cluster manager\n$ open http:\/\/localhost:8080\/\n\n$ pyspark \n--driver-memory 4g \n--executor-memory 4g \n--master spark:\/\/localhost:7077\n\n$ spark-submit \n--master spark:\/\/localhost:7077 \nexamples\/src\/main\/python\/pi.py 10\n\n$ spark-submit \n--driver-memory 2g \n--driver-java-options \"-XX:ThreadStackSize=81920\" \n--total-executor-cores 3 \n--executor-cores 3 \n--executor-memory 12g \n--conf \"spark.executor.extraJavaOptions=-XX:ThreadStackSize=81920\" \n--master spark:\/\/localhost:7077 \n--packages \"mysql:mysql-connector-java:5.1.41,com.hankcs:hanlp:portable-1.3.4,edu.stanford.nlp:stanford-corenlp:3.7.0\" \n--jars \"\/Users\/vinta\/Projects\/albedo\/spark-data\/stanford-corenlp-3.8.0-models.jar\" \n--class ws.vinta.albedo.LogisticRegressionRanker \ntarget\/albedo-1.0.0-SNAPSHOT.jar\n\n# Spark Application UI on the driver\n$ open http:\/\/localhost:4040\/<\/code><\/pre>\n
\nhttps:\/\/spark.apache.org\/docs\/latest\/spark-standalone.html<\/a>
\nhttps:\/\/spark.apache.org\/docs\/latest\/submitting-applications.html<\/a>
\nhttps:\/\/spark.apache.org\/docs\/latest\/configuration.html<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"