spark-defaults.conf

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.

# Example:
# spark.master                     				spark://master:7077
# spark.eventLog.enabled           				true
# spark.eventLog.dir               				hdfs://namenode:8021/directory
# spark.serializer                 				org.apache.spark.serializer.KryoSerializer
# spark.driver.memory              				5g
# spark.executor.extraJavaOptions  				-XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"

#spark.driver.extraJavaOptions             -XX:MaxPermSize=6G -XX:+UseConcMarkSweepGC -Dlog4j.configuration=log4j.properties
spark.driver.cores                        1
spark.driver.memory                       2G

spark.dynamicAllocation.enabled           false
spark.dynamicAllocation.minExecutors			2
spark.dynamicAllocation.maxExecutors			10

# spark.executor.extraJavaOptions           -XX: +UseConcMarkSweepGC -Dlog4j.configuration=log4j.properties
# The number of executors for static allocation. With spark.dynamicAllocation.enabled, the initial set of executors
# will be at least this large.
spark.executor.instances                  2
spark.executor.memory                     8G

# Maximum allowable size of Kryo serialization buffer.
# This must be larger than any object ou attempt to serialize.
# Increase this if you get a "buffer limit exceeded" exception inside Kryo.
spark.kryoserializer.buffer.max           64m

spark.locality.wait                       100ms

spark.rdd.compress                        true

spark.scheduler.mode                      FAIR
spark.scheduler.pool                      vpe

spark.serializer                          org.apache.spark.serializer.KryoSerializer

spark.shuffle.consolidateFiles					  true
spark.shuffle.service.enabled					    true

spark.sql.tungsten.enabled						    false
spark.sql.codegen								          false
spark.sql.unsafe.enabled						      false

spark.streaming.backpressure.enabled			      true
spark.streaming.blockInterval					          100ms
spark.streaming.dynamicAllocation.enabled		    false
spark.streaming.dynamicAllocation.minExecutors	2
spark.streaming.dynamicAllocation.maxExecutors	10
spark.streaming.dynamicAllocation.debug			    true
spark.streaming.dynamicAllocation.delay.rounds	10

spark.streaming.driver.writeAheadLog.closeFileAfterWrite	  true
spark.streaming.receiver.writeAheadLog.enable				        true
spark.streaming.receiver.writeAheadLog.closeFileAfterWrite	true

spark.ui.showConsoleProgress              false