Skip to content

Discard master-slave vocabulary #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Discard master-slave vocabulary #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 29 additions & 29 deletions deploy_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,60 +7,60 @@
import sys

# Deploy the configuration file templates in the spark-ec2/templates directory
# to the root filesystem, substituting variables such as the master hostname,
# to the root filesystem, substituting variables such as the main hostname,
# ZooKeeper URL, etc as read from the environment.

# Find system memory in KB and compute Spark's default limit from that
mem_command = "cat /proc/meminfo | grep MemTotal | awk '{print $2}'"
cpu_command = "nproc"

master_ram_kb = int(
main_ram_kb = int(
os.popen(mem_command).read().strip())
# This is the master's memory. Try to find slave's memory as well
first_slave = os.popen("cat /root/spark-ec2-setup/slaves | head -1").read().strip()
# This is the main's memory. Try to find subordinate's memory as well
first_subordinate = os.popen("cat /root/spark-ec2-setup/subordinates | head -1").read().strip()

slave_mem_command = "ssh -t -o StrictHostKeyChecking=no %s %s" %\
(first_slave, mem_command)
subordinate_mem_command = "ssh -t -o StrictHostKeyChecking=no %s %s" %\
(first_subordinate, mem_command)

slave_cpu_command = "ssh -t -o StrictHostKeyChecking=no %s %s" %\
(first_slave, cpu_command)
subordinate_cpu_command = "ssh -t -o StrictHostKeyChecking=no %s %s" %\
(first_subordinate, cpu_command)

slave_ram_kb = int(os.popen(slave_mem_command).read().strip())
subordinate_ram_kb = int(os.popen(subordinate_mem_command).read().strip())

slave_cpus = int(os.popen(slave_cpu_command).read().strip())
subordinate_cpus = int(os.popen(subordinate_cpu_command).read().strip())

system_ram_kb = min(slave_ram_kb, master_ram_kb)
system_ram_kb = min(subordinate_ram_kb, main_ram_kb)

system_ram_mb = system_ram_kb / 1024
slave_ram_mb = slave_ram_kb / 1024
subordinate_ram_mb = subordinate_ram_kb / 1024
# Leave some RAM for the OS, Hadoop daemons, and system caches
if slave_ram_mb > 100*1024:
slave_ram_mb = slave_ram_mb - 15 * 1024 # Leave 15 GB RAM
elif slave_ram_mb > 60*1024:
slave_ram_mb = slave_ram_mb - 10 * 1024 # Leave 10 GB RAM
elif slave_ram_mb > 40*1024:
slave_ram_mb = slave_ram_mb - 6 * 1024 # Leave 6 GB RAM
elif slave_ram_mb > 20*1024:
slave_ram_mb = slave_ram_mb - 3 * 1024 # Leave 3 GB RAM
elif slave_ram_mb > 10*1024:
slave_ram_mb = slave_ram_mb - 2 * 1024 # Leave 2 GB RAM
if subordinate_ram_mb > 100*1024:
subordinate_ram_mb = subordinate_ram_mb - 15 * 1024 # Leave 15 GB RAM
elif subordinate_ram_mb > 60*1024:
subordinate_ram_mb = subordinate_ram_mb - 10 * 1024 # Leave 10 GB RAM
elif subordinate_ram_mb > 40*1024:
subordinate_ram_mb = subordinate_ram_mb - 6 * 1024 # Leave 6 GB RAM
elif subordinate_ram_mb > 20*1024:
subordinate_ram_mb = subordinate_ram_mb - 3 * 1024 # Leave 3 GB RAM
elif subordinate_ram_mb > 10*1024:
subordinate_ram_mb = subordinate_ram_mb - 2 * 1024 # Leave 2 GB RAM
else:
slave_ram_mb = max(512, slave_ram_mb - 1300) # Leave 1.3 GB RAM
subordinate_ram_mb = max(512, subordinate_ram_mb - 1300) # Leave 1.3 GB RAM

worker_instances_str = ""
worker_cores = slave_cpus
worker_cores = subordinate_cpus

if os.getenv("SPARK_WORKER_INSTANCES") != "":
worker_instances = int(os.getenv("SPARK_WORKER_INSTANCES", 1))
worker_instances_str = "%d" % worker_instances
# Distribute equally cpu cores among worker instances
worker_cores = max(slave_cpus / worker_instances, 1)
worker_cores = max(subordinate_cpus / worker_instances, 1)

template_vars = {
"master_list": os.getenv("MASTERS"),
"active_master": os.getenv("MASTERS").split("\n")[0],
"slave_list": os.getenv("SLAVES"),
"spark_master_opts": os.getenv("SPARK_MASTER_OPTS", ""),
"main_list": os.getenv("MASTERS"),
"active_main": os.getenv("MASTERS").split("\n")[0],
"subordinate_list": os.getenv("SLAVES"),
"spark_main_opts": os.getenv("SPARK_MASTER_OPTS", ""),
"spark_version": os.getenv("SPARK_VERSION"),
"hadoop_major_version": os.getenv("HADOOP_MAJOR_VERSION"),
"java_home": os.getenv("JAVA_HOME"),
Expand Down
Loading