diff --git a/starcluster/commands/createvolume.py b/starcluster/commands/createvolume.py
index b8d754db2..7a3329f72 100644
--- a/starcluster/commands/createvolume.py
+++ b/starcluster/commands/createvolume.py
@@ -16,6 +16,7 @@
# along with StarCluster. If not, see .
import os
+import re
from starcluster import node
from starcluster import volume
@@ -141,6 +142,9 @@ def execute(self, args):
kwargs = self.specified_options_dict
kwargs.update(dict(keypair=keypair, key_location=key_location,
host_instance=host_instance))
+ if 'image_id' not in kwargs.keys():
+ zone_short = re.match(".+-.+-[1-9]+", zone).group()
+ kwargs['image_id'] = static.BASE_AMI_64[zone_short]
vc = volume.VolumeCreator(self.ec2, **kwargs)
if host_instance:
vc._validate_host_instance(host_instance, zone)
diff --git a/starcluster/commands/resizevolume.py b/starcluster/commands/resizevolume.py
index b3aa75a93..f8b41439b 100644
--- a/starcluster/commands/resizevolume.py
+++ b/starcluster/commands/resizevolume.py
@@ -15,6 +15,8 @@
# You should have received a copy of the GNU Lesser General Public License
# along with StarCluster. If not, see .
+import re
+
from starcluster import node
from starcluster import volume
from starcluster import static
@@ -96,6 +98,9 @@ def execute(self, args):
kwargs = self.specified_options_dict
kwargs.update(dict(keypair=keypair, key_location=key_location,
host_instance=host_instance))
+ if 'image_id' not in kwargs.keys():
+ zone_short = re.match(".+-.+-[1-9]+", zone).group()
+ kwargs['image_id'] = static.BASE_AMI_64[zone_short]
vc = volume.VolumeCreator(self.ec2, **kwargs)
if host_instance:
vc._validate_host_instance(host_instance, zone)
diff --git a/starcluster/plugins/sge.py b/starcluster/plugins/sge.py
index 2a3662708..4055541ac 100644
--- a/starcluster/plugins/sge.py
+++ b/starcluster/plugins/sge.py
@@ -77,7 +77,12 @@ def _create_sge_pe(self, name="orte", nodes=None, queue="all.q"):
pe_slots = self.slots_per_host * len(nodes)
if not pe_exists:
penv = mssh.remote_file("/tmp/pe.txt", "w")
- penv.write(sge.sge_pe_template % (name, pe_slots))
+ if name == "orte":
+ template = sge.sge_orte_pe_template
+ elif name == "smp":
+ template = sge.sge_smp_pe_template
+ penv.write(template % (name, pe_slots))
+ #penv.write(sge.sge_pe_template % (name, pe_slots))
penv.close()
mssh.execute("qconf -Ap %s" % penv.name)
else:
@@ -155,7 +160,8 @@ def _setup_sge(self):
for node in self.nodes:
self.pool.simple_job(self._add_to_sge, (node,), jobid=node.alias)
self.pool.wait(numtasks=len(self.nodes))
- self._create_sge_pe()
+ self._create_sge_pe(name="orte")
+ self._create_sge_pe(name="smp")
def _remove_from_sge(self, node):
master = self._master
@@ -166,7 +172,8 @@ def _remove_from_sge(self, node):
master.ssh.execute('qconf -de %s' % node.alias)
node.ssh.execute('pkill -9 sge_execd')
nodes = filter(lambda n: n.alias != node.alias, self._nodes)
- self._create_sge_pe(nodes=nodes)
+ self._create_sge_pe(name="orte", nodes=nodes)
+ self._create_sge_pe(name="smp", nodes=nodes)
def run(self, nodes, master, user, user_shell, volumes):
if not master.ssh.isdir(self.SGE_FRESH):
@@ -192,7 +199,8 @@ def on_add_node(self, node, nodes, master, user, user_shell, volumes):
self._add_sge_admin_host(node)
self._add_sge_submit_host(node)
self._add_to_sge(node)
- self._create_sge_pe()
+ self._create_sge_pe(name="orte")
+ self._create_sge_pe(name="smp")
def on_remove_node(self, node, nodes, master, user, user_shell, volumes):
self._nodes = nodes
diff --git a/starcluster/static.py b/starcluster/static.py
index abc40896e..6f9486186 100644
--- a/starcluster/static.py
+++ b/starcluster/static.py
@@ -81,9 +81,13 @@ def create_sc_config_dirs():
CRASH_FILE = os.path.join(STARCLUSTER_LOG_DIR, 'crash-report-%d.txt' % PID)
# StarCluster BASE AMIs (us-east-1)
-BASE_AMI_32 = "ami-9bf9c9f2"
-BASE_AMI_64 = "ami-3393a45a"
-BASE_AMI_HVM = "ami-6b211202"
+BASE_AMI_32 = {'us-east-1': "ami-9bf9c9f2", 'us-west-1': "ami-52112317",
+ 'us-west-2': "ami-b2badb82"}
+BASE_AMI_64 = {'us-east-1': "ami-3393a45a", 'us-west-1': "ami-56172513",
+ 'us-west-2': "ami-04bedf34"}
+BASE_AMI_HVM = {'us-east-1': "ami-6b211202", 'us-west-1': "ami-06172543",
+ 'us-west-2': "ami-80bedfb0"}
+
SECURITY_GROUP_PREFIX = "@sc-"
SECURITY_GROUP_TEMPLATE = SECURITY_GROUP_PREFIX + "%s"
@@ -138,6 +142,12 @@ def create_sc_config_dirs():
'r3.2xlarge': ['x86_64'],
'r3.4xlarge': ['x86_64'],
'r3.8xlarge': ['x86_64'],
+ 'r4.large': ['x86_64'],
+ 'r4.xlarge': ['x86_64'],
+ 'r4.2xlarge': ['x86_64'],
+ 'r4.4xlarge': ['x86_64'],
+ 'r4.8xlarge': ['x86_64'],
+ 'r4.16xlarge': ['x86_64'],
'cc1.4xlarge': ['x86_64'],
'cc2.8xlarge': ['x86_64'],
'cg1.4xlarge': ['x86_64'],
@@ -156,6 +166,12 @@ def create_sc_config_dirs():
'c4.2xlarge': ['x86_64'],
'c4.4xlarge': ['x86_64'],
'c4.8xlarge': ['x86_64'],
+ 'c5.large': ['x86_64'],
+ 'c5.xlarge': ['x86_64'],
+ 'c5.2xlarge': ['x86_64'],
+ 'c5.4xlarge': ['x86_64'],
+ 'c5.9xlarge': ['x86_64'],
+ 'c5.18xlarge': ['x86_64'],
'i2.xlarge': ['x86_64'],
'i2.2xlarge': ['x86_64'],
'i2.4xlarge': ['x86_64'],
@@ -179,12 +195,17 @@ def create_sc_config_dirs():
CLUSTER_HIMEM_TYPES = ['cr1.8xlarge']
HIMEM_TYPES = ['r3.large', 'r3.xlarge', 'r3.2xlarge', 'r3.4xlarge',
- 'r3.8xlarge']
+ 'r3.8xlarge',
+ 'r4.large', 'r4.xlarge', 'r4.2xlarge', 'r4.4xlarge',
+ 'r4.8xlarge', 'r4.16xlarge']
HI_IO_TYPES = ['hi1.4xlarge']
HI_STORAGE_TYPES = ['hs1.8xlarge']
+C5_COMPUTE_TYPES = ['c5.large', 'c5.xlarge', 'c5.2xlarge', 'c5.4xlarge',
+ 'c5.9xlarge', 'c5.18xlarge']
+
M3_COMPUTE_TYPES = ['c3.large', 'c3.xlarge', 'c3.2xlarge', 'c3.4xlarge',
'c3.8xlarge']
@@ -197,6 +218,7 @@ def create_sc_config_dirs():
DENSE_STORAGE_TYPES = ['d2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge']
HVM_ONLY_TYPES = (CLUSTER_COMPUTE_TYPES + CLUSTER_GPU_TYPES +
+ C5_COMPUTE_TYPES +
CLUSTER_HIMEM_TYPES + I2_STORAGE_TYPES + HIMEM_TYPES +
T2_INSTANCE_TYPES + DENSE_STORAGE_TYPES)
diff --git a/starcluster/templates/config.py b/starcluster/templates/config.py
index 885d741c3..8490a496d 100644
--- a/starcluster/templates/config.py
+++ b/starcluster/templates/config.py
@@ -349,9 +349,9 @@
# [plugin xvfb]
# SETUP_CLASS = starcluster.plugins.xvfb.XvfbSetup
""" % {
- 'x86_ami': static.BASE_AMI_32,
- 'x86_64_ami': static.BASE_AMI_64,
- 'hvm_ami': static.BASE_AMI_HVM,
+ 'x86_ami': static.BASE_AMI_32['us-east-1'],
+ 'x86_64_ami': static.BASE_AMI_64['us-east-1'],
+ 'hvm_ami': static.BASE_AMI_HVM['us-east-1'],
'instance_types': ', '.join(static.INSTANCE_TYPES.keys()),
'shells': ', '.join(static.AVAILABLE_SHELLS.keys()),
}
diff --git a/starcluster/templates/sge.py b/starcluster/templates/sge.py
index dcf189941..97ac4bfaa 100644
--- a/starcluster/templates/sge.py
+++ b/starcluster/templates/sge.py
@@ -59,7 +59,7 @@
CSP_MAIL_ADDRESS="none@none.edu"
"""
-sge_pe_template = """
+sge_orte_pe_template = """
pe_name %s
slots %s
user_lists NONE
@@ -73,6 +73,20 @@
accounting_summary FALSE
"""
+sge_smp_pe_template = """
+pe_name %s
+slots %s
+user_lists NONE
+xuser_lists NONE
+start_proc_args /bin/true
+stop_proc_args /bin/true
+allocation_rule $pe_slots
+control_slaves FALSE
+job_is_first_task TRUE
+urgency_slots min
+accounting_summary FALSE
+"""
+
sgeprofile_template = """
export SGE_ROOT="/opt/sge6"
export SGE_CELL="default"
diff --git a/starcluster/volume.py b/starcluster/volume.py
index 1ae2f507e..2e036d87f 100644
--- a/starcluster/volume.py
+++ b/starcluster/volume.py
@@ -43,7 +43,7 @@ class VolumeCreator(cluster.Cluster):
"""
def __init__(self, ec2_conn, spot_bid=None, keypair=None,
key_location=None, host_instance=None, device='/dev/sdz',
- image_id=static.BASE_AMI_32, instance_type="t1.micro",
+ image_id=static.BASE_AMI_32['us-east-1'], instance_type="t1.micro",
shutdown_instance=False, detach_vol=False,
mkfs_cmd='mkfs.ext3 -F', resizefs_cmd='resize2fs', **kwargs):
self._host_instance = host_instance
@@ -51,13 +51,14 @@ def __init__(self, ec2_conn, spot_bid=None, keypair=None,
self._volume = None
self._aws_block_device = device or '/dev/sdz'
self._real_device = None
- self._image_id = image_id or static.BASE_AMI_32
+ self._image_id = image_id or static.BASE_AMI_32['us-east-1']
self._instance_type = instance_type or 'm1.small'
self._shutdown = shutdown_instance
self._detach_vol = detach_vol
self._mkfs_cmd = mkfs_cmd
self._resizefs_cmd = resizefs_cmd
self._alias_tmpl = "volhost-%s"
+ self._snapshot = None
super(VolumeCreator, self).__init__(
ec2_conn=ec2_conn, spot_bid=spot_bid, keyname=keypair,
key_location=key_location, cluster_tag=static.VOLUME_GROUP_NAME,
@@ -206,7 +207,7 @@ def is_valid(self, size, zone, device):
try:
self.validate(size, zone, device)
return True
- except exception.BaseException, e:
+ except exception.BaseException as e:
log.error(e.msg)
return False