From 64c3ad5e0046e83ac39f395cfe27f681a2ba2558 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Wed, 8 Aug 2018 18:09:36 +0200 Subject: [PATCH 01/31] wip --- .../main/python/slipstream/NodeDecorator.py | 4 ++ .../main/python/slipstream/NodeInstance.py | 25 ++++++++ client/src/main/python/slipstream/UserInfo.py | 3 +- .../cloudconnectors/BaseCloudConnector.py | 60 ++++++++++--------- .../slipstream/command/RunInstancesCommand.py | 7 +++ .../executors/node/NodeDeploymentExecutor.py | 2 +- .../python/slipstream/wrappers/BaseWrapper.py | 9 +++ .../src/main/scripts/slipstream.bootstrap.py | 4 +- .../src/main/scripts/slipstream_bootstrap.py | 1 - 9 files changed, 82 insertions(+), 33 deletions(-) delete mode 120000 client/src/main/scripts/slipstream_bootstrap.py diff --git a/client/src/main/python/slipstream/NodeDecorator.py b/client/src/main/python/slipstream/NodeDecorator.py index 4165e351..376df292 100644 --- a/client/src/main/python/slipstream/NodeDecorator.py +++ b/client/src/main/python/slipstream/NodeDecorator.py @@ -83,10 +83,14 @@ class NodeDecorator(object): SCALE_DISK_ATTACHED_DEVICE = 'disk.attached.device' SCALE_DISK_DETACH_DEVICE = 'disk.detach.device' INSTANCEID_KEY = 'instanceid' + CLOUD_NODE_ID_KEY = 'cloud.node.id' + CLOUD_NODE_IP_KEY = 'cloud.node.ip' + CLOUD_NODE_PASSWORD_KEY = 'cloud.node.password' CLOUDSERVICE_KEY = 'cloudservice' SECURITY_GROUPS_KEY = 'security.groups' MAX_PROVISIONING_FAILURES_KEY = 'max-provisioning-failures' NATIVE_CONTEXTUALIZATION_KEY = 'native-contextualization' + DEPLOYMENT_CONTEXT_KEY = 'deployment.context' SECURITY_GROUP_ALLOW_ALL_NAME = 'slipstream_managed' SECURITY_GROUP_ALLOW_ALL_DESCRIPTION = 'Security group created by SlipStream which allows all kind of traffic.' diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index 71a68125..bbee90bc 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -210,3 +210,28 @@ def get_disk_attach_size(self): def get_disk_detach_device(self): return self.__get(NodeDecorator.SCALE_DISK_DETACH_DEVICE) + def set_deployment_context(self, context=None): + self.__set(NodeDecorator.DEPLOYMENT_CONTEXT_KEY, context) + + def get_deployment_context(self): + return self.__get(NodeDecorator.DEPLOYMENT_CONTEXT_KEY, {}) + + def set_cloud_node_id(self, id): + self.__set(NodeDecorator.CLOUD_NODE_ID_KEY, id) + + def get_cloud_node_id(self): + return self.__get(NodeDecorator.CLOUD_NODE_ID_KEY) + + def set_cloud_node_ip(self, ip): + self.__set(NodeDecorator.CLOUD_NODE_IP_KEY, ip) + + def get_cloud_node_ip(self): + return self.__get(NodeDecorator.CLOUD_NODE_IP_KEY) + + def set_cloud_node_password(self, password): + self.__set(NodeDecorator.CLOUD_NODE_PASSWORD_KEY, password) + + def get_cloud_node_password(self): + return self.__get(NodeDecorator.CLOUD_NODE_PASSWORD_KEY) + + diff --git a/client/src/main/python/slipstream/UserInfo.py b/client/src/main/python/slipstream/UserInfo.py index c0853cab..4e550d96 100644 --- a/client/src/main/python/slipstream/UserInfo.py +++ b/client/src/main/python/slipstream/UserInfo.py @@ -25,6 +25,7 @@ class UserInfo(dict): SSH_PUBKEY_KEY = 'sshPublicKey' NETWORK_PUBLIC_KEY = 'networkPublic' NETWORK_PRIVATE_KEY = 'networkPrivate' + CLOUD_ENDPOINT_KEY = 'endpoint' def __init__(self, cloud_qualifier): super(UserInfo, self).__init__({}) @@ -69,7 +70,7 @@ def get_cloud_password(self): return self.get_cloud(self.CLOUD_PASSWORD_KEY) def get_cloud_endpoint(self): - return self.get_cloud('endpoint') + return self.get_cloud(self.CLOUD_ENDPOINT_KEY) def get_public_keys(self): return self.get_general(self.SSH_PUBKEY_KEY) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 2794f3fc..f730aa9c 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -312,7 +312,7 @@ def __init__(self, configHolder): self.__vms = {} - self.__cloud = os.environ[util.ENV_CONNECTOR_INSTANCE] + #self.__cloud = os.environ[util.ENV_CONNECTOR_INSTANCE] self.__init_threading_related() @@ -506,13 +506,20 @@ def _publish_vm_info(self, vm, node_instance): with lock: already_published = self.__already_published[instance_name] if vm_id and 'id' not in already_published: - self._publish_vm_id(instance_name, vm_id) + # self._publish_vm_id(instance_name, vm_id) + node_instance.set_cloud_node_id(vm_id) already_published.add('id') if vm_ip and 'ip' not in already_published: - self._publish_vm_ip(instance_name, vm_ip) + # self._publish_vm_ip(instance_name, vm_ip) + node_instance.set_cloud_node_ip(vm_ip) already_published.add('ip') if node_instance and vm_ip and 'ssh' not in already_published: - self._publish_url_ssh(vm, node_instance) + # self._publish_url_ssh(vm, node_instance) + + if node_instance: + vm_ip = self._vm_get_ip(vm) or '' + ssh_username, _ = self.__get_vm_username_password(node_instance) + node_instance.set_parameter('url.ssh', 'ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) already_published.add('ssh') def _publish_vm_id(self, instance_name, vm_id): @@ -819,32 +826,26 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, """This method can be redefined by connectors if they need a specific bootstrap script with the SSH contextualization.""" script = '' - addEnvironmentVariableCommand = '' + add_deployment_context_variable_command = '' node_instance_name = node_instance.get_name() if node_instance.is_windows(): - addEnvironmentVariableCommand = 'set' + add_deployment_context_variable_command = 'set' script += 'rem cmd\n' else: - addEnvironmentVariableCommand = 'export' + add_deployment_context_variable_command = 'export' script += '#!/bin/sh -ex\n' - regex = 'SLIPSTREAM_' - if self.is_start_orchestrator(): - regex += '|CLOUDCONNECTOR_' - env_matcher = re.compile(regex) - if pre_export: script += '%s\n' % pre_export - for var, val in os.environ.items(): - if env_matcher.match(var) and var != util.ENV_NODE_INSTANCE_NAME: - if re.search(' ', val): - val = '"%s"' % val - script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, - val) + deployment_context = node_instance.get_deployment_context() + for var, val in deployment_context.items(): + if re.search(' ', val): + val = '"%s"' % val + script += '%s %s=%s\n' % (add_deployment_context_variable_command, var, val) - script += '%s %s=%s\n' % (addEnvironmentVariableCommand, + script += '%s %s=%s\n' % (add_deployment_context_variable_command, util.ENV_NODE_INSTANCE_NAME, node_instance_name) @@ -861,13 +862,14 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, def _build_slipstream_bootstrap_command(self, node_instance, username=None): instance_name = node_instance.get_name() + bootstrap_url = node_instance.get_deployment_context()['SLIPSTREAM_BOOTSTRAP_BIN'] if node_instance.is_windows(): - return self.__build_slipstream_bootstrap_command_for_windows(instance_name) + return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url) else: - return self.__build_slipstream_bootstrap_command_for_linux(instance_name) + return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url) - def __build_slipstream_bootstrap_command_for_windows(self, instance_name): + def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url): command = 'If Not Exist %(reports)s mkdir %(reports)s\n' command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n' @@ -883,32 +885,32 @@ def __build_slipstream_bootstrap_command_for_windows(self, instance_name): command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n' - return command % self._get_bootstrap_command_replacements_for_windows(instance_name) + return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url) - def __build_slipstream_bootstrap_command_for_linux(self, instance_name): + def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url): command = 'mkdir -p %(reports)s %(ss_home)s; ' command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 ' command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) ' command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1' - return command % self._get_bootstrap_command_replacements_for_linux(instance_name) + return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url) - def _get_bootstrap_command_replacements_for_linux(self, instance_name): + def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url): return { 'reports': util.REPORTSDIR, 'bootstrap': os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'), - 'bootstrapUrl': util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN'), + 'bootstrapUrl': bootstrap_url, 'ss_home': util.SLIPSTREAM_HOME, 'nodename': instance_name, 'machine_executor': self._get_machine_executor_type() } - def _get_bootstrap_command_replacements_for_windows(self, instance_name): + def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url): return { 'reports': util.WINDOWS_REPORTSDIR, 'bootstrap': '\\'.join([util.WINDOWS_SLIPSTREAM_HOME, 'slipstream.bootstrap']), - 'bootstrapUrl': util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN'), + 'bootstrapUrl': bootstrap_url, 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, 'machine_executor': self._get_machine_executor_type() diff --git a/client/src/main/python/slipstream/command/RunInstancesCommand.py b/client/src/main/python/slipstream/command/RunInstancesCommand.py index d8d5b4c9..fd7c9eb6 100755 --- a/client/src/main/python/slipstream/command/RunInstancesCommand.py +++ b/client/src/main/python/slipstream/command/RunInstancesCommand.py @@ -158,5 +158,12 @@ def _run_instance(self, node_instance): config={'foo': 'bar'}) cc = cloud_connector_class(ch) + from pprint import pprint + print('user_info: ') + pprint(self.user_info) + print('nodes instance map: ') + pprint({nodename: node_instance}) + print('get_initialization_extra_kwargs: ') + pprint(self.get_initialization_extra_kwargs()) cc.start_nodes_and_clients(self.user_info, {nodename: node_instance}, self.get_initialization_extra_kwargs()) diff --git a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py index 43ae66d1..cffd9460 100644 --- a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py +++ b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py @@ -180,7 +180,7 @@ def _get_scaling_exports(self): def _add_ssh_pubkey(self, login_user): if not util.is_windows(): util.printStep('Adding the public keys') - append_ssh_pubkey_to_authorized_keys(self._get_user_ssh_pubkey(), login_user) + append_ssh_pubkey_to_authorized_keys(self.wrapper.user_ssh_pub_keys, login_user) def _get_user_ssh_pubkey(self): return self.wrapper.get_user_ssh_pubkey() diff --git a/client/src/main/python/slipstream/wrappers/BaseWrapper.py b/client/src/main/python/slipstream/wrappers/BaseWrapper.py index 127b2ebb..656fdfc9 100644 --- a/client/src/main/python/slipstream/wrappers/BaseWrapper.py +++ b/client/src/main/python/slipstream/wrappers/BaseWrapper.py @@ -133,6 +133,8 @@ def __init__(self, config_holder): self.my_node_instance_name = self._get_my_node_instance_name(config_holder) + self.user_ssh_pub_keys = self._get_my_node_instance_name(config_holder) + self._config_holder = config_holder self._user_info = None @@ -148,6 +150,13 @@ def _get_my_node_instance_name(config_holder): except Exception: raise Exceptions.ExecutionException('Failed to get the node instance name of the the current VM') + @staticmethod + def _get_user_ssh_pub_keys(config_holder): + try: + return config_holder.user_ssh_pub_keys + except Exception: + raise Exceptions.ExecutionException('Failed to get the user ssh public keys') + def get_my_node_instance_name(self): return self.my_node_instance_name diff --git a/client/src/main/scripts/slipstream.bootstrap.py b/client/src/main/scripts/slipstream.bootstrap.py index fb21348b..628451e9 100755 --- a/client/src/main/scripts/slipstream.bootstrap.py +++ b/client/src/main/scripts/slipstream.bootstrap.py @@ -327,13 +327,15 @@ def _persist_ss_context(): serviceurl = %s node_instance_name = %s ss_cache_key = %s +user_ssh_pub_keys = %s """ % (os.environ['SLIPSTREAM_DIID'], os.environ['SLIPSTREAM_USERNAME'], os.environ['SLIPSTREAM_API_KEY'].strip('"'), os.environ['SLIPSTREAM_API_SECRET'].strip('"'), os.environ['SLIPSTREAM_SERVICEURL'], os.environ['SLIPSTREAM_NODE_INSTANCE_NAME'], - os.environ['SLIPSTREAM_SS_CACHE_KEY']) + os.environ['SLIPSTREAM_SS_CACHE_KEY'], + os.environ['SLIPSTREAM_USER_SSH_PUB_KEYS']) _write_to_ss_client_bin('slipstream.context', slipstream_context) diff --git a/client/src/main/scripts/slipstream_bootstrap.py b/client/src/main/scripts/slipstream_bootstrap.py deleted file mode 120000 index 6785d710..00000000 --- a/client/src/main/scripts/slipstream_bootstrap.py +++ /dev/null @@ -1 +0,0 @@ -slipstream.bootstrap.py \ No newline at end of file From 32ed9325aff5a5a5e1ffea47cc3f1cb08dba0d9a Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Thu, 9 Aug 2018 15:59:54 +0200 Subject: [PATCH 02/31] wip --- .../main/python/slipstream/NodeDecorator.py | 4 +++- .../main/python/slipstream/NodeInstance.py | 21 +++++++++++++++---- client/src/main/python/slipstream/UserInfo.py | 3 +++ .../cloudconnectors/BaseCloudConnector.py | 9 +++++--- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/client/src/main/python/slipstream/NodeDecorator.py b/client/src/main/python/slipstream/NodeDecorator.py index 376df292..5eec8464 100644 --- a/client/src/main/python/slipstream/NodeDecorator.py +++ b/client/src/main/python/slipstream/NodeDecorator.py @@ -85,7 +85,9 @@ class NodeDecorator(object): INSTANCEID_KEY = 'instanceid' CLOUD_NODE_ID_KEY = 'cloud.node.id' CLOUD_NODE_IP_KEY = 'cloud.node.ip' - CLOUD_NODE_PASSWORD_KEY = 'cloud.node.password' + CLOUD_NODE_SSH_URL_KEY = 'cloud.node.ssh.url' + CLOUD_NODE_SSH_PASSWORD_KEY = 'cloud.node.ssh.password' + CLOUD_NODE_SSH_KEYPAIR_NAME_KEY = 'cloud.node.ssh.keypair.name' CLOUDSERVICE_KEY = 'cloudservice' SECURITY_GROUPS_KEY = 'security.groups' MAX_PROVISIONING_FAILURES_KEY = 'max-provisioning-failures' diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index bbee90bc..bc712fe7 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -228,10 +228,23 @@ def set_cloud_node_ip(self, ip): def get_cloud_node_ip(self): return self.__get(NodeDecorator.CLOUD_NODE_IP_KEY) - def set_cloud_node_password(self, password): - self.__set(NodeDecorator.CLOUD_NODE_PASSWORD_KEY, password) + def set_cloud_node_ssh_url(self, url): + self.__set(NodeDecorator.CLOUD_NODE_SSH_URL_KEY, url) + + def get_cloud_node_ssh_url(self): + return self.__get(NodeDecorator.CLOUD_NODE_SSH_URL_KEY) + + def set_cloud_node_ssh_password(self, password): + self.__set(NodeDecorator.CLOUD_NODE_SSH_PASSWORD_KEY, password) + + def get_cloud_node_ssh_password(self): + return self.__get(NodeDecorator.CLOUD_NODE_SSH_PASSWORD_KEY) + + def set_cloud_node_ssh_keypair_name(self, password): + self.__set(NodeDecorator.CLOUD_NODE_SSH_KEYPAIR_NAME_KEY, password) + + def get_cloud_node_ssh_keypair_name(self): + return self.__get(NodeDecorator.CLOUD_NODE_SSH_KEYPAIR_NAME_KEY) - def get_cloud_node_password(self): - return self.__get(NodeDecorator.CLOUD_NODE_PASSWORD_KEY) diff --git a/client/src/main/python/slipstream/UserInfo.py b/client/src/main/python/slipstream/UserInfo.py index 4e550d96..7449feba 100644 --- a/client/src/main/python/slipstream/UserInfo.py +++ b/client/src/main/python/slipstream/UserInfo.py @@ -75,6 +75,9 @@ def get_cloud_endpoint(self): def get_public_keys(self): return self.get_general(self.SSH_PUBKEY_KEY) + def set_public_keys(self, ssh_pub_keys): + self[self.general + self.SSH_PUBKEY_KEY]= ssh_pub_keys + def get_private_key(self): return self.get_cloud('private.key') diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index f730aa9c..7c83d79f 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -518,9 +518,12 @@ def _publish_vm_info(self, vm, node_instance): if node_instance: vm_ip = self._vm_get_ip(vm) or '' - ssh_username, _ = self.__get_vm_username_password(node_instance) - node_instance.set_parameter('url.ssh', 'ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) - already_published.add('ssh') + ssh_username, ssh_password = self.__get_vm_username_password(node_instance) + node_instance.set_cloud_node_ssh_url('ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) + node_instance.set_cloud_node_ssh_password(ssh_password) + + if ssh_username and ssh_password: + already_published.add('ssh') def _publish_vm_id(self, instance_name, vm_id): # Needed for thread safety From e8a27fc2e34d1265b000de65adc265fd40860c91 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Wed, 15 Aug 2018 16:22:42 +0200 Subject: [PATCH 03/31] good till sending reports and ss-get ss-set ss-display ss-abort command cimi --- client/src/main/python/slipstream/Client.py | 33 ++++++++ .../python/slipstream/SlipStreamHttpClient.py | 56 +++++++++++++- .../slipstream/executors/MachineExecutor.py | 31 +++++++- .../executors/node/NodeDeploymentExecutor.py | 77 ++++++++++++++----- .../executors/node/NodeExecutorFactory.py | 2 +- .../python/slipstream/wrappers/BaseWrapper.py | 35 +++++---- client/src/main/python/ss-abort.py | 3 +- client/src/main/python/ss-display.py | 3 +- client/src/main/python/ss-get.py | 3 +- client/src/main/python/ss-set.py | 3 +- 10 files changed, 202 insertions(+), 44 deletions(-) diff --git a/client/src/main/python/slipstream/Client.py b/client/src/main/python/slipstream/Client.py index 6e1057c3..dc607f3f 100644 --- a/client/src/main/python/slipstream/Client.py +++ b/client/src/main/python/slipstream/Client.py @@ -90,6 +90,31 @@ def getRuntimeParameter(self, key): return value + def kb_getRuntimeParameter(self, key): + value = None + parts = self._getNodeName().split(NodeDecorator.NODE_MULTIPLICITY_SEPARATOR) + nodename = parts[0] + if self.no_block: + value = self.httpClient.kb_get_deployment_parameter(key, nodename) + else: + timer = 0 + while True: + value = self.httpClient.kb_get_deployment_parameter(key, nodename) + + if value is not None: + break + if self.timeout != 0 and timer >= self.timeout: + raise TimeoutException( + "Exceeded timeout limit of %s waiting for key '%s' " + "to be set" % (self.timeout, _key)) + print >> sys.stderr, "Waiting for %s" % _key + sys.stdout.flush() + sleepTime = 5 + time.sleep(sleepTime) + timer += sleepTime + + return value + def launchDeployment(self, params): """ @return: Run location @@ -176,6 +201,14 @@ def setRuntimeParameter(self, key, value): raise ClientError("value exceeds maximum length of %d characters" % self.VALUE_LENGTH_LIMIT) self.httpClient.setRuntimeParameter(_key, stripped_value) + def kb_setRuntimeParameter(self, key, value): + if NodeDecorator.NODE_PROPERTY_SEPARATOR in key: + nodename = None + else: + parts = self._getNodeName().split(NodeDecorator.NODE_MULTIPLICITY_SEPARATOR) + nodename = parts[0] + self.httpClient.kb_set_deployment_parameter(key, util.removeASCIIEscape(value), nodename) + def cancel_abort(self): # Global abort self.httpClient.unset_runtime_parameter(NodeDecorator.globalNamespacePrefix + NodeDecorator.ABORT_KEY, diff --git a/client/src/main/python/slipstream/SlipStreamHttpClient.py b/client/src/main/python/slipstream/SlipStreamHttpClient.py index 909390ba..94a055de 100644 --- a/client/src/main/python/slipstream/SlipStreamHttpClient.py +++ b/client/src/main/python/slipstream/SlipStreamHttpClient.py @@ -18,6 +18,7 @@ from __future__ import print_function import os +import uuid import json from collections import defaultdict @@ -38,6 +39,7 @@ class SlipStreamHttpClient(object): def __init__(self, configHolder): self.category = None self.run_dom = None + self.kb_deployment = None self.ignoreAbort = False self.username = '' self.password = '' @@ -137,7 +139,7 @@ def get_nodes_instances(self, cloud_service_name=None): ''' nodes_instances = {} - self._retrieveAndSetRun() + self._kb_retrieveAndSetRun() nodes_instances_runtime_parameters = \ DomExtractor.extract_nodes_instances_runtime_parameters(self.run_dom, cloud_service_name) @@ -168,6 +170,15 @@ def get_nodes_instances(self, cloud_service_name=None): return nodes_instances + def kb_get_node_instance(self, node_name): + type = self.kb_get_run_type() + if type in ('COMPONENT', 'IMAGE'): + return self.kb_deployment['module'] + elif type == 'APPLICATION': # FIXME more than the module returned + nodes = self.kb_deployment['module']['content']['nodes'] + return filter(lambda d: d['node'] == node_name, nodes)[0] + + def _get_nodename(self): 'Node name derived from the node instance name.' return self.node_instance_name.split( @@ -194,6 +205,27 @@ def _retrieveAndSetRun(self): _, run = self._retrieve(url) self.run_dom = etree.fromstring(run.encode('utf-8')) + @staticmethod + def lookup_recursively_module(module, keys): # FIXME: support for array of maps + temp = module + for k in keys[:-1]: + temp = temp.get(k, {}) + value = temp.get(keys[-1]) + if value: + return value + module_parent = module.get('content', {}).get('parent') + if module_parent: + return SlipStreamHttpClient.lookup_recursively_module(module_parent, keys) + + def kb_get_run_type(self): + self._kb_retrieveAndSetRun() + return self.kb_deployment['module']['type'] + + def _kb_retrieveAndSetRun(self): + if self.kb_deployment is None: + self.kb_deployment = self.api.cimi_get(self.diid).json + return self.kb_deployment + def _retrieve(self, url): return self._httpGet(url, 'application/xml') @@ -207,6 +239,10 @@ def complete_state(self, node_instance_name): url += SlipStreamHttpClient.URL_IGNORE_ABORT_ATTRIBUTE_QUERY return self._httpPost(url, 'reset', 'text/plain') + def kb_complete_state(self, node_instance_name): + return self.kb_set_deployment_parameter( + NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY, 'SendingReports') + def terminate_run(self): return self._httpDelete(self.run_url) @@ -268,6 +304,24 @@ def getRuntimeParameter(self, key, ignoreAbort=False): return content.strip().strip('"').strip("'") + @staticmethod + def kb_from_data_uuid(text): + class NullNameSpace: + bytes = b'' + + return str(uuid.uuid3(NullNameSpace, text)) + + def kb_get_deployment_parameter(self, param_name, node_id=None): + param_id = ':'.join(item or '' for item in [self.diid, node_id, param_name]) + deployment_parameter_href = 'deployment-parameter/' + self.kb_from_data_uuid(param_id) + return self.api.cimi_get(deployment_parameter_href).json.get('value') + + def kb_set_deployment_parameter(self, param_name, value, node_id=None): + param_id = ':'.join(item or '' for item in [self.diid, node_id, param_name]) + print('debug kb_set_deployment_parameter param_id={}'.format(param_id)) + deployment_parameter_href = 'deployment-parameter/' + self.kb_from_data_uuid(param_id) + return self.api.cimi_edit(deployment_parameter_href, {'value': value}) + def setRuntimeParameter(self, key, value, ignoreAbort=False): url = self.run_url + '/' + key if self.ignoreAbort or ignoreAbort: diff --git a/client/src/main/python/slipstream/executors/MachineExecutor.py b/client/src/main/python/slipstream/executors/MachineExecutor.py index d5378ea7..585b37bd 100644 --- a/client/src/main/python/slipstream/executors/MachineExecutor.py +++ b/client/src/main/python/slipstream/executors/MachineExecutor.py @@ -75,7 +75,7 @@ def _execute(self): state = self._get_state() while True: self._execute_state(state) - self._complete_state(state) + self._kb_complete_state(state) state = self._wait_for_next_state(state) def _get_state(self): @@ -129,7 +129,11 @@ def _state_not_implemented(self, state): def _complete_state(self, state): if self._need_to_complete(state): - self.wrapper.complete_state() + self.wrapper.kb_complete_state() + + def _kb_complete_state(self, state): + if self._need_to_complete(state): + self.wrapper.kb_complete_state() @staticmethod def _failure_msg_from_exception(exception): @@ -177,7 +181,7 @@ def _get_sleep_time(self, state): return self.WAIT_NEXT_STATE_SHORT def _retrieve_my_node_instance(self): - node_instance = self.wrapper.get_my_node_instance() + node_instance = self.wrapper.kb_get_my_node_instance() if node_instance is None: raise ExecutionException("Couldn't get the node instance for the current VM.") return node_instance @@ -201,6 +205,10 @@ def _execute_execute_target(self): self._execute_target('execute', abort_on_err=True) self._set_need_to_send_reports() + def _kb_execute_execute_target(self): + self._kb_execute_target('deployment', abort_on_err=True) + self._set_need_to_send_reports() + def _execute_target(self, target_name, exports=None, abort_on_err=False, ssdisplay=True, ignore_abort=False): target = self.node_instance.get_image_target(target_name) @@ -237,6 +245,23 @@ def _execute_target(self, target_name, exports=None, abort_on_err=False, ssdispl else: util.printAndFlush('Nothing to do for script: %s' % full_target_name) + def _kb_get_target(self, target_name): + return self.node_instance['content'].get('targets', {}).get(target_name) + + def _kb_execute_target(self, target_name, exports=None, abort_on_err=False, ssdisplay=True, ignore_abort=False): + target = self._kb_get_target(target_name) + + if target is None: + util.printAndFlush('Nothing to do for script: %s' % target_name) + return + + full_target_name = '%s:%s' % (self.node_instance['name'], target_name) + message = "Executing script '%s'" % full_target_name + util.printStep(message) + + fail_msg = "Failed running '%s' script on '%s'" % (target_name, self._get_node_instance_name()) + self._launch_script(target, exports, abort_on_err, ignore_abort, fail_msg, target_name) + def _need_to_execute_build_step(self, target, subtarget): return MachineExecutor.need_to_execute_build_step(self._get_node_instance(), target, subtarget) diff --git a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py index cffd9460..0af1dd48 100644 --- a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py +++ b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py @@ -70,31 +70,38 @@ def onProvisioning(self): def onExecuting(self): util.printAction('Executing') - self._get_recovery_mode() - if self._is_recovery_mode(): - util.printDetail("Recovery mode enabled, recipes will not be executed.", - verboseThreshold=util.VERBOSE_LEVEL_QUIET) - return - - if self._skip_execute_due_to_vertical_scaling: - util.printDetail("Vertical scaling: skipping execution of execute targets.", - verboseThreshold=util.VERBOSE_LEVEL_QUIET) - self._skip_execute_due_to_vertical_scaling = False - return - - if not self.wrapper.is_scale_state_operational(): - self._execute_build_recipes() - self._execute_execute_target() - else: - self._execute_scale_action_target() + #self._get_recovery_mode() + #if self._is_recovery_mode(): + # util.printDetail("Recovery mode enabled, recipes will not be executed.", + # verboseThreshold=util.VERBOSE_LEVEL_QUIET) + # return + + #if self._skip_execute_due_to_vertical_scaling: + # util.printDetail("Vertical scaling: skipping execution of execute targets.", + # verboseThreshold=util.VERBOSE_LEVEL_QUIET) + # self._skip_execute_due_to_vertical_scaling = False + # return + + #if not self.wrapper.is_scale_state_operational(): + self._kb_execute_build_recipes() + self._kb_execute_execute_target() + #else: + # self._execute_scale_action_target() def _execute_build_recipes(self): util.printDetail('Executing build recipes') - self._execute_target(NodeDecorator.NODE_PRERECIPE, abort_on_err=True) + self._execute_target('preinstall', abort_on_err=True) self._install_user_packages() self._execute_target(NodeDecorator.NODE_RECIPE, abort_on_err=True) + def _kb_execute_build_recipes(self): + util.printDetail('Executing build recipes') + + self._kb_execute_target('preinstall', abort_on_err=True) + self._kb_install_user_packages() + self._kb_execute_target('postinstall', abort_on_err=True) + def _install_user_packages(self): util.printAndFlush('Installing packages') @@ -113,17 +120,35 @@ def _install_user_packages(self): else: util.printAndFlush('No packages to install') + def _kb_install_user_packages(self): + util.printAndFlush('Installing packages') + + # if self.is_image_built(): + # util.printAndFlush('Component already built. No packages to install') + # return + + packages = self._kb_get_target('packages') + if packages: + message = 'Installing packages: %s' % ', '.join(packages) + fail_msg = "Failed installing packages on '%s'" % self._get_node_instance_name() + util.printStep(message) + #self.wrapper.set_statecustom(message) + cmd = util.get_packages_install_command('ubuntu', packages) + self._launch_script('#!/bin/sh -xe\n%s' % cmd, fail_msg=fail_msg, name='Install packages') + else: + util.printAndFlush('No packages to install') + @override def onSendingReports(self): util.printAction('Sending report') if self._need_to_send_reports(): - self._execute_report_target_and_send_reports() + self._kb_execute_report_target_and_send_reports() self._unset_need_to_send_reports() else: util.printDetail('INFO: Conditionally skipped sending reports.', verboseThreshold=util.VERBOSE_LEVEL_QUIET) - self.wrapper.set_scale_state_operational() + # self.wrapper.set_scale_state_operational() def _execute_report_target_and_send_reports(self): exports = {'SLIPSTREAM_REPORT_DIR': util.get_platform_reports_dir()} @@ -137,6 +162,18 @@ def _execute_report_target_and_send_reports(self): finally: super(NodeDeploymentExecutor, self).onSendingReports() + def _kb_execute_report_target_and_send_reports(self): + exports = {'SLIPSTREAM_REPORT_DIR': util.get_platform_reports_dir()} + try: + self._kb_execute_target('reporting', exports=exports, ssdisplay=False, ignore_abort=True) + except ExecutionException as ex: + util.printDetail("Failed executing 'report' with: \n%s" % str(ex), + verboseLevel=self.verboseLevel, + verboseThreshold=util.VERBOSE_LEVEL_NORMAL) + raise + finally: + super(NodeDeploymentExecutor, self).onSendingReports() + @override def onReady(self): super(NodeDeploymentExecutor, self).onReady() diff --git a/client/src/main/python/slipstream/executors/node/NodeExecutorFactory.py b/client/src/main/python/slipstream/executors/node/NodeExecutorFactory.py index 5acb8415..3badc2a3 100644 --- a/client/src/main/python/slipstream/executors/node/NodeExecutorFactory.py +++ b/client/src/main/python/slipstream/executors/node/NodeExecutorFactory.py @@ -38,7 +38,7 @@ class NodeExecutorFactory: @staticmethod def createExecutor(configHolder): wrapper = BaseWrapper(configHolder) - runType = wrapper.get_run_type() + runType = 'Run' # wrapper.get_run_type() return loadModule(get_executor_module_name(runType)). \ getExecutor(wrapper, configHolder) diff --git a/client/src/main/python/slipstream/wrappers/BaseWrapper.py b/client/src/main/python/slipstream/wrappers/BaseWrapper.py index 656fdfc9..0c0f184e 100644 --- a/client/src/main/python/slipstream/wrappers/BaseWrapper.py +++ b/client/src/main/python/slipstream/wrappers/BaseWrapper.py @@ -168,9 +168,13 @@ def complete_state(self, node_instance_name=None): node_instance_name = self.get_my_node_instance_name() self._ss_client.complete_state(node_instance_name) + def kb_complete_state(self, node_instance_name=None): + if not node_instance_name: + node_instance_name = self.get_my_node_instance_name() + self._ss_client.kb_complete_state(node_instance_name) + def fail(self, message): - key = self._qualifyKey(NodeDecorator.ABORT_KEY) - self._fail(key, message) + self._ss_client.kb_set_deployment_parameter(NodeDecorator.ABORT_KEY, message, self.get_my_node_instance_name()) def fail_global(self, message): key = NodeDecorator.globalNamespacePrefix + NodeDecorator.ABORT_KEY @@ -180,23 +184,20 @@ def _fail(self, key, message): util.printError('Failing... %s' % message) traceback.print_exc() value = util.truncate_middle(Client.VALUE_LENGTH_LIMIT, message, '\n(truncated)\n') - self._ss_client.setRuntimeParameter(key, value) + self._ss_client.kb_set_deployment_parameter(key, value) def getState(self): - key = NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY - return self._get_runtime_parameter(key) + return self._ss_client.kb_get_deployment_parameter( + NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY) def get_recovery_mode(self): key = NodeDecorator.globalNamespacePrefix + NodeDecorator.RECOVERY_MODE_KEY return util.str2bool(self._get_runtime_parameter(key)) def isAbort(self): - key = NodeDecorator.globalNamespacePrefix + NodeDecorator.ABORT_KEY - try: - value = self._get_runtime_parameter(key, True) - except Exceptions.NotYetSetException: - value = '' - return (value and True) or False + value = self._ss_client.kb_get_deployment_parameter( + NodeDecorator.globalNamespacePrefix + NodeDecorator.ABORT_KEY) + return value is not None def get_max_iaas_workers(self): """Available only on orchestrator. @@ -557,10 +558,10 @@ def wait_scale_iaas_done(self): timeout_at = 0 # no timeout self._log('Waiting for Orchestrator to finish scaling this node instance (no timeout).') - node_instances = [self.get_my_node_instance()] - - self._wait_rtp_equals(node_instances, NodeDecorator.SCALE_IAAS_DONE_SUCCESS, - self.get_scale_iaas_done, timeout_at) + # node_instances = [self.get_my_node_instance()] + # + # self._wait_rtp_equals(node_instances, NodeDecorator.SCALE_IAAS_DONE_SUCCESS, + # self.get_scale_iaas_done, timeout_at) self._log('All node instances finished pre-scaling.') @@ -600,6 +601,10 @@ def get_my_node_instance(self): node_name = self.get_my_node_instance_name() return self._get_nodes_instances_with_orchestrators().get(node_name) + def kb_get_my_node_instance(self): + node_name = self.get_my_node_instance_name() + return self._ss_client.kb_get_node_instance(node_name) + def discard_run_locally(self): self._ss_client.discard_run() diff --git a/client/src/main/python/ss-abort.py b/client/src/main/python/ss-abort.py index bc03ba3f..0c0d78de 100755 --- a/client/src/main/python/ss-abort.py +++ b/client/src/main/python/ss-abort.py @@ -69,7 +69,8 @@ def doWork(self): else: value = truncate_middle(Client.VALUE_LENGTH_LIMIT, self.reason, '\n(truncated)\n') - client.setRuntimeParameter(NodeDecorator.ABORT_KEY, value) + client.kb_setRuntimeParameter(NodeDecorator.ABORT_KEY, value) + if __name__ == "__main__": try: diff --git a/client/src/main/python/ss-display.py b/client/src/main/python/ss-display.py index dac73ab2..87d02272 100755 --- a/client/src/main/python/ss-display.py +++ b/client/src/main/python/ss-display.py @@ -57,7 +57,8 @@ def parse(self): def doWork(self): ch = ConfigHolder(self.options) client = Client(ch) - client.setRuntimeParameter('statecustom', self.value) + client.kb_setRuntimeParameter('statecustom', self.value) + if __name__ == "__main__": try: diff --git a/client/src/main/python/ss-get.py b/client/src/main/python/ss-get.py index fc32033a..11917e24 100755 --- a/client/src/main/python/ss-get.py +++ b/client/src/main/python/ss-get.py @@ -68,9 +68,10 @@ def _checkArgs(self): def doWork(self): ch = ConfigHolder(self.options) client = Client(ch) - value = client.getRuntimeParameter(self.key) + value = client.kb_getRuntimeParameter(self.key) print(value if value is not None else '') + if __name__ == "__main__": try: MainProgram() diff --git a/client/src/main/python/ss-set.py b/client/src/main/python/ss-set.py index 97259dde..be37cd3c 100755 --- a/client/src/main/python/ss-set.py +++ b/client/src/main/python/ss-set.py @@ -60,7 +60,8 @@ def parse(self): def doWork(self): ch = ConfigHolder(self.options) client = Client(ch) - client.setRuntimeParameter(self.key, self.value) + client.kb_setRuntimeParameter(self.key, self.value) + if __name__ == "__main__": try: From d91435f5264de156cde6299be8103bff3c6b22ea Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 13:17:05 +0200 Subject: [PATCH 04/31] Update BaseCloudConnector.py --- .../cloudconnectors/BaseCloudConnector.py | 106 ++++++++++++------ 1 file changed, 69 insertions(+), 37 deletions(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 2794f3fc..d8e6fb60 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -312,7 +312,7 @@ def __init__(self, configHolder): self.__vms = {} - self.__cloud = os.environ[util.ENV_CONNECTOR_INSTANCE] + self.__cloud = os.environ.get(util.ENV_CONNECTOR_INSTANCE) self.__init_threading_related() @@ -455,7 +455,7 @@ def __wait_nodes_startup_tasks_finished(self): if self.__tasks_runnner != None: self.__tasks_runnner.wait_tasks_processed() - def __start_node_instance_and_client(self, user_info, node_instance): + def __start_node_instance_and_client(self, user_info, node_instance, cimi_deployment_prototype=False): node_instance_name = node_instance.get_name() self._print_detail("Starting instance: %s" % node_instance_name) @@ -464,11 +464,11 @@ def __start_node_instance_and_client(self, user_info, node_instance): node_instance, self._generate_vm_name(node_instance_name)) - self.__add_vm(vm, node_instance) + self.__add_vm(vm, node_instance, cimi_deployment_prototype) if not self.has_capability(self.CAPABILITY_DIRECT_IP_ASSIGNMENT): vm = self._wait_and_get_instance_ip_address(vm) - self.__add_vm(vm, node_instance) + self.__add_vm(vm, node_instance, cimi_deployment_prototype) if not self.is_build_image(): if not node_instance.is_windows() and not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION): @@ -486,11 +486,11 @@ def _get_vm(self, name): except KeyError: raise Exceptions.NotFoundError("VM '%s' not found." % name) - def __add_vm(self, vm, node_instance): + def __add_vm(self, vm, node_instance, cimi_deployment_prototype=False): name = node_instance.get_name() with lock: self.__vms[name] = vm - self._publish_vm_info(vm, node_instance) + self._publish_vm_info(vm, node_instance, cimi_deployment_prototype) def __del_vm(self, name): try: @@ -498,22 +498,39 @@ def __del_vm(self, name): except KeyError: util.printDetail("Failed locally removing VM '%s'. Not found." % name) - def _publish_vm_info(self, vm, node_instance): + def _publish_vm_info(self, vm, node_instance, cimi_deployment_prototype=False): instance_name = node_instance.get_name() vm_id = self._vm_get_id(vm) vm_ip = self._vm_get_ip(vm) with lock: already_published = self.__already_published[instance_name] - if vm_id and 'id' not in already_published: - self._publish_vm_id(instance_name, vm_id) - already_published.add('id') - if vm_ip and 'ip' not in already_published: - self._publish_vm_ip(instance_name, vm_ip) - already_published.add('ip') - if node_instance and vm_ip and 'ssh' not in already_published: - self._publish_url_ssh(vm, node_instance) - already_published.add('ssh') + if cimi_deployment_prototype: + if vm_id and 'id' not in already_published: + node_instance.set_cloud_node_id(vm_id) + already_published.add('id') + if vm_ip and 'ip' not in already_published: + node_instance.set_cloud_node_ip(vm_ip) + already_published.add('ip') + if node_instance and vm_ip and 'ssh' not in already_published: + if node_instance: + vm_ip = self._vm_get_ip(vm) or '' + ssh_username, ssh_password = self.__get_vm_username_password(node_instance) + node_instance.set_cloud_node_ssh_url('ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) + node_instance.set_cloud_node_ssh_password(ssh_password) + + if ssh_username and ssh_password: + already_published.add('ssh') + else: + if vm_id and 'id' not in already_published: + self._publish_vm_id(instance_name, vm_id) + already_published.add('id') + if vm_ip and 'ip' not in already_published: + self._publish_vm_ip(instance_name, vm_ip) + already_published.add('ip') + if node_instance and vm_ip and 'ssh' not in already_published: + self._publish_url_ssh(vm, node_instance) + already_published.add('ssh') def _publish_vm_id(self, instance_name, vm_id): # Needed for thread safety @@ -829,25 +846,33 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, addEnvironmentVariableCommand = 'export' script += '#!/bin/sh -ex\n' - regex = 'SLIPSTREAM_' - if self.is_start_orchestrator(): - regex += '|CLOUDCONNECTOR_' - env_matcher = re.compile(regex) - if pre_export: script += '%s\n' % pre_export - for var, val in os.environ.items(): - if env_matcher.match(var) and var != util.ENV_NODE_INSTANCE_NAME: + deployment_context = node_instance.get_deployment_context() + if deployment_context: + # deployment prototype + for var, val in deployment_context.items(): if re.search(' ', val): val = '"%s"' % val - script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, - val) + script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, val) + else: + regex = 'SLIPSTREAM_' + if self.is_start_orchestrator(): + regex += '|CLOUDCONNECTOR_' + env_matcher = re.compile(regex) + for var, val in os.environ.items(): + if env_matcher.match(var) and var != util.ENV_NODE_INSTANCE_NAME: + if re.search(' ', val): + val = '"%s"' % val + script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, + val) script += '%s %s=%s\n' % (addEnvironmentVariableCommand, util.ENV_NODE_INSTANCE_NAME, node_instance_name) + if pre_bootstrap: script += '%s\n' % pre_bootstrap @@ -862,12 +887,19 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, def _build_slipstream_bootstrap_command(self, node_instance, username=None): instance_name = node_instance.get_name() + cimi_deployment_prototype = True + bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') + + if bootstrap_url is None: + bootstrap_url = util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') + cimi_deployment_prototype = False + if node_instance.is_windows(): - return self.__build_slipstream_bootstrap_command_for_windows(instance_name) + return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url, cimi_deployment_prototype) else: - return self.__build_slipstream_bootstrap_command_for_linux(instance_name) + return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url, cimi_deployment_prototype) - def __build_slipstream_bootstrap_command_for_windows(self, instance_name): + def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url, cimi_deployment_prototype): command = 'If Not Exist %(reports)s mkdir %(reports)s\n' command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n' @@ -883,35 +915,35 @@ def __build_slipstream_bootstrap_command_for_windows(self, instance_name): command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n' - return command % self._get_bootstrap_command_replacements_for_windows(instance_name) + return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url, cimi_deployment_prototype) - def __build_slipstream_bootstrap_command_for_linux(self, instance_name): + def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url, cimi_deployment_prototype): command = 'mkdir -p %(reports)s %(ss_home)s; ' command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 ' command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) ' command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1' - return command % self._get_bootstrap_command_replacements_for_linux(instance_name) + return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url, cimi_deployment_prototype) - def _get_bootstrap_command_replacements_for_linux(self, instance_name): + def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url, cimi_deployment_prototype): return { 'reports': util.REPORTSDIR, 'bootstrap': os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'), - 'bootstrapUrl': util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN'), + 'bootstrapUrl': bootstrap_url, 'ss_home': util.SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': self._get_machine_executor_type() + 'machine_executor': 'node' if cimi_deployment_prototype else self._get_machine_executor_type() } - def _get_bootstrap_command_replacements_for_windows(self, instance_name): + def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url, cimi_deployment_prototype): return { 'reports': util.WINDOWS_REPORTSDIR, 'bootstrap': '\\'.join([util.WINDOWS_SLIPSTREAM_HOME, 'slipstream.bootstrap']), - 'bootstrapUrl': util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN'), + 'bootstrapUrl': bootstrap_url, 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': self._get_machine_executor_type() + 'machine_executor': 'node' if cimi_deployment_prototype else self._get_machine_executor_type() } def _get_machine_executor_type(self): From 022bbac70c79c3758616a026620d9c108c959ad0 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 13:23:14 +0200 Subject: [PATCH 05/31] Update NodeDecorator.py --- client/src/main/python/slipstream/NodeDecorator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/main/python/slipstream/NodeDecorator.py b/client/src/main/python/slipstream/NodeDecorator.py index 4165e351..67361f08 100644 --- a/client/src/main/python/slipstream/NodeDecorator.py +++ b/client/src/main/python/slipstream/NodeDecorator.py @@ -83,10 +83,14 @@ class NodeDecorator(object): SCALE_DISK_ATTACHED_DEVICE = 'disk.attached.device' SCALE_DISK_DETACH_DEVICE = 'disk.detach.device' INSTANCEID_KEY = 'instanceid' + CLOUD_NODE_SSH_URL_KEY = 'cloud.node.ssh.url' + CLOUD_NODE_SSH_PASSWORD_KEY = 'cloud.node.ssh.password' + CLOUD_NODE_SSH_KEYPAIR_NAME_KEY = 'cloud.node.ssh.keypair.name' CLOUDSERVICE_KEY = 'cloudservice' SECURITY_GROUPS_KEY = 'security.groups' MAX_PROVISIONING_FAILURES_KEY = 'max-provisioning-failures' NATIVE_CONTEXTUALIZATION_KEY = 'native-contextualization' + DEPLOYMENT_CONTEXT_KEY = 'deployment.context' SECURITY_GROUP_ALLOW_ALL_NAME = 'slipstream_managed' SECURITY_GROUP_ALLOW_ALL_DESCRIPTION = 'Security group created by SlipStream which allows all kind of traffic.' From a87177380fe343b4a2069460aad227be1edfe4f6 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 13:23:24 +0200 Subject: [PATCH 06/31] Update NodeInstance.py --- .../main/python/slipstream/NodeInstance.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index 71a68125..c03dfa44 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -210,3 +210,29 @@ def get_disk_attach_size(self): def get_disk_detach_device(self): return self.__get(NodeDecorator.SCALE_DISK_DETACH_DEVICE) + def set_deployment_context(self, context=None): + self.__set(NodeDecorator.DEPLOYMENT_CONTEXT_KEY, context) + + def get_deployment_context(self): + return self.__get(NodeDecorator.DEPLOYMENT_CONTEXT_KEY, {}) + + def set_cloud_node_ssh_url(self, url): + self.__set(NodeDecorator.CLOUD_NODE_SSH_URL_KEY, url) + + def get_cloud_node_ssh_url(self): + return self.__get(NodeDecorator.CLOUD_NODE_SSH_URL_KEY) + + def set_cloud_node_ssh_password(self, password): + self.__set(NodeDecorator.CLOUD_NODE_SSH_PASSWORD_KEY, password) + + def get_cloud_node_ssh_password(self): + return self.__get(NodeDecorator.CLOUD_NODE_SSH_PASSWORD_KEY) + + def set_cloud_node_ssh_keypair_name(self, password): + self.__set(NodeDecorator.CLOUD_NODE_SSH_KEYPAIR_NAME_KEY, password) + + def get_cloud_node_ssh_keypair_name(self): + return self.__get(NodeDecorator.CLOUD_NODE_SSH_KEYPAIR_NAME_KEY) + + + From 5d69845903894fdbceee3c8966d804992e5d872a Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 13:31:29 +0200 Subject: [PATCH 07/31] Update NodeDecorator.py --- client/src/main/python/slipstream/NodeDecorator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/python/slipstream/NodeDecorator.py b/client/src/main/python/slipstream/NodeDecorator.py index 67361f08..27f85e17 100644 --- a/client/src/main/python/slipstream/NodeDecorator.py +++ b/client/src/main/python/slipstream/NodeDecorator.py @@ -83,6 +83,7 @@ class NodeDecorator(object): SCALE_DISK_ATTACHED_DEVICE = 'disk.attached.device' SCALE_DISK_DETACH_DEVICE = 'disk.detach.device' INSTANCEID_KEY = 'instanceid' + CLOUD_NODE_IP_KEY = 'cloud.node.ip' CLOUD_NODE_SSH_URL_KEY = 'cloud.node.ssh.url' CLOUD_NODE_SSH_PASSWORD_KEY = 'cloud.node.ssh.password' CLOUD_NODE_SSH_KEYPAIR_NAME_KEY = 'cloud.node.ssh.keypair.name' From c4d21174a69b4fdfd7b36e1dd37a78b39f410831 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 13:32:00 +0200 Subject: [PATCH 08/31] Update NodeInstance.py --- client/src/main/python/slipstream/NodeInstance.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index c03dfa44..556170f2 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -216,6 +216,12 @@ def set_deployment_context(self, context=None): def get_deployment_context(self): return self.__get(NodeDecorator.DEPLOYMENT_CONTEXT_KEY, {}) + def set_cloud_node_ip(self, ip): + self.__set(NodeDecorator.CLOUD_NODE_IP_KEY, ip) + + def get_cloud_node_ip(self): + return self.__get(NodeDecorator.CLOUD_NODE_IP_KEY) + def set_cloud_node_ssh_url(self, url): self.__set(NodeDecorator.CLOUD_NODE_SSH_URL_KEY, url) From 7c4694ba90ed1ee5c1beb0b209b958a627e51e45 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 14:19:25 +0200 Subject: [PATCH 09/31] Update BaseCloudConnector.py --- .../cloudconnectors/BaseCloudConnector.py | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index d8e6fb60..129ed336 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -331,6 +331,8 @@ def __init__(self, configHolder): pass self._user_info = None + + self.cimi_deployment_prototype = False @property def user_info(self): @@ -430,6 +432,7 @@ def start_nodes_and_clients(self, user_info, nodes_instances, init_extra_kwargs= self._initialization(user_info, **init_extra_kwargs) self.__set_contextualization_capabilities(user_info) self.__create_allow_all_security_group_if_needed(nodes_instances) + self.cimi_deployment_prototype = nodes_instances.values()[0].get_deployment_context() is not None try: self.__start_nodes_instantiation_tasks_wait_finished(user_info, nodes_instances) @@ -455,7 +458,7 @@ def __wait_nodes_startup_tasks_finished(self): if self.__tasks_runnner != None: self.__tasks_runnner.wait_tasks_processed() - def __start_node_instance_and_client(self, user_info, node_instance, cimi_deployment_prototype=False): + def __start_node_instance_and_client(self, user_info, node_instance): node_instance_name = node_instance.get_name() self._print_detail("Starting instance: %s" % node_instance_name) @@ -464,11 +467,11 @@ def __start_node_instance_and_client(self, user_info, node_instance, cimi_deploy node_instance, self._generate_vm_name(node_instance_name)) - self.__add_vm(vm, node_instance, cimi_deployment_prototype) + self.__add_vm(vm, node_instance) if not self.has_capability(self.CAPABILITY_DIRECT_IP_ASSIGNMENT): vm = self._wait_and_get_instance_ip_address(vm) - self.__add_vm(vm, node_instance, cimi_deployment_prototype) + self.__add_vm(vm, node_instance) if not self.is_build_image(): if not node_instance.is_windows() and not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION): @@ -486,11 +489,11 @@ def _get_vm(self, name): except KeyError: raise Exceptions.NotFoundError("VM '%s' not found." % name) - def __add_vm(self, vm, node_instance, cimi_deployment_prototype=False): + def __add_vm(self, vm, node_instance): name = node_instance.get_name() with lock: self.__vms[name] = vm - self._publish_vm_info(vm, node_instance, cimi_deployment_prototype) + self._publish_vm_info(vm, node_instance) def __del_vm(self, name): try: @@ -498,14 +501,13 @@ def __del_vm(self, name): except KeyError: util.printDetail("Failed locally removing VM '%s'. Not found." % name) - def _publish_vm_info(self, vm, node_instance, cimi_deployment_prototype=False): + def _publish_vm_info(self, vm, node_instance): instance_name = node_instance.get_name() vm_id = self._vm_get_id(vm) vm_ip = self._vm_get_ip(vm) - with lock: already_published = self.__already_published[instance_name] - if cimi_deployment_prototype: + if self.cimi_deployment_prototype: if vm_id and 'id' not in already_published: node_instance.set_cloud_node_id(vm_id) already_published.add('id') @@ -518,7 +520,7 @@ def _publish_vm_info(self, vm, node_instance, cimi_deployment_prototype=False): ssh_username, ssh_password = self.__get_vm_username_password(node_instance) node_instance.set_cloud_node_ssh_url('ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) node_instance.set_cloud_node_ssh_password(ssh_password) - + if ssh_username and ssh_password: already_published.add('ssh') else: @@ -848,11 +850,9 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, if pre_export: script += '%s\n' % pre_export - - deployment_context = node_instance.get_deployment_context() - if deployment_context: - # deployment prototype - for var, val in deployment_context.items(): + + if self.cimi_deployment_prototype: + for var, val in node_instance.get_deployment_context().items(): if re.search(' ', val): val = '"%s"' % val script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, val) @@ -887,19 +887,17 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, def _build_slipstream_bootstrap_command(self, node_instance, username=None): instance_name = node_instance.get_name() - cimi_deployment_prototype = True - bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') - - if bootstrap_url is None: + if node_instance.get_deployment_context(): + bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') + else: bootstrap_url = util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') - cimi_deployment_prototype = False if node_instance.is_windows(): - return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url, cimi_deployment_prototype) + return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url) else: - return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url, cimi_deployment_prototype) + return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url) - def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url, cimi_deployment_prototype): + def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url): command = 'If Not Exist %(reports)s mkdir %(reports)s\n' command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n' @@ -915,25 +913,25 @@ def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootst command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n' - return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url, cimi_deployment_prototype) + return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url) - def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url, cimi_deployment_prototype): + def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url): command = 'mkdir -p %(reports)s %(ss_home)s; ' command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 ' command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) ' command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1' - return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url, cimi_deployment_prototype) + return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url) - def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url, cimi_deployment_prototype): + def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url): return { 'reports': util.REPORTSDIR, 'bootstrap': os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'), 'bootstrapUrl': bootstrap_url, 'ss_home': util.SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': 'node' if cimi_deployment_prototype else self._get_machine_executor_type() + 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url, cimi_deployment_prototype): @@ -943,7 +941,7 @@ def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstr 'bootstrapUrl': bootstrap_url, 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': 'node' if cimi_deployment_prototype else self._get_machine_executor_type() + 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } def _get_machine_executor_type(self): From bf63c73870e331e6e8ce77fe71c7f96febf3e610 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 14:28:24 +0200 Subject: [PATCH 10/31] Update BaseCloudConnector.py --- .../cloudconnectors/BaseCloudConnector.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 129ed336..2c5f70e9 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -887,17 +887,12 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, def _build_slipstream_bootstrap_command(self, node_instance, username=None): instance_name = node_instance.get_name() - if node_instance.get_deployment_context(): - bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') - else: - bootstrap_url = util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') - if node_instance.is_windows(): - return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url) + return self.__build_slipstream_bootstrap_command_for_windows(instance_name) else: - return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url) + return self.__build_slipstream_bootstrap_command_for_linux(instance_name) - def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url): + def __build_slipstream_bootstrap_command_for_windows(self, instance_name): command = 'If Not Exist %(reports)s mkdir %(reports)s\n' command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n' @@ -913,35 +908,43 @@ def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootst command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n' - return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url) + return command % self._get_bootstrap_command_replacements_for_windows(instance_name) - def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url): + def __build_slipstream_bootstrap_command_for_linux(self, instance_name): command = 'mkdir -p %(reports)s %(ss_home)s; ' command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 ' command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) ' command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1' - return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url) + return command % self._get_bootstrap_command_replacements_for_linux(instance_name) - def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url): + + def __get_bootstrap_url(self): + if self.cimi_deployment_prototype: + return node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') + else: + return util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') + + def _get_bootstrap_command_replacements_for_linux(self, instance_name): return { 'reports': util.REPORTSDIR, 'bootstrap': os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'), - 'bootstrapUrl': bootstrap_url, + 'bootstrapUrl': self.__get_bootstrap_url(), 'ss_home': util.SLIPSTREAM_HOME, 'nodename': instance_name, 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } - def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url, cimi_deployment_prototype): + def _get_bootstrap_command_replacements_for_windows(self, instance_name): return { 'reports': util.WINDOWS_REPORTSDIR, 'bootstrap': '\\'.join([util.WINDOWS_SLIPSTREAM_HOME, 'slipstream.bootstrap']), - 'bootstrapUrl': bootstrap_url, + 'bootstrapUrl': self.__get_bootstrap_url(), 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() + 'machine_executor': 'node' if self. + _prototype else self._get_machine_executor_type() } def _get_machine_executor_type(self): From fafcd54beed6b3c742f97b475e1f1da25919ab67 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 14:30:52 +0200 Subject: [PATCH 11/31] Update BaseCloudConnector.py --- .../python/slipstream/cloudconnectors/BaseCloudConnector.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 2c5f70e9..fcd1e7d8 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -850,7 +850,7 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, if pre_export: script += '%s\n' % pre_export - + if self.cimi_deployment_prototype: for var, val in node_instance.get_deployment_context().items(): if re.search(' ', val): @@ -943,8 +943,7 @@ def _get_bootstrap_command_replacements_for_windows(self, instance_name): 'bootstrapUrl': self.__get_bootstrap_url(), 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': 'node' if self. - _prototype else self._get_machine_executor_type() + 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } def _get_machine_executor_type(self): From 4810e2a23208cc7593ec8d0279ec63fb92089975 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 15:08:14 +0200 Subject: [PATCH 12/31] Update BaseCloudConnector.py --- .../cloudconnectors/BaseCloudConnector.py | 85 +++++++++---------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index fcd1e7d8..28c5b20f 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -33,7 +33,7 @@ from slipstream.listeners.SimplePrintListener import SimplePrintListener from slipstream.listeners.SlipStreamClientListenerAdapter import SlipStreamClientListenerAdapter from slipstream.utils.ssh import remoteRunScriptNohup, waitUntilSshCanConnectOrTimeout, remoteRunScript, \ - generate_keypair, remoteRunCommand + generate_keypair, remoteRunCommand from slipstream.utils.tasksrunner import TasksRunner from slipstream.cloudconnectors.VmScaler import VmScaler from slipstream.wrappers.BaseWrapper import NodeInfoPublisher @@ -46,7 +46,7 @@ class BaseCloudConnector(object): -# ----- METHODS THAT CAN/SHOULD BE IMPLEMENTED IN CONNECTORS ----- + # ----- METHODS THAT CAN/SHOULD BE IMPLEMENTED IN CONNECTORS ----- def _initialization(self, user_info): """This method is called once before calling any others methods of the connector. @@ -219,15 +219,15 @@ def _resize(self, node_instance): # Example code # Cloud VM id. - #vm_id = node_instance.get_instance_id() + # vm_id = node_instance.get_instance_id() # RAM in GB. - #ram = node_instance.get_ram() + # ram = node_instance.get_ram() # Number of CPUs. - #cpu = node_instance.get_cpu() + # cpu = node_instance.get_cpu() # In case cloud uses T-short sizes. - #instance_type = node_instance.get_instance_type() + # instance_type = node_instance.get_instance_type() # IaaS calls go here. @@ -243,17 +243,17 @@ def _attach_disk(self, node_instance): # Example code - #device_name = '' + # device_name = '' # Cloud VM id. - #vm_id = node_instance.get_instance_id() + # vm_id = node_instance.get_instance_id() # Size of the disk to attach (in GB). - #disk_size_GB = node_instance.get_disk_attach_size() + # disk_size_GB = node_instance.get_disk_attach_size() # IaaS calls go here. - #return device_name + # return device_name raise NotImplementedError() @@ -266,16 +266,16 @@ def _detach_disk(self, node_instance): # Example code # Cloud VM id. - #vm_id = node_instance.get_instance_id() + # vm_id = node_instance.get_instance_id() # Name of the block device to detach (/dev/XYZ). - #device = node_instance.get_disk_detach_device() + # device = node_instance.get_disk_detach_device() # IaaS calls go here. raise NotImplementedError() -# ---------------------------------------------------------------- + # ---------------------------------------------------------------- TIMEOUT_CONNECT = 10 * 60 @@ -331,7 +331,7 @@ def __init__(self, configHolder): pass self._user_info = None - + self.cimi_deployment_prototype = False @property @@ -598,7 +598,6 @@ def _remote_run_build_target(self, node_instance, host, target, username, passwo message = 'Nothing to do for target "%s" on node "%s""\n' % (full_target_name, node_instance.get_name()) util.printAndFlush(message) - def _build_image_increment(self, user_info, node_instance, host): prerecipe = node_instance.get_prerecipe() recipe = node_instance.get_recipe() @@ -613,8 +612,8 @@ def _build_image_increment(self, user_info, node_instance, host): ssh_private_key_file, publicKey = self._get_temp_private_key_file_name_and_public_key() self._wait_can_connect_with_ssh_or_abort(host, username=username, - password=password, - sshKey=ssh_private_key_file) + password=password, + sshKey=ssh_private_key_file) if prerecipe: self._print_step('Running Pre-recipe', machine_name) @@ -701,10 +700,10 @@ def _secure_ssh_access(self, ip, username, password, publicKey, user_info=None): self._wait_can_connect_with_ssh_or_abort(ip, username, password) script = self.__get_obfuscation_script(publicKey, username, user_info) self._print_detail("Securing SSH access to %s with:\n%s\n" % (ip, - script)) + script)) _, output = self._run_script(ip, username, script, password=password) self._print_detail("Secured SSH access to %s. Output:\n%s\n" % (ip, - output)) + output)) def _revert_ssh_security(self, ip, username, privateKey, orchestratorPublicKey): self._wait_can_connect_with_ssh_or_abort(ip, username, sshKey=privateKey) @@ -726,11 +725,11 @@ def __launch_bootstrap_script(self, node_instance, ip, username, privateKey): self._wait_can_connect_with_ssh_or_abort(ip, username, sshKey=privateKey) script = self._get_bootstrap_script(node_instance) self._print_detail("Launching bootstrap script on %s:\n%s\n" % (ip, - script)) + script)) _, output = self._run_script_on_backgroud(ip, username, script, - sshKey=privateKey) + sshKey=privateKey) self._print_detail("Launched bootstrap script on %s:\n%s\n" % (ip, - output)) + output)) def __launch_windows_bootstrap_script(self, node_instance, ip): username, password = self.__get_vm_username_password(node_instance, 'administrator') @@ -739,12 +738,12 @@ def __launch_windows_bootstrap_script(self, node_instance, ip): winrm = self._getWinrm(ip, username, password) self._waitCanConnectWithWinrmOrAbort(winrm) self._print_detail("Launching bootstrap script on %s:\n%s\n" % (ip, - script)) + script)) util.printAndFlush(script) winrm.timeout = winrm.set_timeout(600) output = self._runScriptWithWinrm(winrm, script) self._print_detail("Launched bootstrap script on %s:\n%s\n" % (ip, - output)) + output)) def _getWinrm(self, ip, username, password): return WinRMWebService(endpoint='http://%s:5985/wsman' % ip, @@ -850,7 +849,7 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, if pre_export: script += '%s\n' % pre_export - + if self.cimi_deployment_prototype: for var, val in node_instance.get_deployment_context().items(): if re.search(' ', val): @@ -872,7 +871,6 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, util.ENV_NODE_INSTANCE_NAME, node_instance_name) - if pre_bootstrap: script += '%s\n' % pre_bootstrap @@ -886,13 +884,17 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, def _build_slipstream_bootstrap_command(self, node_instance, username=None): instance_name = node_instance.get_name() + if self.cimi_deployment_prototype: + bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') + else: + bootstrap_url = util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') if node_instance.is_windows(): - return self.__build_slipstream_bootstrap_command_for_windows(instance_name) + return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url) else: - return self.__build_slipstream_bootstrap_command_for_linux(instance_name) + return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url) - def __build_slipstream_bootstrap_command_for_windows(self, instance_name): + def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url): command = 'If Not Exist %(reports)s mkdir %(reports)s\n' command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n' @@ -908,39 +910,32 @@ def __build_slipstream_bootstrap_command_for_windows(self, instance_name): command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n' - return command % self._get_bootstrap_command_replacements_for_windows(instance_name) + return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url) - def __build_slipstream_bootstrap_command_for_linux(self, instance_name): + def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url): command = 'mkdir -p %(reports)s %(ss_home)s; ' command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 ' command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) ' command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1' - return command % self._get_bootstrap_command_replacements_for_linux(instance_name) + return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url) - - def __get_bootstrap_url(self): - if self.cimi_deployment_prototype: - return node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') - else: - return util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') - - def _get_bootstrap_command_replacements_for_linux(self, instance_name): + def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url): return { 'reports': util.REPORTSDIR, 'bootstrap': os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'), - 'bootstrapUrl': self.__get_bootstrap_url(), + 'bootstrapUrl': bootstrap_url, 'ss_home': util.SLIPSTREAM_HOME, 'nodename': instance_name, 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } - def _get_bootstrap_command_replacements_for_windows(self, instance_name): + def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url): return { 'reports': util.WINDOWS_REPORTSDIR, 'bootstrap': '\\'.join([util.WINDOWS_SLIPSTREAM_HOME, 'slipstream.bootstrap']), - 'bootstrapUrl': self.__get_bootstrap_url(), + 'bootstrapUrl': bootstrap_url, 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() @@ -1013,7 +1008,8 @@ def attach_disk(self, node_instances, done_reporter=None): def _attach_disk_and_report(self, node_instance, reporter): attached_disk = self._attach_disk(node_instance) if not attached_disk: - raise Exceptions.ExecutionException('Attached disk name not provided by connector after disk attach operation.') + raise Exceptions.ExecutionException( + 'Attached disk name not provided by connector after disk attach operation.') if hasattr(reporter, '__call__'): reporter(node_instance, attached_disk) @@ -1037,4 +1033,3 @@ def _scale_action_runner(self, scale_action, node_instances, done_reporter): scaler = VmScaler(scale_action, self.max_iaas_workers, self.verboseLevel) scaler.set_tasks_and_run(node_instances, done_reporter) scaler.wait_tasks_finished() - From 3d8b42e53a8bbd971c540875a8b00f52e775ce09 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Tue, 21 Aug 2018 15:12:53 +0200 Subject: [PATCH 13/31] Update UserInfo.py --- client/src/main/python/slipstream/UserInfo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/main/python/slipstream/UserInfo.py b/client/src/main/python/slipstream/UserInfo.py index c0853cab..7449feba 100644 --- a/client/src/main/python/slipstream/UserInfo.py +++ b/client/src/main/python/slipstream/UserInfo.py @@ -25,6 +25,7 @@ class UserInfo(dict): SSH_PUBKEY_KEY = 'sshPublicKey' NETWORK_PUBLIC_KEY = 'networkPublic' NETWORK_PRIVATE_KEY = 'networkPrivate' + CLOUD_ENDPOINT_KEY = 'endpoint' def __init__(self, cloud_qualifier): super(UserInfo, self).__init__({}) @@ -69,11 +70,14 @@ def get_cloud_password(self): return self.get_cloud(self.CLOUD_PASSWORD_KEY) def get_cloud_endpoint(self): - return self.get_cloud('endpoint') + return self.get_cloud(self.CLOUD_ENDPOINT_KEY) def get_public_keys(self): return self.get_general(self.SSH_PUBKEY_KEY) + def set_public_keys(self, ssh_pub_keys): + self[self.general + self.SSH_PUBKEY_KEY]= ssh_pub_keys + def get_private_key(self): return self.get_cloud('private.key') From faa38f6445eb7afc6d05459a485339f571b89096 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 21 Aug 2018 15:13:37 +0200 Subject: [PATCH 14/31] wip --- .../main/python/slipstream/NodeDecorator.py | 1 - .../main/python/slipstream/NodeInstance.py | 6 - .../cloudconnectors/BaseCloudConnector.py | 140 ++++++++++-------- 3 files changed, 81 insertions(+), 66 deletions(-) diff --git a/client/src/main/python/slipstream/NodeDecorator.py b/client/src/main/python/slipstream/NodeDecorator.py index 5eec8464..27f85e17 100644 --- a/client/src/main/python/slipstream/NodeDecorator.py +++ b/client/src/main/python/slipstream/NodeDecorator.py @@ -83,7 +83,6 @@ class NodeDecorator(object): SCALE_DISK_ATTACHED_DEVICE = 'disk.attached.device' SCALE_DISK_DETACH_DEVICE = 'disk.detach.device' INSTANCEID_KEY = 'instanceid' - CLOUD_NODE_ID_KEY = 'cloud.node.id' CLOUD_NODE_IP_KEY = 'cloud.node.ip' CLOUD_NODE_SSH_URL_KEY = 'cloud.node.ssh.url' CLOUD_NODE_SSH_PASSWORD_KEY = 'cloud.node.ssh.password' diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index bc712fe7..556170f2 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -216,12 +216,6 @@ def set_deployment_context(self, context=None): def get_deployment_context(self): return self.__get(NodeDecorator.DEPLOYMENT_CONTEXT_KEY, {}) - def set_cloud_node_id(self, id): - self.__set(NodeDecorator.CLOUD_NODE_ID_KEY, id) - - def get_cloud_node_id(self): - return self.__get(NodeDecorator.CLOUD_NODE_ID_KEY) - def set_cloud_node_ip(self, ip): self.__set(NodeDecorator.CLOUD_NODE_IP_KEY, ip) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 7c83d79f..28c5b20f 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -33,7 +33,7 @@ from slipstream.listeners.SimplePrintListener import SimplePrintListener from slipstream.listeners.SlipStreamClientListenerAdapter import SlipStreamClientListenerAdapter from slipstream.utils.ssh import remoteRunScriptNohup, waitUntilSshCanConnectOrTimeout, remoteRunScript, \ - generate_keypair, remoteRunCommand + generate_keypair, remoteRunCommand from slipstream.utils.tasksrunner import TasksRunner from slipstream.cloudconnectors.VmScaler import VmScaler from slipstream.wrappers.BaseWrapper import NodeInfoPublisher @@ -46,7 +46,7 @@ class BaseCloudConnector(object): -# ----- METHODS THAT CAN/SHOULD BE IMPLEMENTED IN CONNECTORS ----- + # ----- METHODS THAT CAN/SHOULD BE IMPLEMENTED IN CONNECTORS ----- def _initialization(self, user_info): """This method is called once before calling any others methods of the connector. @@ -219,15 +219,15 @@ def _resize(self, node_instance): # Example code # Cloud VM id. - #vm_id = node_instance.get_instance_id() + # vm_id = node_instance.get_instance_id() # RAM in GB. - #ram = node_instance.get_ram() + # ram = node_instance.get_ram() # Number of CPUs. - #cpu = node_instance.get_cpu() + # cpu = node_instance.get_cpu() # In case cloud uses T-short sizes. - #instance_type = node_instance.get_instance_type() + # instance_type = node_instance.get_instance_type() # IaaS calls go here. @@ -243,17 +243,17 @@ def _attach_disk(self, node_instance): # Example code - #device_name = '' + # device_name = '' # Cloud VM id. - #vm_id = node_instance.get_instance_id() + # vm_id = node_instance.get_instance_id() # Size of the disk to attach (in GB). - #disk_size_GB = node_instance.get_disk_attach_size() + # disk_size_GB = node_instance.get_disk_attach_size() # IaaS calls go here. - #return device_name + # return device_name raise NotImplementedError() @@ -266,16 +266,16 @@ def _detach_disk(self, node_instance): # Example code # Cloud VM id. - #vm_id = node_instance.get_instance_id() + # vm_id = node_instance.get_instance_id() # Name of the block device to detach (/dev/XYZ). - #device = node_instance.get_disk_detach_device() + # device = node_instance.get_disk_detach_device() # IaaS calls go here. raise NotImplementedError() -# ---------------------------------------------------------------- + # ---------------------------------------------------------------- TIMEOUT_CONNECT = 10 * 60 @@ -312,7 +312,7 @@ def __init__(self, configHolder): self.__vms = {} - #self.__cloud = os.environ[util.ENV_CONNECTOR_INSTANCE] + self.__cloud = os.environ.get(util.ENV_CONNECTOR_INSTANCE) self.__init_threading_related() @@ -332,6 +332,8 @@ def __init__(self, configHolder): self._user_info = None + self.cimi_deployment_prototype = False + @property def user_info(self): if self._user_info is None: @@ -430,6 +432,7 @@ def start_nodes_and_clients(self, user_info, nodes_instances, init_extra_kwargs= self._initialization(user_info, **init_extra_kwargs) self.__set_contextualization_capabilities(user_info) self.__create_allow_all_security_group_if_needed(nodes_instances) + self.cimi_deployment_prototype = nodes_instances.values()[0].get_deployment_context() is not None try: self.__start_nodes_instantiation_tasks_wait_finished(user_info, nodes_instances) @@ -502,28 +505,34 @@ def _publish_vm_info(self, vm, node_instance): instance_name = node_instance.get_name() vm_id = self._vm_get_id(vm) vm_ip = self._vm_get_ip(vm) - with lock: already_published = self.__already_published[instance_name] - if vm_id and 'id' not in already_published: - # self._publish_vm_id(instance_name, vm_id) - node_instance.set_cloud_node_id(vm_id) - already_published.add('id') - if vm_ip and 'ip' not in already_published: - # self._publish_vm_ip(instance_name, vm_ip) - node_instance.set_cloud_node_ip(vm_ip) - already_published.add('ip') - if node_instance and vm_ip and 'ssh' not in already_published: - # self._publish_url_ssh(vm, node_instance) - - if node_instance: - vm_ip = self._vm_get_ip(vm) or '' - ssh_username, ssh_password = self.__get_vm_username_password(node_instance) - node_instance.set_cloud_node_ssh_url('ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) - node_instance.set_cloud_node_ssh_password(ssh_password) - - if ssh_username and ssh_password: - already_published.add('ssh') + if self.cimi_deployment_prototype: + if vm_id and 'id' not in already_published: + node_instance.set_cloud_node_id(vm_id) + already_published.add('id') + if vm_ip and 'ip' not in already_published: + node_instance.set_cloud_node_ip(vm_ip) + already_published.add('ip') + if node_instance and vm_ip and 'ssh' not in already_published: + if node_instance: + vm_ip = self._vm_get_ip(vm) or '' + ssh_username, ssh_password = self.__get_vm_username_password(node_instance) + node_instance.set_cloud_node_ssh_url('ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip())) + node_instance.set_cloud_node_ssh_password(ssh_password) + + if ssh_username and ssh_password: + already_published.add('ssh') + else: + if vm_id and 'id' not in already_published: + self._publish_vm_id(instance_name, vm_id) + already_published.add('id') + if vm_ip and 'ip' not in already_published: + self._publish_vm_ip(instance_name, vm_ip) + already_published.add('ip') + if node_instance and vm_ip and 'ssh' not in already_published: + self._publish_url_ssh(vm, node_instance) + already_published.add('ssh') def _publish_vm_id(self, instance_name, vm_id): # Needed for thread safety @@ -589,7 +598,6 @@ def _remote_run_build_target(self, node_instance, host, target, username, passwo message = 'Nothing to do for target "%s" on node "%s""\n' % (full_target_name, node_instance.get_name()) util.printAndFlush(message) - def _build_image_increment(self, user_info, node_instance, host): prerecipe = node_instance.get_prerecipe() recipe = node_instance.get_recipe() @@ -604,8 +612,8 @@ def _build_image_increment(self, user_info, node_instance, host): ssh_private_key_file, publicKey = self._get_temp_private_key_file_name_and_public_key() self._wait_can_connect_with_ssh_or_abort(host, username=username, - password=password, - sshKey=ssh_private_key_file) + password=password, + sshKey=ssh_private_key_file) if prerecipe: self._print_step('Running Pre-recipe', machine_name) @@ -692,10 +700,10 @@ def _secure_ssh_access(self, ip, username, password, publicKey, user_info=None): self._wait_can_connect_with_ssh_or_abort(ip, username, password) script = self.__get_obfuscation_script(publicKey, username, user_info) self._print_detail("Securing SSH access to %s with:\n%s\n" % (ip, - script)) + script)) _, output = self._run_script(ip, username, script, password=password) self._print_detail("Secured SSH access to %s. Output:\n%s\n" % (ip, - output)) + output)) def _revert_ssh_security(self, ip, username, privateKey, orchestratorPublicKey): self._wait_can_connect_with_ssh_or_abort(ip, username, sshKey=privateKey) @@ -717,11 +725,11 @@ def __launch_bootstrap_script(self, node_instance, ip, username, privateKey): self._wait_can_connect_with_ssh_or_abort(ip, username, sshKey=privateKey) script = self._get_bootstrap_script(node_instance) self._print_detail("Launching bootstrap script on %s:\n%s\n" % (ip, - script)) + script)) _, output = self._run_script_on_backgroud(ip, username, script, - sshKey=privateKey) + sshKey=privateKey) self._print_detail("Launched bootstrap script on %s:\n%s\n" % (ip, - output)) + output)) def __launch_windows_bootstrap_script(self, node_instance, ip): username, password = self.__get_vm_username_password(node_instance, 'administrator') @@ -730,12 +738,12 @@ def __launch_windows_bootstrap_script(self, node_instance, ip): winrm = self._getWinrm(ip, username, password) self._waitCanConnectWithWinrmOrAbort(winrm) self._print_detail("Launching bootstrap script on %s:\n%s\n" % (ip, - script)) + script)) util.printAndFlush(script) winrm.timeout = winrm.set_timeout(600) output = self._runScriptWithWinrm(winrm, script) self._print_detail("Launched bootstrap script on %s:\n%s\n" % (ip, - output)) + output)) def _getWinrm(self, ip, username, password): return WinRMWebService(endpoint='http://%s:5985/wsman' % ip, @@ -829,26 +837,37 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, """This method can be redefined by connectors if they need a specific bootstrap script with the SSH contextualization.""" script = '' - add_deployment_context_variable_command = '' + addEnvironmentVariableCommand = '' node_instance_name = node_instance.get_name() if node_instance.is_windows(): - add_deployment_context_variable_command = 'set' + addEnvironmentVariableCommand = 'set' script += 'rem cmd\n' else: - add_deployment_context_variable_command = 'export' + addEnvironmentVariableCommand = 'export' script += '#!/bin/sh -ex\n' if pre_export: script += '%s\n' % pre_export - deployment_context = node_instance.get_deployment_context() - for var, val in deployment_context.items(): - if re.search(' ', val): - val = '"%s"' % val - script += '%s %s=%s\n' % (add_deployment_context_variable_command, var, val) - - script += '%s %s=%s\n' % (add_deployment_context_variable_command, + if self.cimi_deployment_prototype: + for var, val in node_instance.get_deployment_context().items(): + if re.search(' ', val): + val = '"%s"' % val + script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, val) + else: + regex = 'SLIPSTREAM_' + if self.is_start_orchestrator(): + regex += '|CLOUDCONNECTOR_' + env_matcher = re.compile(regex) + for var, val in os.environ.items(): + if env_matcher.match(var) and var != util.ENV_NODE_INSTANCE_NAME: + if re.search(' ', val): + val = '"%s"' % val + script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, + val) + + script += '%s %s=%s\n' % (addEnvironmentVariableCommand, util.ENV_NODE_INSTANCE_NAME, node_instance_name) @@ -865,7 +884,10 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, def _build_slipstream_bootstrap_command(self, node_instance, username=None): instance_name = node_instance.get_name() - bootstrap_url = node_instance.get_deployment_context()['SLIPSTREAM_BOOTSTRAP_BIN'] + if self.cimi_deployment_prototype: + bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN') + else: + bootstrap_url = util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN') if node_instance.is_windows(): return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url) @@ -906,7 +928,7 @@ def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap 'bootstrapUrl': bootstrap_url, 'ss_home': util.SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': self._get_machine_executor_type() + 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url): @@ -916,7 +938,7 @@ def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstr 'bootstrapUrl': bootstrap_url, 'ss_home': util.WINDOWS_SLIPSTREAM_HOME, 'nodename': instance_name, - 'machine_executor': self._get_machine_executor_type() + 'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type() } def _get_machine_executor_type(self): @@ -986,7 +1008,8 @@ def attach_disk(self, node_instances, done_reporter=None): def _attach_disk_and_report(self, node_instance, reporter): attached_disk = self._attach_disk(node_instance) if not attached_disk: - raise Exceptions.ExecutionException('Attached disk name not provided by connector after disk attach operation.') + raise Exceptions.ExecutionException( + 'Attached disk name not provided by connector after disk attach operation.') if hasattr(reporter, '__call__'): reporter(node_instance, attached_disk) @@ -1010,4 +1033,3 @@ def _scale_action_runner(self, scale_action, node_instances, done_reporter): scaler = VmScaler(scale_action, self.max_iaas_workers, self.verboseLevel) scaler.set_tasks_and_run(node_instances, done_reporter) scaler.wait_tasks_finished() - From 352f12bfa3a20da90566fc27d276ae17e4081069 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 21 Aug 2018 17:40:27 +0200 Subject: [PATCH 15/31] fix check cimi deployment --- .../python/slipstream/cloudconnectors/BaseCloudConnector.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 28c5b20f..67fbc87a 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -432,7 +432,7 @@ def start_nodes_and_clients(self, user_info, nodes_instances, init_extra_kwargs= self._initialization(user_info, **init_extra_kwargs) self.__set_contextualization_capabilities(user_info) self.__create_allow_all_security_group_if_needed(nodes_instances) - self.cimi_deployment_prototype = nodes_instances.values()[0].get_deployment_context() is not None + self.cimi_deployment_prototype = bool(nodes_instances.values()[0].get_deployment_context()) try: self.__start_nodes_instantiation_tasks_wait_finished(user_info, nodes_instances) @@ -860,6 +860,10 @@ def _get_bootstrap_script(self, node_instance, pre_export=None, if self.is_start_orchestrator(): regex += '|CLOUDCONNECTOR_' env_matcher = re.compile(regex) + + if pre_export: + script += '%s\n' % pre_export + for var, val in os.environ.items(): if env_matcher.match(var) and var != util.ENV_NODE_INSTANCE_NAME: if re.search(' ', val): From 9a202bf2fe4b204f5726a933e105ecbf028cd122 Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Wed, 22 Aug 2018 08:00:03 +0200 Subject: [PATCH 16/31] Update BaseCloudConnector.py --- .../python/slipstream/cloudconnectors/BaseCloudConnector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 67fbc87a..518bc96e 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -509,7 +509,7 @@ def _publish_vm_info(self, vm, node_instance): already_published = self.__already_published[instance_name] if self.cimi_deployment_prototype: if vm_id and 'id' not in already_published: - node_instance.set_cloud_node_id(vm_id) + node_instance.set_instance_id(vm_id) already_published.add('id') if vm_ip and 'ip' not in already_published: node_instance.set_cloud_node_ip(vm_ip) From 9cfe1718c6657c95561873f442d3cafc95cd144b Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Wed, 22 Aug 2018 08:01:00 +0200 Subject: [PATCH 17/31] Update NodeInstance.py --- client/src/main/python/slipstream/NodeInstance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index 556170f2..c44324a1 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -119,8 +119,8 @@ def is_windows(self): def get_build_state(self): return self.__get(NodeDecorator.BUILD_STATE_KEY, {}) - def get_instance_id(self): - return self.__get(NodeDecorator.INSTANCEID_KEY) + def set_instance_id(self, id): + return self.__set(NodeDecorator.INSTANCEID_KEY, id) def get_image_id(self): return self.get_image_attribute('id') From b904f16fda8876e75df3a92e33ecbbfa497346de Mon Sep 17 00:00:00 2001 From: Khaled Basbous Date: Wed, 22 Aug 2018 08:03:58 +0200 Subject: [PATCH 18/31] Update NodeInstance.py --- client/src/main/python/slipstream/NodeInstance.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/src/main/python/slipstream/NodeInstance.py b/client/src/main/python/slipstream/NodeInstance.py index c44324a1..31b5d493 100644 --- a/client/src/main/python/slipstream/NodeInstance.py +++ b/client/src/main/python/slipstream/NodeInstance.py @@ -119,6 +119,9 @@ def is_windows(self): def get_build_state(self): return self.__get(NodeDecorator.BUILD_STATE_KEY, {}) + def get_instance_id(self): + return self.__get(NodeDecorator.INSTANCEID_KEY) + def set_instance_id(self, id): return self.__set(NodeDecorator.INSTANCEID_KEY, id) From b18cc7f59a69778582b90fcfe565fe62840cc8f8 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Fri, 24 Aug 2018 22:14:45 +0200 Subject: [PATCH 19/31] fix complete state and fix set runtime parameter and get runtime parameter --- client/src/main/python/slipstream/Client.py | 54 ++++++++++--------- .../python/slipstream/SlipStreamHttpClient.py | 20 ++++--- .../slipstream/executors/MachineExecutor.py | 14 ++--- .../python/slipstream/wrappers/BaseWrapper.py | 4 +- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/client/src/main/python/slipstream/Client.py b/client/src/main/python/slipstream/Client.py index dc607f3f..9e45db2d 100644 --- a/client/src/main/python/slipstream/Client.py +++ b/client/src/main/python/slipstream/Client.py @@ -21,7 +21,6 @@ import subprocess from multiprocessing.dummy import Pool as ThreadPool - from SlipStreamHttpClient import SlipStreamHttpClient from exceptions.Exceptions import NotYetSetException @@ -90,24 +89,34 @@ def getRuntimeParameter(self, key): return value + def kb_extract_param_name_node_name(self, key): + node_name = None + param_name = key + if NodeDecorator.NODE_PROPERTY_SEPARATOR in key: + if not key.startswith(NodeDecorator.globalNamespacePrefix): + param_split = key.split(NodeDecorator.NODE_PROPERTY_SEPARATOR) + node_name = param_split[0] + param_name = param_split[1] + else: + node_name = self._getNodeName() + return node_name, param_name + def kb_getRuntimeParameter(self, key): - value = None - parts = self._getNodeName().split(NodeDecorator.NODE_MULTIPLICITY_SEPARATOR) - nodename = parts[0] + node_name, param_name = self.kb_extract_param_name_node_name(key) if self.no_block: - value = self.httpClient.kb_get_deployment_parameter(key, nodename) + value = self.httpClient.kb_get_deployment_parameter(param_name, node_name) else: timer = 0 while True: - value = self.httpClient.kb_get_deployment_parameter(key, nodename) + value = self.httpClient.kb_get_deployment_parameter(key, node_name) if value is not None: break if self.timeout != 0 and timer >= self.timeout: raise TimeoutException( "Exceeded timeout limit of %s waiting for key '%s' " - "to be set" % (self.timeout, _key)) - print >> sys.stderr, "Waiting for %s" % _key + "to be set" % (self.timeout, key)) + print >> sys.stderr, "Waiting for %s" % key sys.stdout.flush() sleepTime = 5 time.sleep(sleepTime) @@ -164,10 +173,10 @@ def _qualifyKey(self, key): # multiplicity parameter should NOT be qualified make an exception if len(parts) == 1 and propertyPart not in node_level_properties: _key = nodename + \ - NodeDecorator.NODE_MULTIPLICITY_SEPARATOR + \ - NodeDecorator.nodeMultiplicityStartIndex + \ - NodeDecorator.NODE_PROPERTY_SEPARATOR + \ - propertyPart + NodeDecorator.NODE_MULTIPLICITY_SEPARATOR + \ + NodeDecorator.nodeMultiplicityStartIndex + \ + NodeDecorator.NODE_PROPERTY_SEPARATOR + \ + propertyPart return _key if _key not in node_level_properties: @@ -202,20 +211,17 @@ def setRuntimeParameter(self, key, value): self.httpClient.setRuntimeParameter(_key, stripped_value) def kb_setRuntimeParameter(self, key, value): - if NodeDecorator.NODE_PROPERTY_SEPARATOR in key: - nodename = None - else: - parts = self._getNodeName().split(NodeDecorator.NODE_MULTIPLICITY_SEPARATOR) - nodename = parts[0] - self.httpClient.kb_set_deployment_parameter(key, util.removeASCIIEscape(value), nodename) + node_name, param_name = self.kb_extract_param_name_node_name(key) + self.httpClient.kb_set_deployment_parameter(param_name, util.removeASCIIEscape(value), node_name) def cancel_abort(self): # Global abort - self.httpClient.unset_runtime_parameter(NodeDecorator.globalNamespacePrefix + NodeDecorator.ABORT_KEY, - ignore_abort=True) + self.httpClient.kb_unset_deployment_parameter(NodeDecorator.globalNamespacePrefix + NodeDecorator.ABORT_KEY) + + self.httpClient.kb_unset_deployment_parameter(NodeDecorator.ABORT_KEY, self._getNodeName()) - _key = self._qualifyKey(NodeDecorator.ABORT_KEY) - self.httpClient.unset_runtime_parameter(_key, ignore_abort=True) + self.httpClient.kb_set_deployment_parameter(NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY, + "Ready") def executScript(self, script): return self._systemCall(script, retry=False) @@ -298,7 +304,7 @@ def get_rtp_all(self, compname, key): nparams = len(params) pool_size = min(POOL_MAX, nparams) self._printDetail("Get %s RTP instances with pool size: %s" % - (nparams, pool_size)) + (nparams, pool_size)) pool = ThreadPool(pool_size) results = pool.map(self._get_rtp, params) results = [v or '' for v in results] @@ -310,4 +316,4 @@ def get_session(self): return self.httpClient.get_session() def get_api(self): - return self.httpClient.get_api() \ No newline at end of file + return self.httpClient.get_api() diff --git a/client/src/main/python/slipstream/SlipStreamHttpClient.py b/client/src/main/python/slipstream/SlipStreamHttpClient.py index 94a055de..32e3c954 100644 --- a/client/src/main/python/slipstream/SlipStreamHttpClient.py +++ b/client/src/main/python/slipstream/SlipStreamHttpClient.py @@ -239,9 +239,8 @@ def complete_state(self, node_instance_name): url += SlipStreamHttpClient.URL_IGNORE_ABORT_ATTRIBUTE_QUERY return self._httpPost(url, 'reset', 'text/plain') - def kb_complete_state(self, node_instance_name): - return self.kb_set_deployment_parameter( - NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY, 'SendingReports') + def kb_complete_state(self, state, node_instance_name): + return self.kb_set_deployment_parameter(NodeDecorator.COMPLETE_KEY, state, node_instance_name) def terminate_run(self): return self._httpDelete(self.run_url) @@ -311,17 +310,22 @@ class NullNameSpace: return str(uuid.uuid3(NullNameSpace, text)) - def kb_get_deployment_parameter(self, param_name, node_id=None): + def __contruct_deployment_param_href(self, node_id, param_name): param_id = ':'.join(item or '' for item in [self.diid, node_id, param_name]) - deployment_parameter_href = 'deployment-parameter/' + self.kb_from_data_uuid(param_id) + return 'deployment-parameter/' + self.kb_from_data_uuid(param_id) + + def kb_get_deployment_parameter(self, param_name, node_id=None): + deployment_parameter_href = self.__contruct_deployment_param_href(node_id, param_name) return self.api.cimi_get(deployment_parameter_href).json.get('value') def kb_set_deployment_parameter(self, param_name, value, node_id=None): - param_id = ':'.join(item or '' for item in [self.diid, node_id, param_name]) - print('debug kb_set_deployment_parameter param_id={}'.format(param_id)) - deployment_parameter_href = 'deployment-parameter/' + self.kb_from_data_uuid(param_id) + deployment_parameter_href = self.__contruct_deployment_param_href(node_id, param_name) return self.api.cimi_edit(deployment_parameter_href, {'value': value}) + def kb_unset_deployment_parameter(self, param_name, node_id=None): + deployment_parameter_href = self.__contruct_deployment_param_href(node_id, param_name) + return self.api.cimi_edit(deployment_parameter_href, {}, select='value') + def setRuntimeParameter(self, key, value, ignoreAbort=False): url = self.run_url + '/' + key if self.ignoreAbort or ignoreAbort: diff --git a/client/src/main/python/slipstream/executors/MachineExecutor.py b/client/src/main/python/slipstream/executors/MachineExecutor.py index 585b37bd..ea838048 100644 --- a/client/src/main/python/slipstream/executors/MachineExecutor.py +++ b/client/src/main/python/slipstream/executors/MachineExecutor.py @@ -129,11 +129,11 @@ def _state_not_implemented(self, state): def _complete_state(self, state): if self._need_to_complete(state): - self.wrapper.kb_complete_state() + self.wrapper.complete_state() def _kb_complete_state(self, state): if self._need_to_complete(state): - self.wrapper.kb_complete_state() + self.wrapper.kb_complete_state(state) @staticmethod def _failure_msg_from_exception(exception): @@ -471,21 +471,15 @@ def onSendingReports(self): def onReady(self): util.printAction('Ready') - def onFinalizing(self): - util.printAction('Finalizing') - - if self.wrapper.isAbort(): - util.printError("Failed") - else: - util.printAction('Done!') - def onDone(self): + util.printAction('Done!') self._abort_running_in_final_state() def onCancelled(self): self._abort_running_in_final_state() def onAborted(self): + util.printError("Failed") self._abort_running_in_final_state() def _abort_running_in_final_state(self): diff --git a/client/src/main/python/slipstream/wrappers/BaseWrapper.py b/client/src/main/python/slipstream/wrappers/BaseWrapper.py index 0c0f184e..74811907 100644 --- a/client/src/main/python/slipstream/wrappers/BaseWrapper.py +++ b/client/src/main/python/slipstream/wrappers/BaseWrapper.py @@ -168,10 +168,10 @@ def complete_state(self, node_instance_name=None): node_instance_name = self.get_my_node_instance_name() self._ss_client.complete_state(node_instance_name) - def kb_complete_state(self, node_instance_name=None): + def kb_complete_state(self, state, node_instance_name=None): if not node_instance_name: node_instance_name = self.get_my_node_instance_name() - self._ss_client.kb_complete_state(node_instance_name) + self._ss_client.kb_complete_state(state, node_instance_name) def fail(self, message): self._ss_client.kb_set_deployment_parameter(NodeDecorator.ABORT_KEY, message, self.get_my_node_instance_name()) From 6eaf0a3a335ff21f06999fb35705265868320d0b Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Mon, 27 Aug 2018 13:50:48 +0200 Subject: [PATCH 20/31] comment non used is_mutable and scale_operational --- client/src/main/python/slipstream/executors/MachineExecutor.py | 2 +- .../python/slipstream/executors/node/NodeDeploymentExecutor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/main/python/slipstream/executors/MachineExecutor.py b/client/src/main/python/slipstream/executors/MachineExecutor.py index ea838048..6a0a84f1 100644 --- a/client/src/main/python/slipstream/executors/MachineExecutor.py +++ b/client/src/main/python/slipstream/executors/MachineExecutor.py @@ -193,7 +193,7 @@ def _is_recovery_mode(self): return self.recovery_mode == True def _is_mutable(self): - return self.wrapper.is_mutable() + return False # self.wrapper.is_mutable() def _need_to_complete(self, state): return state not in ['Finalizing', 'Done', 'Cancelled', 'Aborted'] diff --git a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py index 0af1dd48..bc352836 100644 --- a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py +++ b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py @@ -226,7 +226,7 @@ def _unset_need_to_send_reports(self): self._send_reports = False def _need_to_send_reports(self): - return self._send_reports or not self.wrapper.is_scale_state_operational() + return self._send_reports # or not self.wrapper.is_scale_state_operational() def _get_scale_action(self): return self.wrapper.get_scale_action() From f40f0df7a6984a83f4c8a623f33a412c536e5b72 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Fri, 28 Sep 2018 17:53:36 +0200 Subject: [PATCH 21/31] add ssh keys on executing --- .../python/slipstream/SlipStreamHttpClient.py | 8 +++++--- .../executors/node/NodeDeploymentExecutor.py | 3 ++- .../python/slipstream/wrappers/BaseWrapper.py | 15 ++++----------- client/src/main/scripts/slipstream.bootstrap.py | 4 +--- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/client/src/main/python/slipstream/SlipStreamHttpClient.py b/client/src/main/python/slipstream/SlipStreamHttpClient.py index 32e3c954..204c86d4 100644 --- a/client/src/main/python/slipstream/SlipStreamHttpClient.py +++ b/client/src/main/python/slipstream/SlipStreamHttpClient.py @@ -218,13 +218,15 @@ def lookup_recursively_module(module, keys): # FIXME: support for array of maps return SlipStreamHttpClient.lookup_recursively_module(module_parent, keys) def kb_get_run_type(self): - self._kb_retrieveAndSetRun() - return self.kb_deployment['module']['type'] + return self._kb_retrieveAndSetRun()['module']['type'] + + def kb_get_userparam_ssh_pubkeys(self): + return self._kb_retrieveAndSetRun().get('sshPublicKeys') def _kb_retrieveAndSetRun(self): if self.kb_deployment is None: self.kb_deployment = self.api.cimi_get(self.diid).json - return self.kb_deployment + return self.kb_deployment def _retrieve(self, url): return self._httpGet(url, 'application/xml') diff --git a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py index bc352836..dc270be7 100644 --- a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py +++ b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py @@ -83,6 +83,7 @@ def onExecuting(self): # return #if not self.wrapper.is_scale_state_operational(): + self._add_ssh_pubkey(self.wrapper.get_user_login()) self._kb_execute_build_recipes() self._kb_execute_execute_target() #else: @@ -217,7 +218,7 @@ def _get_scaling_exports(self): def _add_ssh_pubkey(self, login_user): if not util.is_windows(): util.printStep('Adding the public keys') - append_ssh_pubkey_to_authorized_keys(self.wrapper.user_ssh_pub_keys, login_user) + append_ssh_pubkey_to_authorized_keys(self._get_user_ssh_pubkey(), login_user) def _get_user_ssh_pubkey(self): return self.wrapper.get_user_ssh_pubkey() diff --git a/client/src/main/python/slipstream/wrappers/BaseWrapper.py b/client/src/main/python/slipstream/wrappers/BaseWrapper.py index 74811907..df2c1652 100644 --- a/client/src/main/python/slipstream/wrappers/BaseWrapper.py +++ b/client/src/main/python/slipstream/wrappers/BaseWrapper.py @@ -133,8 +133,6 @@ def __init__(self, config_holder): self.my_node_instance_name = self._get_my_node_instance_name(config_holder) - self.user_ssh_pub_keys = self._get_my_node_instance_name(config_holder) - self._config_holder = config_holder self._user_info = None @@ -150,13 +148,6 @@ def _get_my_node_instance_name(config_holder): except Exception: raise Exceptions.ExecutionException('Failed to get the node instance name of the the current VM') - @staticmethod - def _get_user_ssh_pub_keys(config_holder): - try: - return config_holder.user_ssh_pub_keys - except Exception: - raise Exceptions.ExecutionException('Failed to get the user ssh public keys') - def get_my_node_instance_name(self): return self.my_node_instance_name @@ -246,8 +237,10 @@ def get_cloud_instance_id(self): return self._get_runtime_parameter(key) def get_user_ssh_pubkey(self): - userInfo = self._get_user_info('') - return userInfo.get_public_keys() + return self._ss_client.kb_get_userparam_ssh_pubkeys() + + def get_user_login(self): + return self._ss_client.lookup_recursively_module(self.kb_get_my_node_instance(), ['content', 'loginUser']) def get_pre_scale_done(self, node_instance_or_name=None): """Get pre.scale.done RTP for the current node instance or for the requested one diff --git a/client/src/main/scripts/slipstream.bootstrap.py b/client/src/main/scripts/slipstream.bootstrap.py index 628451e9..fb21348b 100755 --- a/client/src/main/scripts/slipstream.bootstrap.py +++ b/client/src/main/scripts/slipstream.bootstrap.py @@ -327,15 +327,13 @@ def _persist_ss_context(): serviceurl = %s node_instance_name = %s ss_cache_key = %s -user_ssh_pub_keys = %s """ % (os.environ['SLIPSTREAM_DIID'], os.environ['SLIPSTREAM_USERNAME'], os.environ['SLIPSTREAM_API_KEY'].strip('"'), os.environ['SLIPSTREAM_API_SECRET'].strip('"'), os.environ['SLIPSTREAM_SERVICEURL'], os.environ['SLIPSTREAM_NODE_INSTANCE_NAME'], - os.environ['SLIPSTREAM_SS_CACHE_KEY'], - os.environ['SLIPSTREAM_USER_SSH_PUB_KEYS']) + os.environ['SLIPSTREAM_SS_CACHE_KEY']) _write_to_ss_client_bin('slipstream.context', slipstream_context) From ea3d6a22e31ac38a051ae3e3af8263683865e905 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Wed, 10 Oct 2018 16:03:13 +0200 Subject: [PATCH 22/31] support cimi dep subtarget --- .../src/main/python/slipstream/executors/MachineExecutor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/main/python/slipstream/executors/MachineExecutor.py b/client/src/main/python/slipstream/executors/MachineExecutor.py index 6a0a84f1..a6c716cf 100644 --- a/client/src/main/python/slipstream/executors/MachineExecutor.py +++ b/client/src/main/python/slipstream/executors/MachineExecutor.py @@ -251,7 +251,7 @@ def _kb_get_target(self, target_name): def _kb_execute_target(self, target_name, exports=None, abort_on_err=False, ssdisplay=True, ignore_abort=False): target = self._kb_get_target(target_name) - if target is None: + if not target: util.printAndFlush('Nothing to do for script: %s' % target_name) return @@ -260,7 +260,8 @@ def _kb_execute_target(self, target_name, exports=None, abort_on_err=False, ssdi util.printStep(message) fail_msg = "Failed running '%s' script on '%s'" % (target_name, self._get_node_instance_name()) - self._launch_script(target, exports, abort_on_err, ignore_abort, fail_msg, target_name) + for i, sub_t in enumerate(target): + self._launch_script(sub_t, exports, abort_on_err, ignore_abort, fail_msg, '{}[{}]'.format(target_name, i)) def _need_to_execute_build_step(self, target, subtarget): return MachineExecutor.need_to_execute_build_step(self._get_node_instance(), target, subtarget) From 907b1149ec6c7bc8d107e5032bea927aab41cf0b Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Mon, 29 Oct 2018 17:45:11 +0100 Subject: [PATCH 23/31] remove recusive lookup, fix vm name at startup --- .../main/python/slipstream/SlipStreamHttpClient.py | 12 ------------ .../slipstream/cloudconnectors/BaseCloudConnector.py | 10 ++++++++-- .../main/python/slipstream/wrappers/BaseWrapper.py | 2 +- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/client/src/main/python/slipstream/SlipStreamHttpClient.py b/client/src/main/python/slipstream/SlipStreamHttpClient.py index 204c86d4..f554f48b 100644 --- a/client/src/main/python/slipstream/SlipStreamHttpClient.py +++ b/client/src/main/python/slipstream/SlipStreamHttpClient.py @@ -205,18 +205,6 @@ def _retrieveAndSetRun(self): _, run = self._retrieve(url) self.run_dom = etree.fromstring(run.encode('utf-8')) - @staticmethod - def lookup_recursively_module(module, keys): # FIXME: support for array of maps - temp = module - for k in keys[:-1]: - temp = temp.get(k, {}) - value = temp.get(keys[-1]) - if value: - return value - module_parent = module.get('content', {}).get('parent') - if module_parent: - return SlipStreamHttpClient.lookup_recursively_module(module_parent, keys) - def kb_get_run_type(self): return self._kb_retrieveAndSetRun()['module']['type'] diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 4da1b84d..aee43205 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -468,11 +468,17 @@ def __start_node_instance_and_client(self, user_info, node_instance): self._print_detail("Starting instance: %s" % node_instance_name) - self.cimi_deployment_prototype = bool(node_instance.get_deployment_context()) + node_context = node_instance.get_deployment_context() + self.cimi_deployment_prototype = bool(node_context) + if self.cimi_deployment_prototype: + vm_name = node_instance_name + NodeDecorator.NODE_PROPERTY_SEPARATOR + \ + node_context.get('SLIPSTREAM_DIID', '').replace('deployment/', '') + else: + vm_name = self._generate_vm_name(node_instance_name) vm = self._start_image(user_info, node_instance, - self._generate_vm_name(node_instance_name)) + vm_name) self.__add_vm(vm, node_instance) diff --git a/client/src/main/python/slipstream/wrappers/BaseWrapper.py b/client/src/main/python/slipstream/wrappers/BaseWrapper.py index df2c1652..64ab3fda 100644 --- a/client/src/main/python/slipstream/wrappers/BaseWrapper.py +++ b/client/src/main/python/slipstream/wrappers/BaseWrapper.py @@ -240,7 +240,7 @@ def get_user_ssh_pubkey(self): return self._ss_client.kb_get_userparam_ssh_pubkeys() def get_user_login(self): - return self._ss_client.lookup_recursively_module(self.kb_get_my_node_instance(), ['content', 'loginUser']) + return self.kb_get_my_node_instance().get('content', {}).get('loginUser') def get_pre_scale_done(self, node_instance_or_name=None): """Get pre.scale.done RTP for the current node instance or for the requested one From aac2de5fab75de9ff40899d19fe757a10c5405fb Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Wed, 31 Oct 2018 08:30:11 +0100 Subject: [PATCH 24/31] minor --- .../main/python/slipstream/cloudconnectors/BaseCloudConnector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index aee43205..83932abe 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -476,6 +476,7 @@ def __start_node_instance_and_client(self, user_info, node_instance): node_context.get('SLIPSTREAM_DIID', '').replace('deployment/', '') else: vm_name = self._generate_vm_name(node_instance_name) + vm = self._start_image(user_info, node_instance, vm_name) From b2685b9b869d4d3a96fbfa3cff467fbf03195718 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Thu, 8 Nov 2018 14:57:14 +0100 Subject: [PATCH 25/31] pubkey is a collection of keys in cimi deployment --- client/src/main/python/slipstream/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/main/python/slipstream/util.py b/client/src/main/python/slipstream/util.py index a4deafea..d384d76b 100644 --- a/client/src/main/python/slipstream/util.py +++ b/client/src/main/python/slipstream/util.py @@ -717,7 +717,8 @@ def append_ssh_pubkey_to_authorized_keys(pubkey, user=''): except: pass - file_content = '\n# Keys added by SlipStream\n%s\n# End of keys added by SlipStream\n' % pubkey + # pubkey is a collection of keys in cimi deployment + file_content = '\n# Keys added by SlipStream\n%s\n# End of keys added by SlipStream\n' % '\r\n\r\n'.join(pubkey) authorized_keys_path = dot_ssh_path + '/authorized_keys' fileAppendContent(authorized_keys_path, file_content) From 1f0699f993bcedf336b46555e3311a6346172b54 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Fri, 23 Nov 2018 13:40:26 +0100 Subject: [PATCH 26/31] onProvisioning --- .../executors/node/NodeDeploymentExecutor.py | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py index dc270be7..b5197879 100644 --- a/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py +++ b/client/src/main/python/slipstream/executors/node/NodeDeploymentExecutor.py @@ -43,28 +43,30 @@ def __init__(self, wrapper, config_holder=ConfigHolder()): @override def onProvisioning(self): - super(NodeDeploymentExecutor, self).onProvisioning() - - if self.wrapper.is_scale_state_creating(): - self._add_ssh_pubkey(self.node_instance.get_username()) - self.wrapper.set_scale_state_created() - elif self._is_vertical_scaling(): - if not self._is_pre_scale_done(): - self._execute_pre_scale_action_target() - self.wrapper.set_pre_scale_done() - - # Orchestrator applies IaaS action on the node instance. - - self.wrapper.wait_scale_iaas_done() - self.wrapper.unset_pre_scale_done() - self._execute_post_scale_action_target() - self.wrapper.set_scale_action_done() - self._skip_execute_due_to_vertical_scaling = True - elif self._is_horizontal_scale_down(): - self._execute_pre_scale_action_target() - self._execute_report_target_and_send_reports() - self.wrapper.set_pre_scale_done() - # We are ready to be terminated. + util.printAction('Provisioning') + self._add_ssh_pubkey(self.wrapper.get_user_login()) + # super(NodeDeploymentExecutor, self).onProvisioning() + # + # if self.wrapper.is_scale_state_creating(): + # self._add_ssh_pubkey(self.node_instance.get_username()) + # self.wrapper.set_scale_state_created() + # elif self._is_vertical_scaling(): + # if not self._is_pre_scale_done(): + # self._execute_pre_scale_action_target() + # self.wrapper.set_pre_scale_done() + # + # # Orchestrator applies IaaS action on the node instance. + # + # self.wrapper.wait_scale_iaas_done() + # self.wrapper.unset_pre_scale_done() + # self._execute_post_scale_action_target() + # self.wrapper.set_scale_action_done() + # self._skip_execute_due_to_vertical_scaling = True + # elif self._is_horizontal_scale_down(): + # self._execute_pre_scale_action_target() + # self._execute_report_target_and_send_reports() + # self.wrapper.set_pre_scale_done() + # # We are ready to be terminated. @override def onExecuting(self): @@ -83,7 +85,6 @@ def onExecuting(self): # return #if not self.wrapper.is_scale_state_operational(): - self._add_ssh_pubkey(self.wrapper.get_user_login()) self._kb_execute_build_recipes() self._kb_execute_execute_target() #else: From 5282759a9803bd8651e3b88cca1898e33d079b4d Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Mon, 3 Dec 2018 15:31:13 +0100 Subject: [PATCH 27/31] bootstrap direct command force to python 2 and minor --- client/src/main/scripts/slipstream.bootstrap.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/src/main/scripts/slipstream.bootstrap.py b/client/src/main/scripts/slipstream.bootstrap.py index fb21348b..409e4d5d 100755 --- a/client/src/main/scripts/slipstream.bootstrap.py +++ b/client/src/main/scripts/slipstream.bootstrap.py @@ -846,13 +846,12 @@ def _is_linux(): def _get_machine_executor_direct_startup_command(executor_name): - custom_python_bin = os.path.join(os.sep, 'opt', 'python', 'bin') - info('Prepending %s to PATH.' % custom_python_bin) - os.putenv('PATH', '%s:%s' % (custom_python_bin, os.environ['PATH'])) cmd = os.path.join(SLIPSTREAM_CLIENT_HOME, 'sbin', 'slipstream-%s' % executor_name) os.chdir(cmd.rsplit(os.sep, 1)[0]) if sys.platform == 'win32': cmd = 'C:\\Python27\\python ' + cmd + else: + cmd = os.path.join(os.sep, 'usr', 'bin', 'python2') + ' ' + cmd return cmd + ' ' + _get_verbosity_arg() @@ -1063,7 +1062,7 @@ def _process_response(self, response): error(response.status, response.reason) error(response.read()) else: - print('Published abort message to %s' % uri) + print('Published abort message to %s' % url) if __name__ == "__main__": From 6532bc99a0b2771c1f854e396fdaa033e8eae940 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Mon, 3 Dec 2018 19:16:41 +0100 Subject: [PATCH 28/31] minor --- .../python/slipstream/command/RunInstancesCommand.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/client/src/main/python/slipstream/command/RunInstancesCommand.py b/client/src/main/python/slipstream/command/RunInstancesCommand.py index fd7c9eb6..3e388a3f 100755 --- a/client/src/main/python/slipstream/command/RunInstancesCommand.py +++ b/client/src/main/python/slipstream/command/RunInstancesCommand.py @@ -153,17 +153,10 @@ def _run_instance(self, node_instance): verbose_level = self.get_option('verbose') and 3 or 0 ch = ConfigHolder(options={'verboseLevel': verbose_level, 'retry': False, - KEY_RUN_CATEGORY: RUN_CATEGORY_DEPLOYMENT}, + KEY_RUN_CATEGORY: RUN_CATEGORY_DEPLOYMENT}, context={'foo': 'bar'}, config={'foo': 'bar'}) cc = cloud_connector_class(ch) - from pprint import pprint - print('user_info: ') - pprint(self.user_info) - print('nodes instance map: ') - pprint({nodename: node_instance}) - print('get_initialization_extra_kwargs: ') - pprint(self.get_initialization_extra_kwargs()) cc.start_nodes_and_clients(self.user_info, {nodename: node_instance}, self.get_initialization_extra_kwargs()) From 8e8b9cd235b32f08c24cf863748a508179d42354 Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 4 Dec 2018 14:08:44 +0100 Subject: [PATCH 29/31] force python2 for ss-client everywhere --- client/src/main/python/slipstream/__version__.py | 2 +- client/src/main/python/slipstream/command/VMCommandBase.py | 2 +- .../main/python/slipstream/command/VerticalScaleCommandBase.py | 2 +- client/src/main/python/ss-abort.py | 2 +- client/src/main/python/ss-cancel-abort.py | 2 +- client/src/main/python/ss-config-dump.py | 2 +- client/src/main/python/ss-display.py | 2 +- client/src/main/python/ss-execute.py | 2 +- client/src/main/python/ss-get-all.py | 2 +- client/src/main/python/ss-get.py | 2 +- client/src/main/python/ss-login.py | 2 +- client/src/main/python/ss-logout.py | 2 +- client/src/main/python/ss-module-delete.py | 2 +- client/src/main/python/ss-module-download.py | 2 +- client/src/main/python/ss-module-get.py | 2 +- client/src/main/python/ss-module-put.py | 2 +- client/src/main/python/ss-module-upload.py | 2 +- client/src/main/python/ss-node-add.py | 2 +- client/src/main/python/ss-node-remove.py | 2 +- client/src/main/python/ss-random.py | 2 +- client/src/main/python/ss-reports-get.py | 2 +- client/src/main/python/ss-run-get.py | 2 +- client/src/main/python/ss-scale-disk.py | 2 +- client/src/main/python/ss-scale-resize.py | 2 +- client/src/main/python/ss-set.py | 2 +- client/src/main/python/ss-terminate.py | 2 +- client/src/main/python/ss-user-get.py | 2 +- client/src/main/python/ss-user-put.py | 2 +- client/src/main/scripts/slipstream-node | 2 +- client/src/main/scripts/slipstream-orchestrator | 2 +- client/src/main/scripts/slipstream-prepare-bootstrap.py | 2 +- client/src/main/scripts/slipstream.bootstrap.py | 2 -- 32 files changed, 31 insertions(+), 33 deletions(-) diff --git a/client/src/main/python/slipstream/__version__.py b/client/src/main/python/slipstream/__version__.py index ad459cc7..3fd5850f 100644 --- a/client/src/main/python/slipstream/__version__.py +++ b/client/src/main/python/slipstream/__version__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # coding=latin-1 """ SlipStream Client diff --git a/client/src/main/python/slipstream/command/VMCommandBase.py b/client/src/main/python/slipstream/command/VMCommandBase.py index 8f70004d..d257ea01 100755 --- a/client/src/main/python/slipstream/command/VMCommandBase.py +++ b/client/src/main/python/slipstream/command/VMCommandBase.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/slipstream/command/VerticalScaleCommandBase.py b/client/src/main/python/slipstream/command/VerticalScaleCommandBase.py index a8c2a8ce..d9841c95 100755 --- a/client/src/main/python/slipstream/command/VerticalScaleCommandBase.py +++ b/client/src/main/python/slipstream/command/VerticalScaleCommandBase.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-abort.py b/client/src/main/python/ss-abort.py index 0c0d78de..810361c6 100755 --- a/client/src/main/python/ss-abort.py +++ b/client/src/main/python/ss-abort.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-cancel-abort.py b/client/src/main/python/ss-cancel-abort.py index 76817e6a..27147002 100755 --- a/client/src/main/python/ss-cancel-abort.py +++ b/client/src/main/python/ss-cancel-abort.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-config-dump.py b/client/src/main/python/ss-config-dump.py index 91f62b03..48f26673 100755 --- a/client/src/main/python/ss-config-dump.py +++ b/client/src/main/python/ss-config-dump.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-display.py b/client/src/main/python/ss-display.py index 87d02272..7d42cf64 100755 --- a/client/src/main/python/ss-display.py +++ b/client/src/main/python/ss-display.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-execute.py b/client/src/main/python/ss-execute.py index 557be279..cc52ad0e 100755 --- a/client/src/main/python/ss-execute.py +++ b/client/src/main/python/ss-execute.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-get-all.py b/client/src/main/python/ss-get-all.py index ec09f26f..57777e9b 100755 --- a/client/src/main/python/ss-get-all.py +++ b/client/src/main/python/ss-get-all.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-get.py b/client/src/main/python/ss-get.py index 11917e24..56bd754c 100755 --- a/client/src/main/python/ss-get.py +++ b/client/src/main/python/ss-get.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-login.py b/client/src/main/python/ss-login.py index 3487fb81..31ef549d 100755 --- a/client/src/main/python/ss-login.py +++ b/client/src/main/python/ss-login.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-logout.py b/client/src/main/python/ss-logout.py index 3dd1c425..6a433915 100755 --- a/client/src/main/python/ss-logout.py +++ b/client/src/main/python/ss-logout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-module-delete.py b/client/src/main/python/ss-module-delete.py index 57a731d3..6e413cde 100755 --- a/client/src/main/python/ss-module-delete.py +++ b/client/src/main/python/ss-module-delete.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-module-download.py b/client/src/main/python/ss-module-download.py index db612edc..7a2eccac 100755 --- a/client/src/main/python/ss-module-download.py +++ b/client/src/main/python/ss-module-download.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-module-get.py b/client/src/main/python/ss-module-get.py index 04d26a7b..00645a1a 100755 --- a/client/src/main/python/ss-module-get.py +++ b/client/src/main/python/ss-module-get.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-module-put.py b/client/src/main/python/ss-module-put.py index a86e3930..f792f51c 100755 --- a/client/src/main/python/ss-module-put.py +++ b/client/src/main/python/ss-module-put.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-module-upload.py b/client/src/main/python/ss-module-upload.py index 558eba43..271c8154 100755 --- a/client/src/main/python/ss-module-upload.py +++ b/client/src/main/python/ss-module-upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-node-add.py b/client/src/main/python/ss-node-add.py index 47358887..408e724d 100755 --- a/client/src/main/python/ss-node-add.py +++ b/client/src/main/python/ss-node-add.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-node-remove.py b/client/src/main/python/ss-node-remove.py index c5c37da6..f052327f 100755 --- a/client/src/main/python/ss-node-remove.py +++ b/client/src/main/python/ss-node-remove.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-random.py b/client/src/main/python/ss-random.py index 802d49cf..4071aefe 100755 --- a/client/src/main/python/ss-random.py +++ b/client/src/main/python/ss-random.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-reports-get.py b/client/src/main/python/ss-reports-get.py index 712af07c..00371967 100755 --- a/client/src/main/python/ss-reports-get.py +++ b/client/src/main/python/ss-reports-get.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-run-get.py b/client/src/main/python/ss-run-get.py index 6a1975ae..3fd9c425 100755 --- a/client/src/main/python/ss-run-get.py +++ b/client/src/main/python/ss-run-get.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-scale-disk.py b/client/src/main/python/ss-scale-disk.py index 0aa289b7..3e8dfbdf 100755 --- a/client/src/main/python/ss-scale-disk.py +++ b/client/src/main/python/ss-scale-disk.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-scale-resize.py b/client/src/main/python/ss-scale-resize.py index f3b98268..0d217f03 100755 --- a/client/src/main/python/ss-scale-resize.py +++ b/client/src/main/python/ss-scale-resize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-set.py b/client/src/main/python/ss-set.py index be37cd3c..fedd2e40 100755 --- a/client/src/main/python/ss-set.py +++ b/client/src/main/python/ss-set.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-terminate.py b/client/src/main/python/ss-terminate.py index da067f7e..a0af2311 100755 --- a/client/src/main/python/ss-terminate.py +++ b/client/src/main/python/ss-terminate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-user-get.py b/client/src/main/python/ss-user-get.py index 2dd487fa..ed0ffdee 100755 --- a/client/src/main/python/ss-user-get.py +++ b/client/src/main/python/ss-user-get.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/python/ss-user-put.py b/client/src/main/python/ss-user-put.py index 10934b80..0417913f 100755 --- a/client/src/main/python/ss-user-put.py +++ b/client/src/main/python/ss-user-put.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/scripts/slipstream-node b/client/src/main/scripts/slipstream-node index f082eee3..f452be2f 100755 --- a/client/src/main/scripts/slipstream-node +++ b/client/src/main/scripts/slipstream-node @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/scripts/slipstream-orchestrator b/client/src/main/scripts/slipstream-orchestrator index da00eabe..d64042ef 100755 --- a/client/src/main/scripts/slipstream-orchestrator +++ b/client/src/main/scripts/slipstream-orchestrator @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/scripts/slipstream-prepare-bootstrap.py b/client/src/main/scripts/slipstream-prepare-bootstrap.py index 4cd1a7e9..01b1166e 100644 --- a/client/src/main/scripts/slipstream-prepare-bootstrap.py +++ b/client/src/main/scripts/slipstream-prepare-bootstrap.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """ SlipStream Client ===== diff --git a/client/src/main/scripts/slipstream.bootstrap.py b/client/src/main/scripts/slipstream.bootstrap.py index 409e4d5d..d213bf1c 100755 --- a/client/src/main/scripts/slipstream.bootstrap.py +++ b/client/src/main/scripts/slipstream.bootstrap.py @@ -850,8 +850,6 @@ def _get_machine_executor_direct_startup_command(executor_name): os.chdir(cmd.rsplit(os.sep, 1)[0]) if sys.platform == 'win32': cmd = 'C:\\Python27\\python ' + cmd - else: - cmd = os.path.join(os.sep, 'usr', 'bin', 'python2') + ' ' + cmd return cmd + ' ' + _get_verbosity_arg() From 4d494aa0c565c4f04ae1de3eab6fcdd86156934d Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 4 Dec 2018 17:18:02 +0100 Subject: [PATCH 30/31] set_cloud_node_ssh_url with port mapping when available --- .../slipstream/cloudconnectors/BaseCloudConnector.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py index 83932abe..2e294ce7 100644 --- a/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py +++ b/client/src/main/python/slipstream/cloudconnectors/BaseCloudConnector.py @@ -472,8 +472,7 @@ def __start_node_instance_and_client(self, user_info, node_instance): self.cimi_deployment_prototype = bool(node_context) if self.cimi_deployment_prototype: - vm_name = node_instance_name + NodeDecorator.NODE_PROPERTY_SEPARATOR + \ - node_context.get('SLIPSTREAM_DIID', '').replace('deployment/', '') + vm_name = node_instance_name + '--' + node_context.get('SLIPSTREAM_DIID', '').replace('deployment/', '') else: vm_name = self._generate_vm_name(node_instance_name) @@ -540,6 +539,12 @@ def _publish_vm_info(self, vm, node_instance): already_published.add('ssh') if vm_ports_mapping and 'vm_ports_mapping' not in already_published: node_instance.set_cloud_node_ports_mapping(vm_ports_mapping) + ssh_found = re.search('tcp:(\d+):22', str(vm_ports_mapping)) + if ssh_found: + ssh_username, ssh_password = self.__get_vm_username_password(node_instance) + node_instance.set_cloud_node_ssh_url('ssh://{}@{}:{}'.format(ssh_username.strip(), + vm_ip.strip(), + ssh_found.group(1))) already_published.add('vm_ports_mapping') else: if vm_id and 'id' not in already_published: From 07910c2e00bdab4dc600a4e7985295a68f56593b Mon Sep 17 00:00:00 2001 From: khaled basbous Date: Tue, 11 Dec 2018 14:07:01 +0100 Subject: [PATCH 31/31] do not wait for stderr when exit success --- .../python/slipstream/executors/MachineExecutor.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/src/main/python/slipstream/executors/MachineExecutor.py b/client/src/main/python/slipstream/executors/MachineExecutor.py index a6c716cf..a2e6730d 100644 --- a/client/src/main/python/slipstream/executors/MachineExecutor.py +++ b/client/src/main/python/slipstream/executors/MachineExecutor.py @@ -366,10 +366,12 @@ def _run_target_script(self, target_script, exports=None, ignore_abort=False, na util.printDetail("End of the script '%s'" % (_name,)) stderr_last_line = '' - try: - stderr_last_line = result.get(timeout=60) - except Empty: - pass + + if process.returncode != self.SCRIPT_EXIT_SUCCESS: + try: + stderr_last_line = result.get(timeout=60) + except Empty: + pass return process.returncode, stderr_last_line def _write_target_script_to_file(self, target_script, name=None):