Source code for elasticluster.conf

#! /usr/bin/env python
#
# Copyright (C) 2013 GC3, University of Zurich
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
__author__ = 'Nicolas Baer <nicolas.baer@uzh.ch>, Antonio Messina <antonio.s.messina@gmail.com>'

# System imports
import os
import re
import sys

# External modules
from ConfigParser import RawConfigParser

try:
    # Voluptuous version >= 0.8.1
    from voluptuous import message, MultipleInvalid, Invalid, Schema
    from voluptuous import All, Length, Any, Url, Boolean, Optional
except ImportError:
    # Voluptuous version <= 0.7.2
    from voluptuous.voluptuous import message, MultipleInvalid, Invalid
    from voluptuous import Schema, All, Length, Any, Url, Boolean, Optional

# Elasticluster imports
from elasticluster import log
from elasticluster.exceptions import ConfigurationError
from elasticluster.providers.ansible_provider import AnsibleSetupProvider
from elasticluster.cluster import Cluster
from elasticluster.repository import ClusterRepository


[docs]class Configurator(object): """The Configurator is responsible for (I) keeping track of the configuration and (II) offer factory methods to create all kind of objects that need information from the configuration. The cluster configuration dictionary is structured in the following way: (see an example @ https://github.com/gc3-uzh-ch/elasticluster/wiki/Configuration-Module) :: { "<cluster_template>" : { "setup" : { properties of the setup section }, "cloud" : { properties of the cloud section }, "login" : { properties of the login section }, "cluster" : { properties of the cluster section }, "nodes": { "<node_kind>" : { properties of the node}, "<node_kind>" : { properties of the node}, }, }, "<cluster_template>" : { (see above) } } :param dict cluster_conf: see description above :param str storage_path: path to store data :raises MultipleInvalid: configuration validation """ setup_providers_map = {"ansible": AnsibleSetupProvider, } default_storage_dir = os.path.expanduser( "~/.elasticluster/storage") def __init__(self, cluster_conf, storage_path=None): self.general_conf = dict() self.cluster_conf = cluster_conf if storage_path: storage_path = os.path.expanduser(storage_path) storage_path = os.path.expandvars(storage_path) self.general_conf['storage'] = storage_path else: self.general_conf['storage'] = Configurator.default_storage_dir validator = ConfigValidator(self.cluster_conf) validator.validate() @classmethod
[docs] def fromConfig(cls, configfile, storage_path=None): """Helper method to initialize Configurator from an ini file. :param str configfile: path to the ini file :return: :py:class:`Configurator` """ config_reader = ConfigReader(configfile) conf = config_reader.read_config() return Configurator(conf, storage_path=storage_path)
[docs] def create_cloud_provider(self, cluster_template): """Creates a cloud provider by inspecting the configuration properties of the given cluster template. :param str cluster_template: template to use (if not already specified on init) :return: cloud provider that fulfills the contract of :py:class:`elasticluster.providers.AbstractSetupProvider` """ conf = self.cluster_conf[cluster_template]['cloud'] try: if conf['provider'] == 'ec2_boto': from elasticluster.providers.ec2_boto import BotoCloudProvider provider = BotoCloudProvider elif conf['provider'] == 'openstack': from elasticluster.providers.openstack import OpenStackCloudProvider provider = OpenStackCloudProvider elif conf['provider'] == 'google': from elasticluster.providers.gce import GoogleCloudProvider provider = GoogleCloudProvider else: raise Invalid("Invalid provider '%s' for cluster '%s'"% (conf['provider'], cluster_template)) except ImportError, ex: raise Invalid("Unable to load provider '%s': %s" % (conf['provider'], ex)) providerconf = conf.copy() providerconf.pop('provider') providerconf['storage_path'] = self.general_conf['storage'] return provider(**providerconf)
[docs] def create_cluster(self, template, name=None): """Creates a cluster by inspecting the configuration properties of the given cluster template. :param str template: name of the cluster template :param str name: name of the cluster. If not defined, the cluster will be named after the template. :return: :py:class:`elasticluster.cluster.cluster` instance :raises ConfigurationError: cluster template not found in config """ if not name: name = template if template not in self.cluster_conf: raise ConfigurationError( "Invalid configuration for cluster `%s`: %s" "" % (template, name)) conf = self.cluster_conf[template] conf_login = self.cluster_conf[template]['login'] extra = conf['cluster'].copy() extra.pop('cloud') extra.pop('setup_provider') extra['template'] = template cluster = Cluster(name, self.create_cloud_provider(template), self.create_setup_provider(template, name=name), conf_login['user_key_name'], conf_login['user_key_public'], conf_login["user_key_private"], repository=self.create_repository(), **extra) nodes = dict( (k[:-6], int(v)) for k, v in conf['cluster'].iteritems() if k.endswith('_nodes')) for kind, num in nodes.iteritems(): conf_kind = conf['nodes'][kind] extra = conf_kind.copy() extra.pop('image_id', None) extra.pop('flavor', None) extra.pop('security_group', None) extra.pop('image_userdata', None) userdata = conf_kind.get('image_userdata', '') cluster.add_nodes(kind, num, conf_kind['image_id'], conf_login['image_user'], conf_kind['flavor'], conf_kind['security_group'], image_userdata=userdata, **extra) return cluster
[docs] def load_cluster(self, cluster_name): """Loads a cluster from the cluster repository. :param str cluster_name: name of the cluster :return: :py:class:`elasticluster.cluster.cluster` instance """ repository = self.create_repository() cluster = repository.get(cluster_name) if not cluster._setup_provider: cluster._setup_provider = self.create_setup_provider(cluster.extra['template']) return cluster
[docs] def create_setup_provider(self, cluster_template, name=None): """Creates the setup provider for the given cluster template. :param str cluster_template: template of the cluster :param str name: name of the cluster to read configuration properties """ conf = self.cluster_conf[cluster_template]['setup'] conf['general_conf'] = self.general_conf.copy() if name: conf['cluster_name'] = name conf_login = self.cluster_conf[cluster_template]['login'] provider_name = conf.get('provider') if provider_name not in Configurator.setup_providers_map: raise ConfigurationError( "Invalid value `%s` for `setup_provider` in configuration " "file." % provider_name) storage_path = self.general_conf['storage'] if 'playbook_path' in conf: playbook_path = conf['playbook_path'] del conf['playbook_path'] else: playbook_path = None groups = dict((k[:-7], v.split(',')) for k, v in conf.items() if k.endswith('_groups')) environment = dict() for nodekind, grps in groups.iteritems(): if not isinstance(grps, list): groups[nodekind] = [grps] # Environment variables parsing environment[nodekind] = dict() for key, value in conf.iteritems(): prefix = "%s_var_" % nodekind if key.startswith(prefix): var = key.replace(prefix, '') environment[nodekind][var] = value provider = Configurator.setup_providers_map[provider_name] return provider(groups, playbook_path=playbook_path, environment_vars=environment, storage_path=storage_path, sudo_user=conf_login['image_user_sudo'], sudo=conf_login['image_sudo'], **conf)
def create_repository(self): storage_path = self.general_conf['storage'] repository = ClusterRepository(storage_path) return repository
[docs]class ConfigValidator(object): """Validator for the cluster configuration dictionary. :param config: dictionary containing cluster configuration properties """ def __init__(self, config): self.config = config def _pre_validate(self): """Handles all pre validation phase functionality, such as: * reading environment variables * interpolating configuraiton options """ # read cloud provider environment variables (ec2_boto or google) for cluster, props in self.config.iteritems(): if "cloud" in props and "provider" in props['cloud']: for param, value in props['cloud'].iteritems(): PARAM = param.upper() if not value and PARAM in os.environ: props['cloud'][param] = os.environ[PARAM] # interpolate ansible path manual, since configobj does not offer # an easy way to handle this ansible_pb_dir = os.path.join( sys.prefix, 'share/elasticluster/providers/ansible-playbooks') for cluster, props in self.config.iteritems(): if 'setup' in props and 'playbook_path' in props['setup']: if props['setup']['playbook_path'].startswith( "%(ansible_pb_dir)s"): pbpath = props['setup']['playbook_path'] pbpath = pbpath.replace("%(ansible_pb_dir)s", str(ansible_pb_dir)) self.config[cluster]['setup']['playbook_path'] = pbpath def _post_validate(self): """Handles all post validation phase functionality, such as: * expanding file paths """ # expand all paths for cluster, values in self.config.iteritems(): conf = self.config[cluster] if 'playbook_path' in values['setup']: pbpath = os.path.expanduser(values['setup']['playbook_path']) conf['setup']['playbook_path'] = pbpath privkey = os.path.expanduser(values['login']['user_key_private']) conf['login']['user_key_private'] = privkey pubkey = os.path.expanduser(values['login']['user_key_public']) conf['login']['user_key_public'] = pubkey
[docs] def validate(self): """Validates the given configuration :py:attr:`self.config` to comply with elasticluster. As well all types are converted to the expected format if possible. :raisegs: :py:class:`voluptuous.MultipleInvalid` if multiple properties are not compliant :raises: :py:class:`voluptuous.Invalid` if one property is invalid """ self._pre_validate() # custom validators @message("file could not be found") def check_file(v): f = os.path.expanduser(os.path.expanduser(v)) if os.path.exists(f): return f else: raise Invalid("file could not be found `%s`" % v) @message("Unsupported nova API version") def nova_api_version(version): try: from novaclient import client,exceptions client.get_client_class(version) return version except exceptions.UnsupportedVersion as ex: raise Invalid( "Invalid option for `nova_api_version`: %s" % ex) # schema to validate all cluster properties schema = {"cluster": {"cloud": All(str, Length(min=1)), "setup_provider": All(str, Length(min=1)), "login": All(str, Length(min=1))}, "setup": {"provider": All(str, Length(min=1)), Optional("playbook_path"): check_file()}, "login": {"image_user": All(str, Length(min=1)), "image_user_sudo": All(str, Length(min=1)), "image_sudo": Boolean(str), "user_key_name": All(str, Length(min=1)), "user_key_private": check_file(), "user_key_public": check_file()}} cloud_schema_ec2 = {"provider": 'ec2_boto', "ec2_url": Url(str), "ec2_access_key": All(str, Length(min=1)), "ec2_secret_key": All(str, Length(min=1)), "ec2_region": All(str, Length(min=1)), Optional("request_floating_ip"): Boolean(str)} cloud_schema_gce = {"provider": 'google', "gce_client_id": All(str, Length(min=1)), "gce_client_secret": All(str, Length(min=1)), "gce_project_id": All(str, Length(min=1))} cloud_schema_openstack = {"provider": 'openstack', "auth_url": All(str, Length(min=1)), "username": All(str, Length(min=1)), "password": All(str, Length(min=1)), "project_name": All(str, Length(min=1)), Optional("request_floating_ip"): Boolean(str), Optional("region_name"): All(str, Length(min=1)), Optional("nova_api_version"): nova_api_version()} node_schema = { "flavor": All(str, Length(min=1)), "image_id": All(str, Length(min=1)), "security_group": All(str, Length(min=1)), } # validation validator = Schema(schema, required=True, extra=True) validator_node = Schema(node_schema, required=True, extra=True) ec2_validator = Schema(cloud_schema_ec2, required=True, extra=False) gce_validator = Schema(cloud_schema_gce, required=True, extra=False) openstack_validator = Schema(cloud_schema_openstack, required=True, extra=False) if not self.config: raise Invalid("No clusters found in configuration.") for cluster, properties in self.config.iteritems(): self.config[cluster] = validator(properties) if 'provider' not in properties['cloud']: raise Invalid( "Missing `provider` option in cluster `%s`" % cluster) cloud_props = properties['cloud'] if properties['cloud']['provider'] == "ec2_boto": self.config[cluster]['cloud'] = ec2_validator(cloud_props) elif properties['cloud']['provider'] == "google": self.config[cluster]['cloud'] = gce_validator(cloud_props) elif properties['cloud']['provider'] == "openstack": self.config[cluster]['cloud'] = openstack_validator(cloud_props) if 'nodes' not in properties or len(properties['nodes']) == 0: raise Invalid( "No nodes configured for cluster `%s`" % cluster) for node, props in properties['nodes'].iteritems(): # check name pattern to conform hostnames match = re.search(r'^[a-zA-Z0-9-]*$', node) if not match: raise Invalid( "Invalid name `%s` for node group. A valid node group " "can only consist of letters, digits or the hyphens " "character (`-`)" % node) validator_node(props) self._post_validate()
[docs]class ConfigReader(object): """Reads the configuration properties from a ini file. :param str configfile: path to configfile """ cluster_section = "cluster" login_section = "login" setup_section = "setup" cloud_section = "cloud" node_section = "node" def __init__(self, configfile): self.configfile = configfile configparser = RawConfigParser() config_tmp = configparser.read(self.configfile) self.conf = dict() for section in configparser.sections(): self.conf[section] = dict(configparser.items(section)) #self.conf = ConfigObj(self.configfile, interpolation=False) @message("file could not be found") def check_file(v): f = os.path.expanduser(os.path.expanduser(v)) if os.path.exists(f): return f else: raise Invalid("file could not be found `%s`" % v) @message("Unsupported nova API version") def nova_api_version(version): try: from novaclient import client, exceptions client.get_client_class(version) return version except exceptions.UnsupportedVersion as ex: raise Invalid( "Invalid option for `nova_api_version`: %s" % ex) self.schemas = { "cloud": Schema( {"provider": Any('ec2_boto', 'google', 'openstack'), "ec2_url": Url(str), "ec2_access_key": All(str, Length(min=1)), "ec2_secret_key": All(str, Length(min=1)), "ec2_region": All(str, Length(min=1)), "auth_url": All(str, Length(min=1)), "username": All(str, Length(min=1)), "password": All(str, Length(min=1)), "tenant_name": All(str, Length(min=1)), Optional("region_name"): All(str, Length(min=1)), "gce_project_id": All(str, Length(min=1)), "gce_client_id": All(str, Length(min=1)), "gce_client_secret": All(str, Length(min=1)), "nova_client_api": nova_api_version()}, extra=True), "cluster": Schema( {"cloud": All(str, Length(min=1)), "setup_provider": All(str, Length(min=1)), "login": All(str, Length(min=1))}, required=True, extra=True), "setup": Schema( {"provider": All(str, Length(min=1)), }, required=True, extra=True), "login": Schema( {"image_user": All(str, Length(min=1)), "image_user_sudo": All(str, Length(min=1)), "image_sudo": Boolean(str), "user_key_name": All(str, Length(min=1)), "user_key_private": check_file(), "user_key_public": check_file()}, required=True) }
[docs] def read_config(self): """Reads the configuration properties from the ini file and links the section to comply with the cluster config dictionary format. :return: dictionary containing all configuration properties from the ini file in compliance to the cluster config format :raises: :py:class:`voluptuous.MultipleInvalid` if not all sections present or broken links between secitons """ clusters = dict((key, value) for key, value in self.conf.iteritems() if re.search(ConfigReader.cluster_section + "/(.*)", key) and key.count("/") == 1) conf_values = dict() errors = MultipleInvalid() for cluster in clusters: name = re.search(ConfigReader.cluster_section + "/(.*)", cluster).groups()[0] if not name: errors.add("Invalid section name `%s`" % cluster) continue cluster_conf = dict(self.conf[cluster]) try: self.schemas['cluster'](cluster_conf) except MultipleInvalid as ex: for error in ex.errors: errors.add("Section `%s`: %s" % (cluster, error)) continue cloud_name = ConfigReader.cloud_section + "/" + cluster_conf[ 'cloud'] login_name = ConfigReader.login_section + "/" + cluster_conf[ 'login'] setup_name = ConfigReader.setup_section + "/" + cluster_conf[ 'setup_provider'] values = dict() values['cluster'] = cluster_conf try: values['setup'] = dict(self.conf[setup_name]) self.schemas['setup'](values['setup']) except KeyError, ex: errors.add( "cluster `%s` setup section `%s` does not exists" % ( cluster, setup_name)) except MultipleInvalid, ex: for error in ex.errors: errors.add(error) try: values['login'] = dict(self.conf[login_name]) self.schemas['login'](values['login']) except KeyError, ex: errors.add( "cluster `%s` login section `%s` does not exists" % ( cluster, login_name)) except MultipleInvalid, ex: for error in ex.errors: errors.add(error) try: values['cloud'] = dict(self.conf[cloud_name]) self.schemas['cloud'](values['cloud']) except KeyError, ex: errors.add( "cluster `%s` cloud section `%s` does not exists" % ( cluster, cloud_name)) except MultipleInvalid, ex: for error in ex.errors: errors.add(Invalid("section %s: %s" % (cloud_name, error))) try: # nodes can inherit the properties of cluster or overwrite them nodes = dict((key, value) for key, value in values['cluster'].items() if key.endswith('_nodes')) values['nodes'] = dict() for node in nodes.iterkeys(): node_name = re.search("(.*)_nodes", node).groups()[0] property_name = "%s/%s/%s" % (ConfigReader.cluster_section, name, node_name) if property_name in self.conf: node_values = dict( (key, value.strip("'").strip('"')) for key, value in self.conf[property_name].iteritems()) node_values = dict( values['cluster'].items() + node_values.items()) values['nodes'][node_name] = node_values else: values['nodes'][node_name] = values['cluster'] if errors.errors: for error in errors.errors: log.warning( "Ignoring Cluster `%s`: %s" % (name, error)) log.warning("Ignoring cluster `%s`." % name) else: conf_values[name] = values except KeyError, ex: errors.add("Error in section `%s`" % cluster) if errors.errors: raise errors return conf_values