diff --git a/example.yaml b/example.yaml index 60b0b4a..90b7b6b 100644 --- a/example.yaml +++ b/example.yaml @@ -1,54 +1,53 @@ -use_keyring: true - phabricator: username: richart-test host: https://scitassrv18.epfl.ch/api/ token: cli-n5dutb2wv26ivcpo66yvb3sbk64g global: + use_keyring: true backend: epfl username: richart groups: import-scheme: type: sub-project # or project project: test_import # only for sub-project type - name: test_{orig_name} + name: test_{original_name} # repositories: # import-scheme: # type: same | git | svn # partial-import: /path # branches: [] # if not specified all are imported # tags: [] # if not specified all are imported # policies: # if anonymous access view will be public # type: create-separate-groups # create 3 groups # type: groups # use existing groups # type: default # importer only # type: best-effort # try its best to match # names: lsms-{repo}-{policy} # names: # view: lsms # push: lsms # edit: hpc-lsms groups: hpc-lsms: lsms-unit: import-scheme: name: lsms repositories: iohelper: backend: epfl type: git import-scheme: branches: [master] tags: [] test-interface: type: svn # import-scheme: all create-groups: all-in-one: my_project diff --git a/getmystuph/README.md b/getmystuph/README.md new file mode 100644 index 0000000..9750eca --- /dev/null +++ b/getmystuph/README.md @@ -0,0 +1,23 @@ +## Sections description: + +### Groups + +The group section defines which groups have to be imported and how this has to be done. + +```yaml +groups: + : + backend: phabricator | epfl | ... + username: + import-scheme: + type: project | sub-project | ignore + project: + name: +``` + +For groups you have to define from which backend you have to get them. They will be transferred as project in phabricator. The way they are imported can be configured through the **import-scheme**. + +The two main ways to import groups are for now as projects or as sub-projects. This is defined by the **type** key. In both case the name of the (sub-)project to create can be defined by the **name** entry. +In the case of sub-projects, the parent project name has to be given with the key **project**. + +Names could use placeholders that will be replaced, for know only __{original_name}__ can be used. This will be replaced by the original name of the group. diff --git a/getmystuph/__init__.py b/getmystuph/__init__.py index 4362fb2..c4e4099 100644 --- a/getmystuph/__init__.py +++ b/getmystuph/__init__.py @@ -1,54 +1,55 @@ # -*- coding: utf-8 -*- ''' @package getmystuph @file __init__.py @copyright BSD @author Nicolas Richart @section COPYRIGHT Copyright (©) 2015 EPFL (Ecole Polytechnique Fédérale de Lausanne) SCITAS - Scientific IT and Application Support This file is part of getmystuph ''' import sys as __gms_sys + def export(definition): """ Decorator to export definitions from sub-modules to the top-level package :param definition: definition to be exported :return: definition """ __module = __gms_sys.modules[definition.__module__] __pkg = __gms_sys.modules[__module.__package__] __pkg.__dict__[definition.__name__] = definition if '__all__' not in __pkg.__dict__: __pkg.__dict__['__all__'] = [] __pkg.__all__.append(definition.__name__) return definition try: from termcolor import colored except ImportError: # noinspection PyUnusedLocal def colored(string, *args, **kwargs): return string __all__ = ['colored'] @export def dry_do(msg): print('{0}: {1}'.format(colored('WD', 'blue', attrs=['bold']), msg)) from . import directory # noqa : F401 from . import repo # noqa : F401 from . import directory_backends # noqa : F401 from . import repo_backends # noqa : F401 from . import backends diff --git a/getmystuph/importers/group_importer.py b/getmystuph/importers/group_importer.py index 7cec38a..2735006 100644 --- a/getmystuph/importers/group_importer.py +++ b/getmystuph/importers/group_importer.py @@ -1,112 +1,112 @@ # -*- coding: utf-8 -*- import logging from . import Importer from .. import export from .. import colored from .. import Directory __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) - @export class GroupImporter(Importer): + __default_import_scheme = {'type': 'project', 'name': '{original_name}'} + def __init__(self, name, config, user_db, **kwargs): - super().__init__(name, config, **kwargs) + super().__init__(name, config, + self.__default_import_scheme, **kwargs) self._in_directory = Directory(**self._config, **kwargs) - if 'import-scheme' in self._config: - self._out_directory = user_db.directory - + self._out_directory = user_db.directory self._db = user_db def _create_group(self, name, **kwargs): _logger.debug('Checking phid for group {0}'.format(name)) gid = self._out_directory.get_group_unique_id(name) if gid != '': _logger.debug('Group {0} -> {1}'.format(name, gid)) _logger.warning( '{0} already exists in \'{1}\' try to update it' .format(name, self._config['import-scheme']['backend'])) return (gid, False) else: gid = self._out_directory.create_group(name, **kwargs) return (gid, True) def _get_users(self): gid = self._in_directory.get_group_unique_id(self._name) if gid == '': _logger.error('{0} is not a valid group in the directory {1}' .format(self._colored_name, self._in_directory)) raise ValueError('{0} is not a valid group in the directory {1}' .format(self._colored_name, self._in_directory)) _logger.debug(' --> group id {0} -> {1}' .format(self._colored_name, colored(gid, attrs=['bold']))) _users = self._in_directory.get_users_from_group(gid) self._db.add_users(_users, self._in_directory) def transfer(self): _logger.info('Locking for group: {0}'.format(self._colored_name)) _logger.debug(' --> group info {0}'.format(colored(self._config, attrs=['bold']))) self._get_users() - if 'import-scheme' not in self._config: + import_scheme = self._config['import-scheme'] + if import_scheme['type'] == 'ignore': return users_ids = [u['oid'] for u in self._db.users.values() if 'oid' in u] - import_scheme = self._config['import-scheme'] _name = self._name _newly_created = True if 'name' in import_scheme: - _name = import_scheme['name'].format(orig_name=self._name) + _name = import_scheme['name'].format(original_name=self._name) if 'type' not in import_scheme or \ import_scheme['type'] not in ['project', 'sub-project']: msg = "You should specify a type of " + \ "import-scheme for group {0}".format(self._colored_name) _logger.error(msg) raise ValueError(msg) if import_scheme['type'] == 'project': _gid, _newly_created = self._create_group(_name, members=users_ids) elif import_scheme['type'] == 'sub-project': if 'project' not in import_scheme: _msg = 'To create {0} as a subproject you ' + \ 'have to specify a parent project' \ .format(self._colored_name) _logger.error(_msg) raise ValueError(_msg) _project_name = import_scheme['project'] _pgid, _newly_created = self._create_group(_project_name) gid = self._out_directory.get_group_unique_id(_name) if gid != '': _logger.warning( '{0} already exists in \'{1}\' try to update it' .format(_name, self._config['import-scheme']['backend'])) _newly_created = False else: gid = self._out_directory.create_subgroup( _name, _pgid, members=users_ids) _newly_created = True if not _newly_created: self._out_directory.set_group_users(gid, users_ids) diff --git a/getmystuph/importers/importer.py b/getmystuph/importers/importer.py index 8a05695..4f9923f 100644 --- a/getmystuph/importers/importer.py +++ b/getmystuph/importers/importer.py @@ -1,29 +1,32 @@ # -*- coding: utf-8 -*- import logging import copy from .. import export from .. import colored __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class Importer: - def __init__(self, name, config, **kwargs): - self._config = config + def __init__(self, name, config, default_import_scheme, **kwargs): + self._config = copy.copy(config) self._name = name self._colored_name = colored(self._name, 'red', attrs=['bold']) opts = copy.copy(kwargs) self._dry_run = opts.pop("dry_run", False) + + if 'import_scheme' not in self._config: + self._config['import_scheme'] = default_import_scheme diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index bd7ec51..533a8bc 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,27 +1,29 @@ # -*- coding: utf-8 -*- import logging from .. import export from . import Importer __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class RepoImporter(Importer): + __default_import_scheme = {''} + def __init__(self, name, config, user_db): - super().__init__(name, config) + super().__init__(name, config, self.__default_import_scheme) self._db = user_db def transfer(self): pass diff --git a/getmystuph_in_phab.py b/getmystuph_in_phab.py index 1d18b71..550f8cc 100755 --- a/getmystuph_in_phab.py +++ b/getmystuph_in_phab.py @@ -1,173 +1,177 @@ #!/usr/bin/env python3 import argparse import yaml import keyring import getpass import collections import phabricator as phab import getmystuph from getmystuph import colored from getmystuph.utils import get_password import getmystuph.importers as getimp import logging import logging.config try: with open('.logging.conf', 'r') as fh: config = yaml.load(fh) logging.config.dictConfig(config) except: logging.basicConfig(level=logging.ERROR) _logger = logging.getLogger('getmystuph').getChild('main') class PhabImporter(object): """ Parses a YAML configuration file: """ _config = None def __init__(self, args): self._config = yaml.load(args.config) self._dry_run = args.dry_run _logger.debug('ConfigurationParser: {0}'.format(self._config)) if 'phabricator' not in self._config: # this will use default infos from ~/.arcrc self._config['phabricator'] = {} try: self._phab = phab.Phabricator(**self._config['phabricator']) self._phab.update_interfaces() # this request is just to make an actual connection self._phab.user.whoami() except Exception as e: _logger.error( 'Could not connect to phabricator, either give the' + ' connection information in the \'phabricator\' ' + 'section of the config file:\n' + ' phabricator:\n' + ' username: richart\n' + ' host: https://c4science.ch\n' + ' token: cli-g3amff25kdpnnv2tqvigmr4omnn7\n') raise e self._keyring = None - if 'use_keyring' in self._config and self._config['use_keyring']: + if 'use_keyring' in self._config['global'] and \ + self._config['global']['use_keyring']: self._keyring = keyring.get_keyring() self._imported_groups = {} self._out_directory = getmystuph.Directory(phabricator=self._phab, - backend='c4science', dry_run=self._dry_run) + backend='c4science', + dry_run=self._dry_run) self._users_db = getimp.ImportUserDB(self._out_directory) def _import_repository(self, name, info): _logger.info('Getting repo {0}'.format( getmystuph.Repo.color_name(name))) _logger.debug(' --> repo info {0}'.format(colored(info, attrs=['bold']))) password = get_password(info['backend'], info['username'], keyring=self._keyring) repo = getmystuph.Repo(name, password=password, **info) _permissions = repo.permissions _logger.debug("Permissions for repo {0}: {1}".format( repo.color_name(name), colored(_permissions, attrs=['bold']))) _users = self._complete_users(repo.directory, _permissions.all_users) for u in _users: print(u) if 'import_scheme' in info and \ info['import_scheme'] != 'all': try: query = repo.get_query() with query: print(query.list_branches()) print(query.list_tags()) except: _logger.warning( "No fine grain operation possible for repo {0}".format( repo.color_name(name))) def __get_full_info(self, info, _type): if info is None: info = {} if 'global' in self._config: global_conf = self._config['global'] if _type in global_conf: for key, value in global_conf[_type].items(): if key not in info: info[key] = value elif type(value) == dict: info[key] = dict(value, **info[key]) import_always = ['backend', 'username'] for key in import_always: if key in global_conf and key not in info: info[key] = global_conf[key] if 'username' not in info: info['username'] = getpass.getuser() if 'import-scheme' not in info: info['import-scheme'] = {} if 'backend' not in info['import-scheme']: info['import-scheme']['backend'] = 'c4science' info['import-scheme']['phabricator'] = self._phab return info def import_all(self): methods = collections.OrderedDict( [('groups', 'GroupImporter'), ('repositories', 'RepoImporter')]) for _type in methods.keys(): if _type in self._config: all_info = self._config[_type] if type(all_info) == list: all_info = {key: {} for key in all_info} for name, info in all_info.items(): info = self.__get_full_info(info, _type) importer = getattr(getimp, methods[_type])(name, info, self._users_db) importer.transfer() # Set up cli arguments parser = argparse.ArgumentParser( description='Import projects into c4science' ) parser.add_argument( '--config', help='configuration file (YAML)', type=argparse.FileType(), required=True ) parser.add_argument( '--dry-run', help='Do not do the creations', action='store_true' ) args = parser.parse_args() imp = PhabImporter(args) imp.import_all() + +# LocalWords: keyring