diff --git a/example.yaml b/example.yaml index 501bbf5..60b0b4a 100644 --- a/example.yaml +++ b/example.yaml @@ -1,54 +1,54 @@ use_keyring: true phabricator: username: richart-test host: https://scitassrv18.epfl.ch/api/ token: cli-n5dutb2wv26ivcpo66yvb3sbk64g global: backend: epfl username: richart groups: import-scheme: type: sub-project # or project project: test_import # only for sub-project type name: test_{orig_name} # repositories: # import-scheme: # type: same | git | svn # partial-import: /path # branches: [] # if not specified all are imported # tags: [] # if not specified all are imported # policies: # if anonymous access view will be public # type: create-separate-groups # create 3 groups # type: groups # use existing groups # type: default # importer only # type: best-effort # try its best to match # names: lsms-{repo}-{policy} # names: # view: lsms # push: lsms # edit: hpc-lsms groups: hpc-lsms: lsms-unit: import-scheme: name: lsms repositories: iohelper: backend: epfl type: git import-scheme: branches: [master] tags: [] test-interface: type: svn - import-scheme: all +# import-scheme: all create-groups: all-in-one: my_project diff --git a/getmystuph/importers/group_importer.py b/getmystuph/importers/group_importer.py index 83921d9..4748431 100644 --- a/getmystuph/importers/group_importer.py +++ b/getmystuph/importers/group_importer.py @@ -1,110 +1,112 @@ # -*- coding: utf-8 -*- import logging from . import Importer from .. import export from .. import colored from .. import Directory __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class GroupImporter(Importer): def __init__(self, name, config, user_db): super().__init__(name, config) self._in_directory = Directory(**self._config) if 'import-scheme' in self._config: - self._out_directory = Directory(**self._config['import-scheme']) + self._out_directory = user_db.directory self._db = user_db def _create_group(self, name, **kwargs): _logger.debug('Checking phid for group {0}'.format(name)) - phid = self._out_directory.get_group_unique_id(name) + gid = self._out_directory.get_group_unique_id(name) - if phid != '': - _logger.debug('Group {0} -> {1}'.format(name, phid)) + if gid != '': + _logger.debug('Group {0} -> {1}'.format(name, gid)) _logger.warning( - '{0} already exists in c4science try to update it' - .format(name)) - return (phid, False) + '{0} already exists in \'{1}\' try to update it' + .format(name, self._config['import-scheme']['backend'])) + return (gid, False) else: - phid = self._out_directory.create_group(name, **kwargs) - return (phid, True) + gid = self._out_directory.create_group(name, **kwargs) + return (gid, True) def _get_users(self): gid = self._in_directory.get_group_unique_id(self._name) if gid == '': _logger.error('{0} is not a valid group in the directory {1}' .format(self._colored_name, self._in_directory)) raise ValueError('{0} is not a valid group in the directory {1}' .format(self._colored_name, self._in_directory)) _logger.debug(' --> group id {0} -> {1}' .format(self._colored_name, colored(gid, attrs=['bold']))) _users = self._in_directory.get_users_from_group(gid) - self._db.add_users(self._directory, _users) + self._db.add_users(_users, self._in_directory) def transfer(self): _logger.info('Locking for group: {0}'.format(self._colored_name)) _logger.debug(' --> group info {0}'.format(colored(self._config, attrs=['bold']))) + self._get_users() + if 'import-scheme' not in self._config: return - users_ids = [u['oid'] for u in self._db.users if 'oid' in u] + users_ids = [u['oid'] for u in self._db.users.values() if 'oid' in u] import_scheme = self._config['import-scheme'] _name = self._name _newly_created = True if 'name' in import_scheme: _name = import_scheme['name'].format(orig_name=self._name) if 'type' not in import_scheme or \ import_scheme['type'] not in ['project', 'sub-project']: msg = "You should specify a type of " + \ "import-scheme for group {0}".format(self._colored_name) _logger.error(msg) raise ValueError(msg) if import_scheme['type'] == 'project': _gid, _newly_created = self._create_group(_name, members=users_ids) elif import_scheme['type'] == 'sub-project': if 'project' not in import_scheme: _msg = 'To create {0} as a subproject you ' + \ 'have to specify a parent project' \ .format(self._colored_name) _logger.error(_msg) raise ValueError(_msg) _project_name = import_scheme['project'] _pgid, _newly_created = self._create_group(_project_name) gid = self._out_directory.get_group_unique_id(_name) if gid != '': _logger.warning( - '{0} already exists in c4science try to update it' - .format(_name)) + '{0} already exists in \'{1}\' try to update it' + .format(_name, self._config['import-scheme']['backend'])) _newly_created = False else: gid = self._out_directory.create_subgroup( _name, _pgid, members=users_ids) _newly_created = True if not _newly_created: self._out_directory.set_group_users(gid, users_ids) diff --git a/getmystuph/importers/import_user_db.py b/getmystuph/importers/import_user_db.py index 7f9c903..77897d2 100644 --- a/getmystuph/importers/import_user_db.py +++ b/getmystuph/importers/import_user_db.py @@ -1,41 +1,45 @@ # -*- coding: utf-8 -*- import logging from .. import export __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class ImportUserDB: def __init__(self, out_directory): self._out_directory = out_directory self._users = {} def add_users(self, users, in_directory): for _user in users: if _user not in self._users: _user_info = {'id': _user} _mail = in_directory.get_user_email(_user) _user_info['email'] = _mail _user_info['name'] = in_directory.get_user_name(_user) _out_id = self._out_directory.get_user_unique_id(_mail) if _out_id: _user_info['oid'] = _out_id self._users[_user] = _user_info @property def users(self): return self._users + + @property + def directory(self): + return self._out_directory diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index 1ab3813..bd7ec51 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,14 +1,27 @@ # -*- coding: utf-8 -*- import logging +from .. import export +from . import Importer + __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) + + +@export +class RepoImporter(Importer): + def __init__(self, name, config, user_db): + super().__init__(name, config) + self._db = user_db + + def transfer(self): + pass diff --git a/getmystuph_in_phab.py b/getmystuph_in_phab.py index 263a461..0e6007c 100755 --- a/getmystuph_in_phab.py +++ b/getmystuph_in_phab.py @@ -1,160 +1,165 @@ #!/usr/bin/env python3 import argparse import yaml import keyring import getpass import collections import phabricator as phab import getmystuph from getmystuph import colored from getmystuph.utils import get_password import getmystuph.importers as getimp import logging import logging.config try: with open('.logging.conf', 'r') as fh: config = yaml.load(fh) logging.config.dictConfig(config) except: logging.basicConfig(level=logging.ERROR) _logger = logging.getLogger('getmystuph').getChild('main') class PhabImporter(object): """ Parses a YAML configuration file: """ _config = None def __init__(self, args): self._config = yaml.load(args.config) _logger.debug('ConfigurationParser: {0}'.format(self._config)) if 'phabricator' not in self._config: # this will use default infos from ~/.arcrc self._config['phabricator'] = {} try: self._phab = phab.Phabricator(**self._config['phabricator']) self._phab.update_interfaces() # this request is just to make an actual connection self._phab.user.whoami() except Exception as e: _logger.error( - 'Could not connect to phabricator, either give the' + + 'Could not connet to phabricator, either give the' + ' connection information in the \'phabricator\' ' + 'section of the config file:\n' + ' phabricator:\n' + ' username: richart\n' + ' host: https://c4science.ch\n' + ' token: cli-g3amff25kdpnnv2tqvigmr4omnn7\n') raise e self._keyring = None if 'use_keyring' in self._config and self._config['use_keyring']: self._keyring = keyring.get_keyring() self._imported_groups = {} - self._users_db = getimp.ImportUserDB(self._phab) + + self._out_directory = getmystuph.Directory(phabricator=self._phab, + backend='c4science') + self._users_db = getimp.ImportUserDB(self._out_directory) def _import_repository(self, name, info): _logger.info('Getting repo {0}'.format( getmystuph.Repo.color_name(name))) _logger.debug(' --> repo info {0}'.format(colored(info, attrs=['bold']))) - password = getmystuph.get_password(info['backend'], - info['username'], - keyring=self._keyring) + password = get_password(info['backend'], + info['username'], + keyring=self._keyring) repo = getmystuph.Repo(name, password=password, **info) _permissions = repo.permissions _logger.debug("Permissions for repo {0}: {1}".format( repo.color_name(name), colored(_permissions, attrs=['bold']))) _users = self._complete_users(repo.directory, _permissions.all_users) for u in _users: print(u) if 'import_scheme' in info and \ info['import_scheme'] != 'all': try: query = repo.get_query() with query: print(query.list_branches()) print(query.list_tags()) except: _logger.warning( "No fine grain operation possible for repo {0}".format( repo.color_name(name))) def __get_full_info(self, info, _type): if info is None: info = {} if 'global' in self._config: global_conf = self._config['global'] if _type in global_conf: for key, value in global_conf[_type].items(): if key not in info: info[key] = value elif type(value) == dict: info[key] = dict(value, **info[key]) import_always = ['backend', 'username'] for key in import_always: if key in global_conf and key not in info: info[key] = global_conf[key] if 'username' not in info: info['username'] = getpass.getuser() - if 'import-scheme' in info: - if 'backend' not in info['import-scheme']: + if 'import-scheme' not in info: + info['import-scheme'] = {} + + if 'backend' not in info['import-scheme']: info['import-scheme']['backend'] = 'c4science' info['import-scheme']['phabricator'] = self._phab return info def import_all(self): methods = collections.OrderedDict( [('groups', 'GroupImporter'), ('repositories', 'RepoImporter')]) for _type in methods.keys(): if _type in self._config: all_info = self._config[_type] if type(all_info) == list: all_info = {key: {} for key in all_info} for name, info in all_info.items(): info = self.__get_full_info(info, _type) importer = getattr(getimp, methods[_type])(name, info, self._users_db) importer.transfer() # Set up cli arguments parser = argparse.ArgumentParser( description='Import projects into c4science' ) parser.add_argument( '--config', help='configuration file (YAML)', type=argparse.FileType(), required=True ) args = parser.parse_args() imp = PhabImporter(args) imp.import_all()