diff --git a/example.yaml b/example.yaml index 90e708b..666fb22 100644 --- a/example.yaml +++ b/example.yaml @@ -1,75 +1,83 @@ # ----------------------------------------------------------------------------- # Date fin du mois pour le script # Fin avril fin commit # Fin aout # ---------------------------------------------------------------------------- global: use_keyring: true in: backend: epfl username: richart out: +# backend: c4science backend: phabricator username: richart-test host: https://scitassrv18.epfl.ch/api/ token: cli-n5dutb2wv26ivcpo66yvb3sbk64g groups: __all__: import-scheme: type: sub-project # or project project: test_import # only for sub-project type name: test_{original_name} scitas-ge-unit: import-scheme: type: project name: scitas-ge hpc-lsms: lsms-unit: import-scheme: name: lsms repositories: __all__: type: git import-scheme: type: same #| git | svn name: gms_{original_name} permissions: - scheme: groups + scheme: import #| project | user | static + # project: name -> tag to add + # view: | author | public + # push: | author | public + # edit: | author | public iohelper: import-scheme: branches: [master] tags: [] toto: akantu: import-scheme: branches: [master] tags: [] test-interface: type: svn - - + import-scheme: + permissions: + scheme: project + project: scitas-ge + view: public # partial-import: /path # branches: [] # if not specified all are imported # tags: [] # if not specified all are imported # policies: # if anonymous access view will be public # type: create-separate-groups # create 3 groups # type: groups # use existing groups # type: default # importer only # type: best-effort # try its best to match # names: lsms-{repo}-{policy} # names: # view: lsms # push: lsms # edit: hpc-lsms diff --git a/getmystuph/backends/c4science/repo.py b/getmystuph/backends/c4science/repo.py index 0cf89c1..79dcbc5 100644 --- a/getmystuph/backends/c4science/repo.py +++ b/getmystuph/backends/c4science/repo.py @@ -1,32 +1,32 @@ import copy import logging from ... import export -from ..repo.phabricator import PhabRepo +from ..repos.phabricator import PhabRepo __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class C4ScienceRepo(PhabRepo): def __init__(self, **kwargs): args = copy.copy(kwargs) if 'host' in args and args['host'] != 'https://c4science.ch/api/': _msg = 'You cannot specify an \'host\'' + \ ' if you use the c4science backend' _logger.error(_msg) raise RuntimeError(_msg) args['host'] = 'https://c4science.ch/api/' super().__init__(**args) diff --git a/getmystuph/backends/directories/ldap.py b/getmystuph/backends/directories/ldap.py index 16448ea..cd6b31d 100644 --- a/getmystuph/backends/directories/ldap.py +++ b/getmystuph/backends/directories/ldap.py @@ -1,154 +1,154 @@ # -*- coding: utf-8 -*- from ... import export from ...directory import Directory import ldap3 as ldap __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" @export class LDAPDirectory(Directory): def __init__(self, uri, *args, **kwargs): self.__ldap_basedn = kwargs.pop('basedn', '') self.__ldap_scope = kwargs.pop('scope', ldap.SUBTREE) self.__ldap_user_unique_id = kwargs.pop('uidNumber', 'uidNumber') self.__ldap_user_gecos = kwargs.pop('gecos', 'gecos') self.__ldap_user_id = kwargs.pop('uid', 'uid') self.__ldap_user_email = kwargs.pop('email', 'email') self.__ldap_user_filter = kwargs.pop('user_filter', '(&(objectClass=posixAccount)({attr}={value}))') # NOQA: ignore=E501 self.__ldap_user_group_attrs = kwargs.pop('user_group_attrs', 'memberOf') # NOQA: ignore=E501 self.__ldap_group_unique_id = kwargs.pop('gidNumber', 'gidNumber') self.__ldap_group_id = kwargs.pop('gid', 'cn') self.__ldap_group_filter = kwargs.pop('group_filter', '(&(objectClass=posixGroup)({attr}={value}))') # NOQA: ignore=E501 self.__ldap_group_member_filter = kwargs.pop('group_member_filter', 'uidNumber') # NOQA: ignore=E501 self.__ldap_group_user_attrs = kwargs.pop('group_user_attrs', 'memberUid') # NOQA: ignore=E501 super(LDAPDirectory, self).__init__(*args, **kwargs) self.__ldap_uri = uri self.__server = ldap.Server(self.__ldap_uri) self.__ldap = ldap.Connection(self.__server, auto_bind=True) def __get_one(self, fltr, attr): """get the first ldap entry of attribute (attr) for a given filter (fltr)""" return self.__get_all(fltr, attr)[0] def __get_one_attr(self, fltr, attr): """get the first ldap entry of attribute (attr) for a given filter (fltr)""" _res = self.__get_all(fltr, attr) if len(_res) != 0: return _res[0][attr].value - return '' + return None def __get_all(self, fltr, attr): """get all the ldap attributes entries (attr) for a given filter (fltr)""" if type(attr) is not list: attrs = [attr] else: attrs = attr _res = self.__ldap.search(search_base=self.__ldap_basedn, search_scope=self.__ldap_scope, search_filter=fltr, attributes=attrs) if _res: return self.__ldap.entries else: return [] def is_valid_user(self, id): _res = self.__get_all( self.__ldap_user_filter.format( attr=self.__ldap_user_unique_id, value=id), self.__ldap_user_unique_id ) return len(_res) != 0 def is_valid_group(self, id): _res = self.__get_one( self.__ldap_user_filter.format( attr=self.__ldap_group_unique_id, value=id), self.__ldap_group_unique_id ) return len(_res) != 0 def get_users_from_group(self, id): _users = [] _members = self.__get_one_attr( self.__ldap_group_filter.format( attr=self.__ldap_group_unique_id, value=id), self.__ldap_group_user_attrs ) if self.__ldap_group_member_filter != self.__ldap_user_unique_id: for m in _members: _filter = \ self.__ldap_user_filter.format( attr=self.__ldap_group_member_filter, value=m) _id = self.__get_one_attr( _filter, self.__ldap_user_unique_id, ) if _id: _users.append(_id) else: for m in _members: if self.is_valid_user(m): _users.append(m) return _users def get_group_unique_id(self, name): return self.__get_one_attr( self.__ldap_group_filter.format( attr=self.__ldap_group_id, value=name), self.__ldap_group_unique_id) def get_user_unique_id(self, email): return self.__get_one_attr( self.__ldap_user_filter.format( attr=self.__ldap_user_email, value=email), self.__ldap_user_unique_id) def get_group_name(self, id): return self.__get_one_attr( self.__ldap_group_filter.format( attr=self.__ldap_group_unique_id, value=id), self.__ldap_group_id) def get_user_name(self, id): return self.__get_one_attr( self.__ldap_user_filter.format( attr=self.__ldap_user_unique_id, value=id), self.__ldap_user_gecos) def get_user_email(self, id): return self.__get_one_attr( self.__ldap_user_filter.format( attr=self.__ldap_user_unique_id, value=id), self.__ldap_user_email) diff --git a/getmystuph/backends/directories/phabricator.py b/getmystuph/backends/directories/phabricator.py index b69171d..453bc51 100644 --- a/getmystuph/backends/directories/phabricator.py +++ b/getmystuph/backends/directories/phabricator.py @@ -1,149 +1,145 @@ # -*- coding: utf-8 -*- import logging from ... import colored from ... import export from ... import dry_do from ... import Directory from ...utils import get_phabricator_instance __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class PhabDirectory(Directory): def __init__(self, *args, host=None, username=None, token=None, **kwargs): super().__init__(**kwargs) self._phab = get_phabricator_instance(host=host, username=username, token=token) def _set_default_policy(self, phid): me = self._phab.user.whoami() _default_policy = [{"type": "edit", "value": me['phid']}, {"type": "view", "value": "obj.project.members"}, {"type": "join", "value": "no-one"}] - if self._dry_run: - phid = "PHID-PROJ-notarealproject" - _msg = "Setting default policy for project {0} to {1}".format( colored(phid, 'red', attrs=['bold']), ', '.join(["{0}: {1}".format(colored(m["type"], attrs=['bold']), colored(m['value'], 'blue')) for m in _default_policy])) _logger.debug(_msg) if not self._dry_run: self._phab.project.edit(transactions=_default_policy, objectIdentifier=phid) else: dry_do(_msg) def is_valid_user(self, id): - return self.get_user_name(id) != '' + return self.get_user_name(id) is not None def is_valid_group(self, id): - return self.get_group_name(id) != '' + return self.get_group_name(id) is not None def get_users_from_group(self, id): + print(id) _res = self._phab.project.search(constraints={'phids': [id]}, attachments={'members': True}) if _res['data']: return [member['phid'] for member in _res['data'][0]['attachments']['members']['members']] return [] def get_group_unique_id(self, name): _res = self._phab.project.query(names=[name]) if _res['data']: return list(_res['data'].keys())[0] - return '' + return None def get_user_unique_id(self, email): _res = self._phab.user.query(emails=[email]) if _res: return _res[0]['phid'] - return '' + return None def get_group_name(self, gid): - _res = self._phab.project.query(phid=[gid]) + _res = self._phab.project.query(phids=[gid]) if _res['data']: return _res[0]['data'][gid]['name'] - return '' + return None def get_user_name(self, uid): - _res = self._phab.user.query(phid=[uid]) + _res = self._phab.user.query(phids=[uid]) if _res: return _res[0]['realName'] - return '' + return None def get_user_email(self, uid): raise RuntimeError("This information is not accessible") def create_group(self, name, members=[]): _unique_members = list(set(members)) _msg = 'Creating group {0} with members {1}'.format( colored(name, 'red', attrs=['bold']), colored(_unique_members, attrs=['bold'])) _logger.debug(_msg) if not self._dry_run: _res = self._phab.project.create(name=name, members=_unique_members) _phid = _res['phid'] - self._set_default_policy(_phid) - return _phid else: - dry_do(_msg) - self._set_default_policy(None) - return None + _phid = "PHID-PROJ-notarealproject" + + self._set_default_policy(_phid) + return _phid def set_group_users(self, gid, uids): _unique_uids = list(set(uids)) _msg = 'Setting users {0} as members of group {1}'.format( colored(_unique_uids, attrs=['bold']), colored(gid, 'red', attrs=['bold'])) _logger.debug(_msg) transactions = [{"type": "members.set", "value": _unique_uids}] if not self._dry_run: self._phab.project.edit(transactions=transactions, objectIdentifier=gid) else: dry_do(_msg) def create_subgroup(self, name, pgid, members=None): _msg = 'Creating group {0} as a subgroup of {1}'.format( colored(name, 'red', attrs=['bold']), colored(pgid, attrs=['bold'])) _logger.debug(_msg) transactions = [{"type": "parent", "value": pgid}, {"type": "name", "value": name}] if members is not None: _unique_members = list(set(members)) transactions.append({"type": "members.set", "value": _unique_members}) if not self._dry_run: _res = self._phab.project.edit(transactions=transactions) _phid = _res['object']['phid'] - self._set_default_policy(_phid) - return _phid else: - dry_do(_msg) - self._set_default_policy(None) - return None + _phid = 'PHID-PROJ-notarealproject' + + self._set_default_policy(_phid) + return _phid diff --git a/getmystuph/importers/group_importer.py b/getmystuph/importers/group_importer.py index 03c9b28..4ef7fef 100644 --- a/getmystuph/importers/group_importer.py +++ b/getmystuph/importers/group_importer.py @@ -1,127 +1,134 @@ # -*- coding: utf-8 -*- import logging from . import Importer from .. import export from .. import colored from .. import Directory __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class GroupImporter(Importer): __default_import_scheme = {'type': 'project', 'name': '{original_name}', 'user-import-strategy': 'additive'} def __init__(self, name, config, **kwargs): super().__init__(name, config, self.__default_import_scheme, **kwargs) if 'user-import-strategy' not in self._config['import-scheme']: self._config['import-scheme']['user-import-strategy'] = 'additive' self._in_directory = Directory(**self._backend_in) self._out_directory = self._backend_out['directory'] def _create_group(self, name, **kwargs): _logger.debug('Checking phid for group {0}'.format(name)) gid = self._out_directory.get_group_unique_id(name) - if gid != '': + if gid is not None: _logger.debug('Group {0} -> {1}'.format(name, gid)) _logger.warning( '{0} already exists in \'{1}\' try to update it' .format(name, self._out_directory.backend_name)) return (gid, False) else: gid = self._out_directory.create_group(name, **kwargs) return (gid, True) - def _get_users_ids(self): - gid = self._in_directory.get_group_unique_id(self._name) - if gid == '': + def _get_users_ids(self, name): + gid = self._in_directory.get_group_unique_id(name) + if gid is None: _msg = '{0} is not a valid group in the directory {1}'.format( - self._colored_name, self._in_directory.backend_name) + colored(name, 'red', attrs=['bold']), + self._in_directory.backend_name) _logger.error(_msg) raise ValueError(_msg) _logger.debug(' --> group id {0} -> {1}' - .format(self._colored_name, + .format(colored(name, 'red', attrs=['bold']), colored(gid, attrs=['bold']))) _users = self._in_directory.get_users_from_group(gid) self._user_db.add_users(_users, self._in_directory) - return self._user_db.get_user_oids(_users) + return (gid, self._user_db.get_user_oids(_users)) - def transfer(self): - _logger.info('Locking for group: {0}'.format(self._colored_name)) + def transfer(self, name): + _colored_name = colored(name, 'green', attrs=['bold']) + _logger.info('Locking for group: {0} ({1})'.format(_colored_name, + self._colored_name)) _logger.debug(' --> group info {0}'.format(colored(self._config, attrs=['bold']))) import_scheme = self._config['import-scheme'] if import_scheme['type'] == 'ignore': return - users_ids = self._get_users_ids() + _in_gid, users_ids = self._get_users_ids(name) - _name = self._name + _name = name _newly_created = True if 'name' in import_scheme: - _name = import_scheme['name'].format(original_name=self._name) + _name = import_scheme['name'].format(original_name=name) if 'type' not in import_scheme or \ import_scheme['type'] not in ['project', 'sub-project']: msg = "You should specify a type of " + \ - "import-scheme for group {0}".format(self._colored_name) + "import-scheme for group {0}".format(_colored_name) _logger.error(msg) raise ValueError(msg) if import_scheme['type'] == 'project': - _gid, _newly_created = self._create_group(_name, - members=users_ids) + _out_gid, _newly_created = self._create_group(_name, + members=users_ids) elif import_scheme['type'] == 'sub-project': if 'project' not in import_scheme: _msg = 'To create {0} as a sub-project you ' + \ 'have to specify a parent project' \ - .format(self._colored_name) + .format(_colored_name) _logger.error(_msg) raise ValueError(_msg) _project_name = import_scheme['project'] _pgid, _newly_created = self._create_group(_project_name) - _gid = self._out_directory.get_group_unique_id(_name) - if _gid != '': + _out_gid = self._out_directory.get_group_unique_id(_name) + if _out_gid is not None: _logger.warning( '{0} already exists in \'{1}\' try to update it' .format(_name, self._out_directory.backend_name)) _newly_created = False else: - _gid = self._out_directory.create_subgroup( + _out_gid = self._out_directory.create_subgroup( _name, _pgid, members=users_ids) _newly_created = True if not _newly_created: if import_scheme['user-import-strategy'] == 'additive': _logger.debug( 'Looking for existing users in {0} ({1})'.format(_name, - _gid)) + _out_gid)) _existing_users = \ - self._out_directory.get_users_from_group(_gid) + self._out_directory.get_users_from_group(_out_gid) if users_ids: users_ids.extend(_existing_users) else: users_ids = _existing_users - self._out_directory.set_group_users(_gid, users_ids) + self._out_directory.set_group_users(_out_gid, users_ids) + + self._user_db.import_group(_in_gid, _out_gid) + + return _out_gid diff --git a/getmystuph/importers/import_user_db.py b/getmystuph/importers/import_user_db.py index 712c578..daba663 100644 --- a/getmystuph/importers/import_user_db.py +++ b/getmystuph/importers/import_user_db.py @@ -1,55 +1,77 @@ # -*- coding: utf-8 -*- import logging from .. import export from .. import colored __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class ImportUserDB: def __init__(self, out_directory): self._out_directory = out_directory self._users = {} + self._imported_groups = {} + + def import_group(self, _id, _imported_id): + self._imported_groups[_id] = _imported_id def add_users(self, users, in_directory): for _user in users: if _user not in self._users: _user_info = {'id': _user} try: _mail = in_directory.get_user_email(_user) _user_info['email'] = _mail _user_info['name'] = in_directory.get_user_name(_user) _out_id = self._out_directory.get_user_unique_id(_mail) - if _out_id: + if _out_id is not None: _user_info['oid'] = _out_id self._users[_user] = _user_info except: - _logger.warning("The user {0} does not exists in {1}".format( - colored(_user, attrs=['bold']), - colored(in_directory.backend_name, attrs=['bold']))) + _logger.warning( + "The user {0} does not exists in {1}".format( + colored(_user, attrs=['bold']), + colored(in_directory.backend_name, + attrs=['bold']))) def get_user_oids(self, users_ids): return [self._users[_user]['oid'] for _user in users_ids if _user in self._users and 'oid' in self._users[_user]] @property def users(self): return self._users @property def directory(self): return self._out_directory + + def group(self, _id): + if _id in self._imported_groups: + _logger.debug('Found group {0} in cache: {1}'.format( + colored(_id, attrs=['bold']), + colored(self._imported_groups[_id], attrs=['bold']))) + return self._imported_groups[_id] + return None + + def user(self, _id): + if _id in self._users: + _logger.debug('Found user {0} in cache: {1}'.format( + colored(_id, attrs=['bold']), + colored(self._users[_id]['oid'], attrs=['bold']))) + return self._users[_id]['oid'] + return None diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index f17dab2..c18a1d7 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,66 +1,179 @@ # -*- coding: utf-8 -*- import logging import copy from .. import export from .. import colored from .. import Repo from . import Importer +from . import GroupImporter __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class RepoImporter(Importer): __default_import_scheme = {'type': 'git', - 'permissions': {'import-scheme': 'groups'}} + 'permissions': {'scheme': 'import'}} def __init__(self, name, config, **kwargs): super().__init__(name, config, self.__default_import_scheme, **kwargs) + if 'group_importer' in kwargs and kwargs['group_importer'] is not None: + self._group_importer = kwargs['group_importer'] + else: + self._group_importer = GroupImporter( + '__all__', {}, + keyring=self._keyring, + dry_run=self._dry_run, + backend_in=self._backend_in, + backend_out=self._backend_out, + user_db=self._users_db) + _logger.info( 'Initializing importer for Repo {0}' ' with configuration: {1}'.format(self._colored_name, self._config)) - _config = copy.copy(config) - _type = _config.pop('type', self.__default_import_scheme['type']) - self._in_repo = Repo(name=name, keyring=self._keyring, - dry_run=self._dry_run, - type=_type, - **self._backend_in) - - def transfer(self): - _logger.info('Locking for repo: {0}'.format(self._colored_name)) - import_scheme = copy.copy(self._config['import-scheme']) - _type = import_scheme['type'] - if _type == 'same': - _type = self._in_repo.repo_type + def transfer(self, name): + _colored_name = colored(name, 'red', attrs=['bold']) + _logger.info('Locking for repo: {0} ({1})'.format(_colored_name, + self._colored_name)) + _import_scheme = copy.copy(self._config['import-scheme']) _logger.debug(' --> repo info {0}'.format(colored(self._config, attrs=['bold']))) - _name = import_scheme.pop('name', self._name).format( + _config = copy.copy(self._config) + _type = _config.pop('type', self.__default_import_scheme['type']) + _in_repo = Repo(name=name, + keyring=self._keyring, + dry_run=self._dry_run, + type=_type, + **self._backend_in) + + _type = _import_scheme['type'] + if _type == 'same': + _type = _in_repo.repo_type + _name = _import_scheme.pop('name', self._name).format( original_name=self._name) - self._out_repo = Repo(name=_name, - keyring=self._keyring, - dry_run=self._dry_run, - type=_type, - **self._backend_out) + _out_repo = Repo(name=_name, + keyring=self._keyring, + dry_run=self._dry_run, + type=_type, + **self._backend_out) + + _out_repo.create() + + _permissions_scheme = _import_scheme['permissions'] + + _perms = {'edit': [], + 'view': [], + 'push': []} + + if _permissions_scheme['scheme'] == 'import': + _in_perms = _in_repo.permissions + + _logger.debug("Replicating permissions {0}".format(_in_perms)) + + def _add_to_perms(_perms, ug_perm, _id): + if ug_perm == Repo.ADMIN: + _perms['edit'].append(_id) + _perms['push'].append(_id) + _perms['view'].append(_id) + + if ug_perm == Repo.WRITE: + _perms['push'].append(_id) + _perms['view'].append(_id) + + if ug_perm == Repo.READ: + _perms['view'].append(_id) + + for _group in _in_perms.groups: + _in_gid = _group['id'] + _out_gid = self._user_db.group(_in_gid) + if _out_gid is None: + name = _in_repo.directory.get_group_name(_in_gid) + if name is not None: + _out_gid = self._group_importer.transfer(name) + + if _out_gid is not None: + _add_to_perms(_perms, _group['perm'], _out_gid) + else: + _logger.error( + ' Could not find {0} in directory {1}'.format( + _in_gid, self._user_db.directory.backend_name)) + + for user in _in_perms.users: + _in_id = user['id'] + _out_id = self._user_db.user(_in_id) + + if _out_id is None: + self._user_db.add_users([_in_id], _in_repo.directory) + _out_id = self._user_db.user(_in_id) + + if _out_id is not None: + _add_to_perms(_perms, user['perm'], _out_id) + else: + _logger.error( + ' Could not find {0} in directory {1}'.format( + colored(_in_id, attrs=['bold']), + self._user_db.directory.backend_name)) + + if _in_perms.anonymous: + _perms['view'] = ['public'] + + elif _permissions_scheme['scheme'] == 'project': + if 'project' in _permissions_scheme: + _gid = self._user_db.directory.get_group_unique_id( + _permissions_scheme['project']) + if _gid is not None: + _perms['edit'] = [_gid] + _perms['view'] = [_gid] + _perms['push'] = [_gid] + else: + _msg = 'You should specify a project name in the ' + \ + 'permissions of repo {0} to be able to use the ' + \ + '\'project\' import scheme'.format(_colored_name) + _logger.error(_msg) + raise RuntimeError(_msg) + + elif _permissions_scheme['scheme'] == 'user': + _perms['edit'] = ['obj.repository.author'] + _perms['view'] = ['obj.repository.author'] + _perms['push'] = ['obj.repository.author'] + + elif _permissions_scheme['scheme'] == 'static': + for _perm_type in ['edit', 'view', 'push']: + if _perm_type not in _permissions_scheme: + _msg = 'You should specify a \'{0}\' in the ' + \ + 'permissions of repo {1} to be able to use the ' + \ + '\'project\' import scheme'.format(_perm_type, + _colored_name) + _logger.error(_msg) + raise RuntimeError(_msg) - self._out_repo.create() + for _perm_type in ['edit', 'view', 'push']: + if _perm_type in _permissions_scheme: + _perm = _permissions_scheme[_perm_type] + if _perm == 'user': + _perm = 'obj.repository.author' + _perms[_perm_type] = [_perm] - #permissions_import_scheme = import_scheme['import-scheme']['permissions'] - #self._in_repo.permissions + _logger.debug('Creating permissions for repo {0}:'.format( + _colored_name)) + for k, v in _perms.items(): + _logger.debug(' {0} {1}'.format( + k, ', '.join([colored(_id, attrs=['bold']) for _id in v]))) diff --git a/getmystuph/repo.py b/getmystuph/repo.py index 41c1c67..3cede91 100644 --- a/getmystuph/repo.py +++ b/getmystuph/repo.py @@ -1,163 +1,167 @@ # -*- coding: utf-8 -*- import copy import logging import tempfile from . import export from . import colored from .backends import _get_class from .directory import Directory __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class Repo(object): '''Interface class to define for your backend''' READ = 1 WRITE = 2 ADMIN = 4 _repo_backends = dict() def __new__(cls, *args, **kwargs): """ Factory constructor depending on the chosen backend """ option = copy.copy(kwargs) backend = option.pop('backend', None) repo_type = option.pop('type', 'git') _class = _get_class(repo_type, backend) return super(Repo, cls).__new__(_class) def __init__(self, name, *args, **kwargs): self._name = name self._colored_name = self.color_name(name) options = copy.copy(kwargs) self._username = options.pop('username', None) self._type = options.pop('type', None) self._dry_run = options.pop("dry_run", False) self._backend_name = options.pop("backend", None) self._directory = options.pop('directory', None) - + if self._directory is None: + self._directory = Directory(type='directory', + backend=self._backend_name, + username=self._username, + dry_run=self._dry_run, + **options) @property def backend_name(self): return self._backend_name @classmethod def color_name(cls, name): return colored(name, 'red', attrs=['bold']) @property def directory(self): return self._directory class Permissions(object): def __init__(self, repo): self._groups = None self._users = None self._anonymous = False self._repo = repo @property def groups(self): return self._groups @property def users(self): return self._users @property def anonymous(self): return self._anonymous @property def all_users(self): _users = [u['id'] for u in self._users] _directory = self._repo.directory for g in self._groups: _users.extend(_directory.get_users_from_group(g['id'])) return set(_users) def __repr__(self): return ''.format( self._groups, self._users, self._anonymous) @property def permissions(self): ''' Returns a dictionary of permissions of the form: {'groups': [{'id': id, 'perm': perm, ...}, ...], 'users': [{'id': id, 'perm': perm, ...}, ...], 'anonymous': True/False} perm should be read, write, admin, or None ''' return self.Permissions(self) def get_query(self): if self._type == 'git': from .repo_backends import RepoGit return RepoGit(self._name, self._url, self._username) else: raise RuntimeError( 'No backend for \'{0}\' implemented yet'.format(self._type)) - class RepoQuery(object): def __init__(self, name, url, username, **kwargs): self._name = name self._url = url self._username = username self._dry_run = kwargs.pop('dry_run', False) def __enter__(self): def debug_mktemp(name): _path = '/tmp/richart/{0}'.format(name) import os try: os.mkdir(_path) except FileExistsError: pass return _path self._stage_path = tempfile.TemporaryDirectory( prefix=self._name + '-') # self._stage_path = debug_mktemp(self._name) _logger.debug('Creating stage folder {0} for repo {1}'.format( colored(self.working_dir, attrs=['bold']), Repo.color_name(self._name))) def __exit__(self, *arg, **kwargs): _logger.debug('Cleaning staged folder {0}'.format( colored(self.working_dir, attrs=['bold']))) self._stage_path.cleanup() def list_tags(self): return [] def list_branches(self): return [] @property def working_dir(self): return self._stage_path.name # return self._stage_path diff --git a/getmystuph_in_phab.py b/getmystuph_in_phab.py index f309917..3aefb2a 100755 --- a/getmystuph_in_phab.py +++ b/getmystuph_in_phab.py @@ -1,176 +1,195 @@ #!/usr/bin/env python3 import argparse import yaml import keyring -import getpass import collections -import phabricator as phab import getmystuph from getmystuph import colored from getmystuph.utils import get_password import getmystuph.importers as getimp import logging import logging.config _logger = logging.getLogger('getmystuph').getChild('main') + class PhabImporter(object): """ Parses a YAML configuration file: """ _config = None def __init__(self, args): self._config = yaml.load(args.config) self._dry_run = args.dry_run _logger.debug('ConfigurationParser: {0}'.format(self._config)) def check_section(sect, config): if sect not in config: _msg = 'The configuration does not contain ' \ 'a \'{0}\' section'.format(sect) _logger.error(_msg) raise RuntimeError(_msg) check_section('global', self._config) check_section('in', self._config['global']) check_section('backend', self._config['global']['in']) check_section('out', self._config['global']) check_section('backend', self._config['global']['out']) self._out_backend = self._config['global']['out']['backend'] self._keyring = None if 'use_keyring' in self._config['global'] and \ self._config['global']['use_keyring']: self._keyring = keyring.get_keyring() self._imported_groups = {} self._out_directory = \ getmystuph.Directory(dry_run=self._dry_run, keyring=self._keyring, **self._config['global']['out']) self._config['global']['out']['directory'] = self._out_directory self._users_db = getimp.ImportUserDB(self._out_directory) def _import_repository(self, name, info): _logger.info('Getting repo {0}'.format( getmystuph.Repo.color_name(name))) _logger.debug(' --> repo info {0}'.format(colored(info, attrs=['bold']))) password = get_password(info['backend'], info['username'], keyring=self._keyring) repo = getmystuph.Repo(name, password=password, **info) _permissions = repo.permissions _logger.debug("Permissions for repo {0}: {1}".format( repo.color_name(name), colored(_permissions, attrs=['bold']))) _users = self._complete_users(repo.directory, _permissions.all_users) for u in _users: print(u) if 'import_scheme' in info and \ info['import_scheme'] != 'all': try: query = repo.get_query() with query: print(query.list_branches()) print(query.list_tags()) except: _logger.warning( "No fine grain operation possible for repo {0}".format( repo.color_name(name))) def __get_full_info(self, info, _type): if info is None: info = {} if '__all__' in self._config[_type]: _global_conf = self._config[_type]['__all__'] for key, value in _global_conf.items(): if key not in info: info[key] = value elif type(value) == dict: info[key] = dict(value, **info[key]) return info def import_all(self): methods = collections.OrderedDict( [('groups', 'GroupImporter'), ('repositories', 'RepoImporter')]) + + if 'groups' in self._config and \ + '__all__' in self._config['groups']: + _group_importer = getimp.GroupImporter( + '__all__', self._config['groups']['__all__'], + backend_in=self._config['global']['in'], + backend_out=self._config['global']['out'], + user_db=self._users_db, + keyring=self._keyring, + dry_run=self._dry_run) + else: + _group_importer = None + for _type in methods.keys(): if _type in self._config: all_info = self._config[_type] if type(all_info) == list: all_info = {key: {} for key in all_info} for name, info in all_info.items(): if name == '__all__': continue try: info = self.__get_full_info(info, _type) + if _type == 'repositories': + _repo_group_importer = _group_importer + else: + _repo_group_importer = None + importer = getattr(getimp, methods[_type])( name, info, backend_in=self._config['global']['in'], backend_out=self._config['global']['out'], user_db=self._users_db, keyring=self._keyring, - dry_run=self._dry_run) - importer.transfer() + dry_run=self._dry_run, + group_importer=_repo_group_importer) + + importer.transfer(name) except RuntimeError as _e: _logger.error('An error occurred while trying to' ' import the {0} {1}: {2}'.format( _type, name, _e)) # Set up cli arguments parser = argparse.ArgumentParser( description='Import projects into c4science' ) parser.add_argument( '--config', help='configuration file (YAML)', type=argparse.FileType(), required=True ) parser.add_argument( '--dry-run', help='Do not do the creations', action='store_true' ) parser.add_argument( '-v', '--verbose', help='Make the tool verbose', action='store_true' ) args = parser.parse_args() if args.verbose: with open('.logging.conf', 'r') as fh: config = yaml.load(fh) logging.config.dictConfig(config) else: logging.basicConfig(level=logging.ERROR) imp = PhabImporter(args) imp.import_all() # LocalWords: keyring