diff --git a/example.yaml b/example.yaml index f96ff9e..a350cf7 100644 --- a/example.yaml +++ b/example.yaml @@ -1,107 +1,110 @@ global: user_db_cache: "cache.yaml" # file to save ldap queries to all: [svn, git] # Only migrate one type of repo use_keyring: true # Save your password for each repo access in: # Source of the repositories backend: epfl # only epfl backend implemented username: out: backend: c4science username: token: cli-n5dutb2wv26ivcpo66yvb3sbk64g # API token obtained in the settings groups: __all__: import-scheme: # Global project options type: sub-project # project | sub-project | ignore # if sub-project, specify a project to import the group # to. If set to project, will create one project per group. # ignore will skip the group creation project: test_import # only for sub-project type, which project to attach the sub-project to name: test_{original_name} # if you want to prefix or suffix your Project user-import-scheme: additive # additive (default) or synchronize # Sync will delete user on c4science if they # don't exist in the group force-members: username # List or single username from the source backend that force-members: # Will be added to the group instead of the source - username1 # members - username2 scitas-ge-unit: # Specific options for the scitas-ge-unit ldap group import-scheme: type: project name: scitas-ge hpc-lsms: # this will created the project if no repo is given, otherwhise # will inherit from __all__ import-scheme lsms-unit: # Example of a EPFL unit to import as lsms Project import-scheme: name: lsms repositories: __all__: # __all__ is the default for other repo, you can override this in each repo type: git # svn | git (default git) import-scheme: type: same # git | svn # the type to migrate to # (only git->git, svn->svn and svn->git are supported) name: gms_{original_name} # default: '{original_name}' + tags: project_name # String or List of project to tag the repository with + # When using project in the same subproject hierarchy, + # only the child will be kept permissions: scheme: import # import | project | user | static # # import # Import the groups and users structure from the source # scheme: import # # project # Set the repo policy to a specific Project name # scheme: project # project: # # user # Set the repo policy to the repository author (you) # scheme: user # # static # Custom permissions # scheme: static # view: | _author_ | _public_ | _users_ # push: | _author_ | _public_ | _users_ # edit: | _author_ | _public_ | _users_ iohelper: # The repository name import-scheme: permissions: # Custom permissions scheme: static view: _public_ # Public repository on c4science push: _users_ # Only c4science users can push edit: lsms # User from the lsms Project can edit # Some more examples toto: # Repository wihtout options, include a blank line after akantu: # Only import master and no tags import-scheme: branches: [master] tags: [] test: # Import an svn repository type: svn import-scheme: permissions: scheme: project project: scitas-ge view: _public_ test: # Transform an svn repo to git type: svn import-scheme: name: test-git type: git diff --git a/getmystuph/backends/directories/phabricator.py b/getmystuph/backends/directories/phabricator.py index 5493537..e1824fc 100644 --- a/getmystuph/backends/directories/phabricator.py +++ b/getmystuph/backends/directories/phabricator.py @@ -1,171 +1,176 @@ import logging import re from ... import colored from ... import export from ... import dry_do from ... import Directory from ...utils import get_phabricator_instance from .. import color_phid __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class PhabDirectory(Directory): def __init__(self, *args, host=None, username=None, token=None, **kwargs): super().__init__(**kwargs) self._host = '{0}/api/'.format(host.rstrip('/')) self._phab = get_phabricator_instance(host=self._host, username=username, token=token) def _set_default_policy(self, phid): _default_policy = [{"type": "edit", "value": self.whoami}, {"type": "view", "value": "obj.project.members"}, {"type": "join", "value": "no-one"}] _msg = "Setting default policy for project {0} to {1}".format( color_phid(phid), ', '.join(["{0}: {1}".format(colored(m["type"], attrs=['bold']), color_phid(m['value'])) for m in _default_policy])) _logger.debug(_msg) if not self._dry_run: self._phab.project.edit(transactions=_default_policy, objectIdentifier=phid) else: dry_do(_msg) def color_name(self, name, **kwargs): regex = re.compile(r'PHID-([A-Z]{4})-.+') match = regex.match(name) if match: return color_phid(name) else: return super().color_name(name, **kwargs) def is_valid_user(self, id): return self.get_user_name(id) is not None def is_valid_group(self, id): return self.get_group_name(id) is not None def get_users_from_group(self, id): _res = self._phab.project.search(constraints={'phids': [id]}, attachments={'members': True}) if _res['data']: return [member['phid'] for member in _res['data'][0]['attachments']['members']['members']] return [] def get_group_unique_id(self, name): - _res = self._phab.project.query(names=[name]) + _islist = type(name) == list + _names = name if _islist else [name] + _res = self._phab.project.query(names=_names) if _res['data']: - return list(_res['data'].keys())[0] + if _islist: + return list(_res['data'].keys()) + else: + return list(_res['data'].keys())[0] return None def get_user_unique_id(self, email): _res = self._phab.user.query(emails=[email]) if _res: return _res[0]['phid'] return None def get_user_unique_id_from_login(self, name): _res = self._phab.user.query(emails=[name]) if _res: return _res[0]['phid'] return None def get_group_name(self, gid): _res = self._phab.project.query(phids=[gid]) if _res['data']: return _res[0]['data'][gid]['name'] return None def get_user_name(self, uid): _res = self._phab.user.query(phids=[uid]) if _res: return _res[0]['realName'] return None def get_user_email(self, uid): raise RuntimeError("This information is not accessible") def create_group(self, name, members=[], icon='project'): _unique_members = list(set(members)) _msg = 'Creating group {0} with members [{1}] and icon "{2}"'.format( self.color_name(name, type='group'), ', '.join([color_phid(_id) for _id in _unique_members]), icon) _logger.debug(_msg) if not self._dry_run: _res = self._phab.project.create(name=name, members=_unique_members, icon=icon) _phid = _res['phid'] else: _phid = "PHID-PROJ-notarealproject" self._set_default_policy(_phid) return _phid def set_group_users(self, gid, uids): _unique_uids = list(set(uids)) _msg = 'Setting users {0} as members of group {1}'.format( ', '.join([color_phid(_id) for _id in _unique_uids]), color_phid(gid)) _logger.debug(_msg) transactions = [{"type": "members.set", "value": _unique_uids}] if not self._dry_run: self._phab.project.edit(transactions=transactions, objectIdentifier=gid) else: dry_do(_msg) def create_subgroup(self, name, pgid, members=[], icon='project'): _unique_members = list(set(members)) _msg = 'Creating group {0} as a subgroup of {1} with members {2} and icon "{3}"' \ .format( colored(name, 'red', attrs=['bold']), colored(pgid, attrs=['bold']), ', '.join([color_phid(_id) for _id in _unique_members]), icon) _logger.debug(_msg) transactions = [{"type": "parent", "value": pgid}, {"type": "name", "value": name}, {"type": "icon", "value": icon}] if members is not None: transactions.append({"type": "members.set", "value": _unique_members}) if not self._dry_run: _res = self._phab.project.edit(transactions=transactions) _phid = _res['object']['phid'] else: _phid = 'PHID-PROJ-notarealproject' self._set_default_policy(_phid) return _phid @property def whoami(self): me = self._phab.user.whoami() return me['phid'] diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index 849462b..b148198 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,230 +1,231 @@ # -*- coding: utf-8 -*- import logging import copy from .. import export from .. import colored from .. import color_code from ..repo import Repo from .importer import Importer __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class RepoImporter(Importer): __default_import_scheme = {'type': 'git', 'permissions': {'scheme': 'import'}} def __init__(self, name, config, **kwargs): super().__init__(name, config, self.__default_import_scheme, **kwargs) _logger.debug( 'Initializing importer for Repo {0}' ' with configuration: {1}'.format(self._colored_name, self._config)) def transfer(self, name): _colored_name = colored(name, color_code['repo'], attrs=['bold']) _logger.info('Locking for repo: {0} ({1})'.format(_colored_name, self._colored_name)) _import_scheme = copy.copy(self._config['import-scheme']) _logger.debug(' --> repo info {0}'.format(colored(self._config, attrs=['bold']))) _config = copy.copy(self._config) _type = _config.pop('type', self.__default_import_scheme['type']) _in_repo = Repo(name=name, keyring=self._keyring, dry_run=self._dry_run, type=_type, **self._backend_in) _type = _import_scheme['type'] if _type == 'same': _type = _in_repo.repo_type _name = _import_scheme.pop('name', name).format( original_name=name) _out_repo = Repo(name=_name, keyring=self._keyring, dry_run=self._dry_run, type=_type, **self._backend_out) _queries = {'git': {'git': 'RepoGit', 'svn': 'RepoGitSvn'}, 'svn': {'svn': 'RepoSvnSync'}} _module = _out_repo.repo_type if _module != _in_repo.repo_type: _module = '_'.join([_module, _in_repo.repo_type]) _query_class = None if _type in _queries and _in_repo.repo_type in _queries[_type]: _class_name = _queries[_type][_in_repo.repo_type] _module = __import__( 'getmystuph.backends.repos.{0}'.format(_module), globals(), locals(), [_class_name], 0) _query_class = getattr(_module, _class_name) else: _msg = 'Cannot import a {0} repo in a {1} repo'.format( _type, _in_repo.repo_type) _logger.error(_msg) raise RuntimeError(_msg) - _permissions_scheme = _import_scheme['permissions'] - - _projects = [] - if 'project' in _permissions_scheme: - if type(_permissions_scheme['project']) == list: - _projects = _permissions_scheme['project'] + _phids = [] + if 'tags' in _import_scheme: + if type(_import_scheme['tags']) == list: + _projects = _import_scheme['tags'] else: - _projects = [_permissions_scheme['project']] + _projects = [_import_scheme['tags']] + _phids = self._user_db.get_group_unique_id(_projects) - _out_repo.create(projects=_projects) + _out_repo.create(projects=_phids) _out_perms = Repo.Permissions(_out_repo) + _permissions_scheme = _import_scheme['permissions'] + if 'scheme' not in _permissions_scheme: _logger.error('You must provide a permissions scheme') if _permissions_scheme['scheme'] == 'import': _in_perms = _in_repo.permissions _logger.debug("Replicating permissions {0}".format(_in_perms)) for _type in {'group', 'user'}: _perms_ug = getattr(_in_perms, '{0}s'.format(_type)) for _entity in _perms_ug: _in_id = _entity['id'] _out_id = getattr( self._user_db, _type)(_in_id, create=True) if _type == 'user' and \ _out_id == self._user_db.directory.whoami: _out_id = '_author_' if _out_id is not None: getattr(_out_perms, 'add_{0}'.format(_type))(_out_id, _entity['perm']) else: _logger.warning( 'No permissions to replicate for repository {0}'.format( _colored_name)) elif _permissions_scheme['scheme'] == 'project': if 'project' in _permissions_scheme: _gid = self._user_db.get_group_unique_id( _permissions_scheme['project']) if _gid is not None: _out_perms.groups = [ {'id': _gid, 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] else: _msg = str( 'The project {0} you specified in the ' + 'permissions of repo {1} does not exists' + ' in {2}').format( colored( _permissions_scheme['project'], color_code['group'], attrs=['bold']), _colored_name, self._user_db.directory.backend_name) _logger.error(_msg) raise RuntimeError(_msg) else: _msg = 'You should specify a project name in the ' + \ 'permissions of repo {0}' .format(_colored_name) + \ ' to be able to use the \'project\' import scheme' _logger.error(_msg) raise RuntimeError(_msg) elif _permissions_scheme['scheme'] == 'user': _out_perms.groups = [ {'id': '_author_', 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] elif _permissions_scheme['scheme'] == 'static': for _perm_type in ['edit', 'view', 'push']: if _perm_type not in _permissions_scheme: _msg = 'You should specify a \'{0}\' in the ' + \ 'permissions of repo {1} to be able to use the ' + \ '\'project\' import scheme'.format(_perm_type, _colored_name) _logger.error(_msg) raise RuntimeError(_msg) _equivalent = {'edit': Repo.EDIT, 'view': Repo.VIEW, 'push': Repo.PUSH} for _perm_type in _equivalent.keys(): if _perm_type in _permissions_scheme: _perm_list = _permissions_scheme[_perm_type] if type(_perm_list) is not list: _perm_list = [_perm_list] if _equivalent[_perm_type] is Repo.VIEW: _out_perms.anonymous = False _out_perms.remove_permission(_equivalent[_perm_type]) for _entity in _perm_list: if _entity == '_author_' or \ _entity == '_users_' or \ _entity == '_public_': _out_perms.add_user(_entity, _equivalent[_perm_type]) else: _id = self._user_db.directory.get_group_unique_id( _entity) if _id is not None: _out_perms.add_group(_id, _equivalent[_perm_type]) else: _logger.error( 'The project {0} was not found in {1}'.format( _entity, self._user_db.directory.backend_name)) if _out_perms is not None: _out_repo.set_permissions(_out_perms) if _query_class is not None: _out_repo.enable() with _query_class(_in_repo, out_repo=_out_repo, dry_run=self._dry_run, keyring=self._keyring, user_db=self._user_db) as _clone: _clone.clone() _branches = _clone.branches _tags = _clone.tags for b in _branches: _logger.debug("Branch: {0}".format( colored(b, attrs=['bold']))) for t in _tags: _logger.debug("Tag: {0}".format( colored(t, attrs=['bold']))) _out_repo.wait_enabled() _clone.push()