diff --git a/example.yaml b/example.yaml index 473cdb3..639f44a 100644 --- a/example.yaml +++ b/example.yaml @@ -1,113 +1,117 @@ global: user_db_cache: "cache.yaml" # file to save ldap queries to all: [svn, git] # Only migrate one type of repo use_keyring: true # Save your password for each repo access in: # Source of the repositories backend: epfl # only epfl backend implemented username: out: backend: c4science username: token: cli-n5dutb2wv26ivcpo66yvb3sbk64g # API token obtained in the settings groups: __all__: import-scheme: # Global project options type: sub-project # project | sub-project | ignore # if sub-project, specify a project to import the group # to. If set to project, will create one project per group. # ignore will skip the group creation project: test_import # only for sub-project type, which project to attach the sub-project to name: test_{original_name} # if you want to prefix or suffix your Project user-import-scheme: additive # additive (default) or synchronize # Sync will delete user on c4science if they # don't exist in the group force-members: username # List or single username from the source backend that force-members: # Will be added to the group instead of the source - username1 # members - username2 scitas-ge-unit: # Specific options for the scitas-ge-unit ldap group import-scheme: type: project name: scitas-ge hpc-lsms: # this will created the project if no repo is given, otherwhise # will inherit from __all__ import-scheme lsms-unit: # Example of a EPFL unit to import as lsms Project import-scheme: name: lsms repositories: __all__: # __all__ is the default for other repo, you can override this in each repo type: git # svn | git (default git) import-scheme: type: same # git | svn | skip # the type to migrate to. # (only git->git, svn->svn and svn->git are supported) # Skip will ignore the repository (if for instance it already has # been imported. name: gms_{original_name} # default: '{original_name}' tags: project_name # String or List of project to tag the repository with # When using project in the same subproject hierarchy, # only the child will be kept + permission_only: false # default: false + # if this is set to true it will ONLY create the repository and set the permissions in accordance to the permissions scheme selected + ignore_if_present: false # default: false + # if set to true it will skip the import if the repository is already present in the destination permissions: scheme: import # import | project | user | static # # import # Import the groups and users structure from the source # scheme: import # # project # Set the repo policy to a specific Project name # scheme: project # project: # # user # Set the repo policy to the repository author (you) # scheme: user # # static # Custom permissions # scheme: static # view: | _author_ | _public_ | _users_ # push: | _author_ | _public_ | _users_ # edit: | _author_ | _public_ | _users_ iohelper: # The repository name import-scheme: permissions: # Custom permissions scheme: static view: _public_ # Public repository on c4science push: _users_ # Only c4science users can push edit: lsms # User from the lsms Project can edit # Some more examples toto: # Repository wihtout options, include a blank line after akantu: # Only import master and no tags import-scheme: branches: [master] tags: [] test: # Import an svn repository type: svn import-scheme: permissions: scheme: project project: scitas-ge view: _public_ test: # Transform an svn repo to git type: svn import-scheme: name: test-git type: git diff --git a/getmystuph/backends/repos/phabricator.py b/getmystuph/backends/repos/phabricator.py index 4633723..ad5ee9f 100644 --- a/getmystuph/backends/repos/phabricator.py +++ b/getmystuph/backends/repos/phabricator.py @@ -1,295 +1,297 @@ # -*- coding: utf-8 -*- import logging import re import time # import copy from ... import export from ... import colored from ... import dry_do from ...repo import Repo from ...utils import get_phabricator_instance from .. import color_phid __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class PhabRepo(Repo): def __init__(self, *args, host=None, username=None, token=None, **kwargs): super().__init__(username=username, **kwargs) # options = copy.copy(kwargs) # _create = options.pop('create', False) self._host = '{0}/api/'.format(host.rstrip('/')) self._server = None _server_re = re.compile(r'https?://(.*)') _match = _server_re.match(host.rstrip('/')) if _match is not None: self._server = _match.group(1) if self._server is None: raise RuntimeError( 'Cannot extract the server name for repo {0} from {1}'.format( self._colored_name, host)) self._phab = get_phabricator_instance(host=self._host, username=username, token=token) _data = self._phab.diffusion.repository.search( queryKey="all", constraints={"name": self._name, "types": [self.repo_type]})['data'] self._phab_id = None for _repo in _data: _repo_name = _repo['fields']['name'] if _repo_name == self._name: self._phab_id = _repo['phid'] self._id = _repo['id'] _logger.debug('Repositories {0} has id {1}'.format( self._colored_name, self._phab_id)) if self._phab_id is None: _logger.debug('Repositories {0} not in phabricator'.format( self._colored_name)) def create(self, projects=[]): if self._phab_id is not None: _msg = 'The repository {0}:{1} already exists'.format( self._colored_name, self._phab_id) _logger.error(_msg) _msg = 'Adding tags [{0}] to repository {1}'.format( ','.join(colored(p, attrs=['bold']) for p in projects), self._colored_name) _logger.info(_msg) if self._dry_run: dry_do(_msg) return self._phab.diffusion.repository.edit( transactions=[{'type': 'projects.add', 'value': projects}], objectIdentifier=self._phab_id) - return + return False # raise RuntimeError(_msg) if self._dry_run: self._phab_id = "PHID-REPO-notarealrepo" else: _data = self._phab.diffusion.repository.edit( transactions=[{'type': 'name', 'value': self._name}, {'type': 'vcs', 'value': self.repo_type}, {'type': 'projects.add', 'value': projects}]) self._creation_data = _data['object'] self._phab_id = self._creation_data['phid'] _msg = 'Created repository {0} id {1} [{2}]'.format( self._colored_name, color_phid(self._phab_id), self.repo_type) _logger.info(_msg) if self._dry_run: dry_do(_msg) - self._id = 666 + self._id = 666 # this is a fake id else: self._id = self._creation_data['id'] + return True + @property def url(self): if self.repo_type == 'git': self._url = 'git@{0}:/diffusion/{1}/{2}.git'.format( self._server, self._id, self._name) elif self.repo_type == 'svn': self._url = 'svn+ssh://git@{0}/diffusion/{1}/'.format( self._server, self._id) return self._url def enable(self): _msg = 'Activating repository {0} [{1}]'.format( self._colored_name, color_phid(self._phab_id)) _logger.info(_msg) if self._dry_run: dry_do(_msg) else: self._phab.diffusion.repository.edit( transactions=[{'type': 'status', 'value': 'active'}], objectIdentifier=self._phab_id) def _dangerous_changes(self, state): _msg = '{0}ctivating dangerous changes on repository {1}'.format( 'A' if state else 'De-a', self._colored_name, color_phid(self._phab_id)) _logger.info(_msg) if self._dry_run: dry_do(_msg) return self._phab.diffusion.repository.edit( transactions=[{ 'type': 'allowDangerousChanges', 'value': state }], objectIdentifier=self._phab_id) def join_the_dark_side(self): self._dangerous_changes(True) def join_the_good_side(self): self._dangerous_changes(False) def wait_enabled(self, timeout=3600): _msg = 'Checking if {0} [{1}] is activated'.format( self._colored_name, color_phid(self._phab_id)) _logger.info(_msg) if self._dry_run: dry_do(_msg) else: _time = 0 while True: _data = self._phab.diffusion.repository.search( queryKey="all", constraints={"phids": [self._phab_id]})['data'] if not len(_data) == 1: raise RuntimeError('Cannot find the repo {0}'.format( self._colored_name)) _status = _data[0]['fields']['status'] _importing = _data[0]['fields']['isImporting'] if _status == 'active' and not _importing: return if _time > timeout: _msg = 'Timeout reached before repo activation' _logger.debug(_msg) raise RuntimeError(_msg) time.sleep(1) _time += 1 def set_permissions(self, permissions): _perms = {'edit': [], 'push': [], 'view': []} _equivalent = {'edit': Repo.EDIT, 'view': Repo.VIEW, 'push': Repo.PUSH} _phab_perms = {'edit': 'edit', 'view': 'view', 'push': 'policy.push'} _special_perms = {'_author_': 'obj.repository.author', '_users_': 'users', '_public_': 'public'} for _type in {'group', 'user'}: _perms_ug = getattr(permissions, '{0}s'.format(_type)) for _entity in _perms_ug: _id = _entity['id'] if _id in _special_perms: _id = _special_perms[_id] for _phab, _gen in _equivalent.items(): if _entity['perm'] & _gen: _perms[_phab].append(_id) if permissions.anonymous: _perms['view'] = ['public'] for _type in ['push', 'view', 'edit']: if _type not in _perms: continue _perms[_type] = list(set(_perms[_type])) if len(_perms[_type]) > 1: if 'public' in _perms[_type]: _perms[_type] = 'public' continue if 'users' in _perms[_type]: _perms[_type] = 'users' continue # create custom policy to replace the list regex = re.compile(r'PHID-([A-Z]{4})-.+') _lists = {'PROJ': [], 'USER': []} for _p in _perms[_type]: match = regex.match(_p) if match is not None: _lists[match.group(1)].append(_p) elif _p == 'obj.repository.author': _lists['USER'].append(self._directory.whoami) _policy = [ { "action": "allow", "rule": 'PhabricatorUsersPolicyRule', "value": _lists['USER'] }, { "action": "allow", "rule": 'PhabricatorProjectsPolicyRule', "value": _lists['PROJ'] }] _msg = 'Creating policy for users [{0}] and projects [{1}]' \ .format( ', '.join([color_phid(_el) for _el in _lists['USER']]), ', '.join([color_phid(_el) for _el in _lists['PROJ']])) _logger.debug(_msg) if self._dry_run: dry_do(_msg) _phid = 'PHID-PLCY-notapolicy' else: _data = self._phab.policy.create( objectType='REPO', default='deny', policy=_policy) _phid = _data['phid'] _logger.info('Replacing list {0} by policy {1}'.format( _perms[_type], _phid)) _perms[_type] = _phid else: _perms[_type] = _perms[_type][0] _msg = 'Setting \'{0}\' permissions for {1} to {2}:'.format( _type, self._colored_name, color_phid(_perms[_type])) _logger.info(_msg) if not self._dry_run: self._phab.diffusion.repository.edit( transactions=[{'type': _phab_perms[_type], 'value': _perms[_type]}], objectIdentifier=self._phab_id) else: dry_do(_msg) diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index 4745993..bdf3482 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,280 +1,291 @@ import logging import copy from .. import export from .. import colored from .. import color_code from ..repo import Repo from .importer import Importer __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class RepoImporter(Importer): __default_import_scheme = {'type': 'git', + 'permission_only': False, + 'ignore_if_present': False, 'permissions': {'scheme': 'import'}} def __init__(self, name, config, **kwargs): super().__init__(name, config, self.__default_import_scheme, **kwargs) self._trust_cert = kwargs.pop('trust_cert', True) _logger.debug( 'Initializing importer for Repo {0}' ' with configuration: {1}'.format(self._colored_name, self._config)) def transfer(self, name): _colored_name = colored(name, color_code['repo'], attrs=['bold']) _logger.info('Locking for repo: {0} ({1})'.format(_colored_name, self._colored_name)) _import_scheme = copy.copy(self._config['import-scheme']) _logger.debug(' --> repo info {0}'.format(colored(self._config, attrs=['bold']))) _config = copy.copy(self._config) _type = _config.pop('type', self.__default_import_scheme['type']) _in_repo = Repo(name=name, keyring=self._keyring, dry_run=self._dry_run, user_db=self._user_db, type=_type, **self._backend_in) _type = _import_scheme['type'] if _type == 'skip': _logger.info('Skipping repository {0}'.format(_colored_name)) return if _type == 'same': _type = _in_repo.repo_type _name = _import_scheme.pop('name', name).format( original_name=name) _out_repo = Repo(name=_name, keyring=self._keyring, dry_run=self._dry_run, user_db=self._user_db, type=_type, **self._backend_out) _queries = {'git': {'git': 'RepoGit', 'svn': 'RepoGitSvn'}, 'svn': {'svn': 'RepoSvnSync'}} _module = _out_repo.repo_type if _module != _in_repo.repo_type: _module = '_'.join([_module, _in_repo.repo_type]) _query_class = None if _type in _queries and _in_repo.repo_type in _queries[_type]: _class_name = _queries[_type][_in_repo.repo_type] _module = __import__( 'getmystuph.backends.repos.{0}'.format(_module), globals(), locals(), [_class_name], 0) _query_class = getattr(_module, _class_name) else: _msg = 'Cannot import a {0} repo in a {1} repo'.format( _type, _in_repo.repo_type) _logger.error(_msg) raise RuntimeError(_msg) _tags_phids = [] if 'tags' in _import_scheme: if type(_import_scheme['tags']) == list: _tags = _import_scheme['tags'] else: _tags = [_import_scheme['tags']] for _tag in _tags: if _tag is None: continue _phid = self._user_db.get_group_unique_id(_tag) if _phid is not None: _tags_phids.append(_phid) - _out_repo.create(projects=_tags_phids) + _created = _out_repo.create(projects=_tags_phids) + if not _created and _import_scheme['ignore_if_present']: + _logger.info('Skipping existing repository {0}'.format( + _colored_name)) + return _out_perms = Repo.Permissions(_out_repo) _permissions_scheme = _import_scheme['permissions'] if 'scheme' not in _permissions_scheme: _logger.error('You must provide a permissions scheme') if _permissions_scheme['scheme'] == 'import': _in_perms = _in_repo.permissions _logger.debug("Replicating permissions {0}".format(_in_perms)) for _type in {'group', 'user'}: _perms_ug = getattr(_in_perms, '{0}s'.format(_type)) for _entity in _perms_ug: _in_id = _entity['id'] _out_id = getattr( self._user_db, _type)(_in_id, create=True) if _type == 'user' and \ _out_id == self._user_db.directory.whoami: _out_id = '_author_' if _out_id is not None: getattr(_out_perms, 'add_{0}'.format(_type))(_out_id, _entity['perm']) else: _logger.warning( 'No permissions to replicate for repository {0}'.format( _colored_name)) elif _permissions_scheme['scheme'] == 'project': if 'project' in _permissions_scheme: _gid = self._user_db.get_group_unique_id( _permissions_scheme['project']) if _gid is not None: _out_perms.groups = [ {'id': _gid, 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] else: _msg = str( 'The project {0} you specified in the ' + 'permissions of repo {1} does not exists' + ' in {2}').format( colored( _permissions_scheme['project'], color_code['group'], attrs=['bold']), _colored_name, self._user_db.directory.backend_name) _logger.error(_msg) raise RuntimeError(_msg) else: _msg = 'You should specify a project name in the ' + \ 'permissions of repo {0}' .format(_colored_name) + \ ' to be able to use the \'project\' import scheme' _logger.error(_msg) raise RuntimeError(_msg) elif _permissions_scheme['scheme'] == 'sub-project': if 'project' in _permissions_scheme: _project_name = _permissions_scheme['project'] _gid = self._user_db.get_group_unique_id(_project_name, create=True) _info = {'View': {'perm': Repo.VIEW}, 'Edit': {'perm': Repo.EDIT}, 'Push': {'perm': Repo.PUSH}} _in_perms = _in_repo.permissions for _perm in ['View', 'Push', 'Edit']: _perm_filter = _info[_perm]['perm'] _members = set() for _in_id in _in_perms.all_users( perm_filter=_perm_filter): _out_id = self._user_db.user(_in_id) if _out_id is not None: _members.add(_out_id) _info[_perm]['id'] = self._user_db.get_group_unique_id( '{0} {1}'.format(_project_name, _perm), create=True, members=_members, transfer_options={'force_type': 'sub-project', 'project': _project_name}) _out_perms.groups = [ {'id': _gid, 'perm': Repo.VIEW}, {'id': _info['Edit']['id'], 'perm': Repo.EDIT}, {'id': _info['Push']['id'], 'perm': Repo.PUSH}, ] else: _msg = 'You should specify a project name in the ' + \ 'permissions of repo {0}' .format(_colored_name) + \ ' to be able to use the \'project\' import scheme' _logger.error(_msg) raise RuntimeError(_msg) elif _permissions_scheme['scheme'] == 'user': _out_perms.groups = [ {'id': '_author_', 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] elif _permissions_scheme['scheme'] == 'static': for _perm_type in ['edit', 'view', 'push']: if _perm_type not in _permissions_scheme: _msg = 'You should specify a \'{0}\' in the ' + \ 'permissions of repo {1} to be able to use the ' + \ '\'project\' import scheme'.format(_perm_type, _colored_name) _logger.error(_msg) raise RuntimeError(_msg) _equivalent = {'edit': Repo.EDIT, 'view': Repo.VIEW, 'push': Repo.PUSH} for _perm_type in _equivalent.keys(): if _perm_type in _permissions_scheme: _perm_list = _permissions_scheme[_perm_type] if type(_perm_list) is not list: _perm_list = [_perm_list] if _equivalent[_perm_type] is Repo.VIEW: _out_perms.anonymous = False _out_perms.remove_permission(_equivalent[_perm_type]) for _entity in _perm_list: if _entity == '_author_' or \ _entity == '_users_' or \ _entity == '_public_': _out_perms.add_user(_entity, _equivalent[_perm_type]) else: _id = self._user_db.directory.get_group_unique_id( _entity) if _id is not None: _out_perms.add_group(_id, _equivalent[_perm_type]) else: _logger.error( 'The project {0} was not found in {1}'.format( _entity, self._user_db.directory.backend_name)) if _out_perms is not None: _out_repo.set_permissions(_out_perms) + if _import_scheme['permission_only']: + _logger.info('Skipping data import for repository {0}'.format( + _colored_name)) + return + if _query_class is not None: _out_repo.enable() with _query_class(_in_repo, out_repo=_out_repo, dry_run=self._dry_run, keyring=self._keyring, user_db=self._user_db, trust_cert=self._trust_cert) as _clone: _clone.clone() _branches = _clone.branches _tags = _clone.tags for b in _branches: _logger.debug("Branch: {0}".format( colored(b, attrs=['bold']))) for t in _tags: _logger.debug("Tag: {0}".format( colored(t, attrs=['bold']))) _out_repo.wait_enabled() _clone.push()