diff --git a/getmystuph/importers/import_user_db.py b/getmystuph/importers/import_user_db.py index 8a4f03c..0b39c5d 100644 --- a/getmystuph/importers/import_user_db.py +++ b/getmystuph/importers/import_user_db.py @@ -1,259 +1,259 @@ # -*- coding: utf-8 -*- import logging import yaml from .. import export from .. import colored from .group_importer import GroupImporter __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class ImportUserDB: def __init__(self, in_directory, out_directory, **kwargs): self._in_directory = in_directory self._out_directory = out_directory self._users = {} self._imported_groups = {} self._default_importer_config = {} self._cache_file = kwargs.pop('cache_file', None) if self._cache_file is not None: self._populate() self._keyring = kwargs.pop('keyring', None) self._dry_run = kwargs.pop('dry_run', False) def _populate(self): try: with open(self._cache_file, 'r+') as _cache_file: _data = yaml.load(_cache_file) if 'users' in _data: self._users = _data['users'] if 'groups' in _data: self._imported_groups = _data['groups'] except FileNotFoundError: pass def save(self): if self._cache_file is None: return with open(self._cache_file, 'w') as _cache_file: yaml.dump( {'users': self._users, 'groups': self._imported_groups}, _cache_file, default_flow_style=False) def import_group(self, _id, _imported_id, _in_name, _out_name): self._imported_groups[_id] = {'out_id': _imported_id, 'in_name': _in_name, 'out_name': _out_name} def add_users(self, users): for _user in users: - if _user in self._users: + if _user in self._users and 'oid' in self._users[_user]: _logger.debug( 'User {0} ({1}) from {2} already in cache [{3} - {4}]' .format( self._in_directory.color_name( self._users[_user]['name'], type='user'), self._in_directory.color_name(_user), self._in_directory.backend_name, self._out_directory.color_name( self._users[_user]['oid']), self._out_directory.backend_name)) continue _user_info = {'id': _user} try: _mail = self._in_directory.get_user_email(_user) _user_info['email'] = _mail _user_info['name'] = self._in_directory.get_user_name(_user) _user_info['in_login'] = self._in_directory.get_user_login( _user) _out_id = self._out_directory.get_user_unique_id(_mail) if _out_id is not None: _logger.debug( 'Found {0} ({1}) in {2} as {3} in {4}'.format( self._in_directory.color_name(_user_info['name'], type='user'), self._in_directory.color_name(_user), self._in_directory.backend_name, self._out_directory.color_name(_out_id), self._out_directory.backend_name)) _user_info['oid'] = _out_id self._users[_user] = _user_info else: _logger.debug( 'Did not find {0} ({1}) from {2} in {3}'.format( self._in_directory.color_name(_user_info['name'], type='user'), self._in_directory.color_name(_user), self._in_directory.backend_name, self._out_directory.backend_name)) except: _logger.warning( "The user {0} does not exists in {1}".format( self._in_directory.color_name(_user), colored(self._in_directory.backend_name, attrs=['bold']))) def get_user_oids(self, users_ids): return [self._users[_user]['oid'] for _user in users_ids if (_user in self._users) and ('oid' in self._users[_user])] @property def default_importer(self): return self._default_importer_config @default_importer.setter def default_importer(self, importer_config): self._default_importer_config = importer_config @property def users(self): return self._users @property def directory(self): return self._out_directory def get_importer(self, name): if name not in self._default_importer_config: name = '__all__' importer = GroupImporter( name, self._default_importer_config[name], backend_in={'directory': self._in_directory}, backend_out={'directory': self._out_directory}, user_db=self, keyring=self._keyring, dry_run=self._dry_run) return importer @property def in_directory(self): return self._in_directory def group(self, _id, create=False): if _id in self._imported_groups: _logger.debug('Found group {0} in cache: {1}'.format( self._in_directory.color_name(_id), self._out_directory.color_name( self._imported_groups[_id]['out_name']))) return self._imported_groups[_id]['out_id'] else: _name = self._in_directory.get_group_name(_id) if _name is not None: _importer = self.get_importer(_name) _t_name = _importer.transfered_name(_name) _out = self._out_directory.get_group_unique_id( _t_name, create=create, in_name=_name) if _out is None: _logger.error( ' Could not find {1} ({0}) in directory {2} {3}' .format( self._in_directory.color_name(_id), self._in_directory.color_name( _t_name, type='group'), self._out_directory.backend_name, create)) return _out else: _logger.error( ' Could not find {0} in directory {1}'.format( _id, self._in_directory.backend_name)) return None def group_by_in_name(self, _name): for _id, data in self._imported_groups.items(): if data['in_name'] == _name: return data['out_id'] return None def group_by_out_name(self, _name): for _id, data in self._imported_groups.items(): if data['out_name'] == _name: return data['out_id'] return None def get_group_unique_id(self, _name, create=False, in_name=None, members=[]): _gid = self.group_by_out_name(_name) if _gid is not None: return _gid _gid = self._out_directory.get_group_unique_id(_name) if _gid is not None: _logger.debug("Found group {0} in {1}: {2}".format( self._out_directory.color_name(_name), self._out_directory.backend_name, self._out_directory.color_name(_gid) )) if members: members.extend(self._out_directory.get_users_from_group(_gid)) self._out_directory.set_group_users(members) return _gid if not create: return None if in_name is None: in_name = _name _logger.debug("Group {0} not in {1}, try to create it".format( self._out_directory.color_name(_name), self._out_directory.backend_name )) _importer = self.get_importer(in_name) _gid = _importer.transfer(in_name) if members: self._out_directory.set_group_users(members) return _gid def user_by_in_login(self, login): for _id, data in self._users.items(): if data['in_login'] == login: return data _id = self._in_directory.get_user_unique_id_from_login(login) if _id is not None: _name = self._in_directory.get_user_name(_id) _email = self._in_directory.get_user_email(_id) self._users[_id] = { 'name': _name, 'email': _email, 'in_login': login } return self._users[_id] return None def user(self, _id, **kwargs): if _id in self._users and 'oid' in self._users[_id]: _logger.debug('Found user {0} in cache: {1}'.format( self._in_directory.color_name(_id), self._out_directory.color_name(self._users[_id]['oid']))) return self._users[_id]['oid'] else: _email = self._in_directory.get_user_email(_id) if _email is not None: return self._out_directory.get_user_unique_id(_email) return None diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index dfc8496..3e377ec 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,264 +1,268 @@ import logging import copy from .. import export from .. import colored from .. import color_code from ..repo import Repo from .importer import Importer __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class RepoImporter(Importer): __default_import_scheme = {'type': 'git', 'permissions': {'scheme': 'import'}} def __init__(self, name, config, **kwargs): super().__init__(name, config, self.__default_import_scheme, **kwargs) self._trust_cert = kwargs.pop('trust_cert', True) _logger.debug( 'Initializing importer for Repo {0}' ' with configuration: {1}'.format(self._colored_name, self._config)) def transfer(self, name): _colored_name = colored(name, color_code['repo'], attrs=['bold']) _logger.info('Locking for repo: {0} ({1})'.format(_colored_name, self._colored_name)) _import_scheme = copy.copy(self._config['import-scheme']) _logger.debug(' --> repo info {0}'.format(colored(self._config, attrs=['bold']))) _config = copy.copy(self._config) _type = _config.pop('type', self.__default_import_scheme['type']) _in_repo = Repo(name=name, keyring=self._keyring, dry_run=self._dry_run, type=_type, **self._backend_in) _type = _import_scheme['type'] if _type == 'skip': _logger.info('Skipping repository {0}'.format(_colored_name)) return if _type == 'same': _type = _in_repo.repo_type _name = _import_scheme.pop('name', name).format( original_name=name) _out_repo = Repo(name=_name, keyring=self._keyring, dry_run=self._dry_run, type=_type, **self._backend_out) _queries = {'git': {'git': 'RepoGit', 'svn': 'RepoGitSvn'}, 'svn': {'svn': 'RepoSvnSync'}} _module = _out_repo.repo_type if _module != _in_repo.repo_type: _module = '_'.join([_module, _in_repo.repo_type]) _query_class = None if _type in _queries and _in_repo.repo_type in _queries[_type]: _class_name = _queries[_type][_in_repo.repo_type] _module = __import__( 'getmystuph.backends.repos.{0}'.format(_module), globals(), locals(), [_class_name], 0) _query_class = getattr(_module, _class_name) else: _msg = 'Cannot import a {0} repo in a {1} repo'.format( _type, _in_repo.repo_type) _logger.error(_msg) raise RuntimeError(_msg) - _phids = [] + _tags_phids = [] if 'tags' in _import_scheme: if type(_import_scheme['tags']) == list: - _projects = _import_scheme['tags'] + _tags = _import_scheme['tags'] else: - _projects = [_import_scheme['tags']] - _phids = self._user_db.get_group_unique_id(_projects) + _tags = [_import_scheme['tags']] - _out_repo.create(projects=_phids) + for _tag in _tags: + if _tag == None: continue + _phid = self._user_db.get_group_unique_id(_tag) + _tags_phids.append(_phid) + + _out_repo.create(projects=_tags_phids) _out_perms = Repo.Permissions(_out_repo) _permissions_scheme = _import_scheme['permissions'] if 'scheme' not in _permissions_scheme: _logger.error('You must provide a permissions scheme') if _permissions_scheme['scheme'] == 'import': _in_perms = _in_repo.permissions _logger.debug("Replicating permissions {0}".format(_in_perms)) for _type in {'group', 'user'}: _perms_ug = getattr(_in_perms, '{0}s'.format(_type)) for _entity in _perms_ug: _in_id = _entity['id'] _out_id = getattr( self._user_db, _type)(_in_id, create=True) if _type == 'user' and \ _out_id == self._user_db.directory.whoami: _out_id = '_author_' if _out_id is not None: getattr(_out_perms, 'add_{0}'.format(_type))(_out_id, _entity['perm']) else: _logger.warning( 'No permissions to replicate for repository {0}'.format( _colored_name)) elif _permissions_scheme['scheme'] == 'project': if 'project' in _permissions_scheme: _gid = self._user_db.get_group_unique_id( _permissions_scheme['project']) if _gid is not None: _out_perms.groups = [ {'id': _gid, 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] else: _msg = str( 'The project {0} you specified in the ' + 'permissions of repo {1} does not exists' + ' in {2}').format( colored( _permissions_scheme['project'], color_code['group'], attrs=['bold']), _colored_name, self._user_db.directory.backend_name) _logger.error(_msg) raise RuntimeError(_msg) else: _msg = 'You should specify a project name in the ' + \ 'permissions of repo {0}' .format(_colored_name) + \ ' to be able to use the \'project\' import scheme' _logger.error(_msg) raise RuntimeError(_msg) elif _permissions_scheme['scheme'] == 'sub-project': if 'project' in _permissions_scheme: _project_name = _permissions_scheme['project'] _gid = self._user_db.get_group_unique_id(_project_name, create=True) _info = {'View': {'perm': Repo.VIEW}, 'Edit': {'perm': Repo.EDIT}, 'Push': {'perm': Repo.PUSH}} _in_perms = _in_repo.permissions for _perm in _info.keys(): _members = _in_perms.all_users( perm_filter=_info[_perm]['perm']) _info[_perm]['id'] = self._user_db.get_group_unique_id( '{0} {1}'.format(_project_name, _perm), create=True, members=_members) _out_perms.groups = [ {'id': _gid, 'perm': Repo.VIEW}, {'id': _info['Edit']['id'], 'perm': Repo.EDIT}, {'id': _info['Push']['id'], 'perm': Repo.PUSH}, ] else: _msg = 'You should specify a project name in the ' + \ 'permissions of repo {0}' .format(_colored_name) + \ ' to be able to use the \'project\' import scheme' _logger.error(_msg) raise RuntimeError(_msg) elif _permissions_scheme['scheme'] == 'user': _out_perms.groups = [ {'id': '_author_', 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] elif _permissions_scheme['scheme'] == 'static': for _perm_type in ['edit', 'view', 'push']: if _perm_type not in _permissions_scheme: _msg = 'You should specify a \'{0}\' in the ' + \ 'permissions of repo {1} to be able to use the ' + \ '\'project\' import scheme'.format(_perm_type, _colored_name) _logger.error(_msg) raise RuntimeError(_msg) _equivalent = {'edit': Repo.EDIT, 'view': Repo.VIEW, 'push': Repo.PUSH} for _perm_type in _equivalent.keys(): if _perm_type in _permissions_scheme: _perm_list = _permissions_scheme[_perm_type] if type(_perm_list) is not list: _perm_list = [_perm_list] if _equivalent[_perm_type] is Repo.VIEW: _out_perms.anonymous = False _out_perms.remove_permission(_equivalent[_perm_type]) for _entity in _perm_list: if _entity == '_author_' or \ _entity == '_users_' or \ _entity == '_public_': _out_perms.add_user(_entity, _equivalent[_perm_type]) else: _id = self._user_db.directory.get_group_unique_id( _entity) if _id is not None: _out_perms.add_group(_id, _equivalent[_perm_type]) else: _logger.error( 'The project {0} was not found in {1}'.format( _entity, self._user_db.directory.backend_name)) if _out_perms is not None: _out_repo.set_permissions(_out_perms) if _query_class is not None: _out_repo.enable() with _query_class(_in_repo, out_repo=_out_repo, dry_run=self._dry_run, keyring=self._keyring, user_db=self._user_db, trust_cert=self._trust_cert) as _clone: _clone.clone() _branches = _clone.branches _tags = _clone.tags for b in _branches: _logger.debug("Branch: {0}".format( colored(b, attrs=['bold']))) for t in _tags: _logger.debug("Tag: {0}".format( colored(t, attrs=['bold']))) _out_repo.wait_enabled() _clone.push()