diff --git a/getmystuph/backends/repos/git.py b/getmystuph/backends/repos/git.py index b3b7249..107c774 100644 --- a/getmystuph/backends/repos/git.py +++ b/getmystuph/backends/repos/git.py @@ -1,102 +1,99 @@ # -*- coding: utf-8 -*- import os import git import logging from ... import colored from ... import dry_do from ...repo import RepoQuery __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart", "Jean-Baptiste Aubort"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) class RepoGit(RepoQuery): """This class handles the common part on git repositories, cloning, retreiving tags/branches doing subtrees """ - def __enter__(self): - super().__enter__() + def clone(self): _logger.info('Cloning repo {0} [{1}] in {2}'.format( self._in_repo.color_name(self._name), self._url, colored(self.working_dir, attrs=['bold']))) if not os.path.isdir(os.path.join(self.working_dir, '.git')): self._repo = git.Repo.clone_from(self._url, self.working_dir) else: _logger.warning('Repo {0} is already cloned in {1}'.format( self._in_repo.color_name(self._name), colored(self.working_dir, attrs=['bold']))) self._repo = git.Repo(self.working_dir) if self._out_repo is not None: _logger.debug('Adding remote {0} [{1}] to clone of {2}'.format( colored(self._out_repo.backend_name, attrs=['bold']), self._out_repo.url, self._in_repo.color_name(self._name))) if self._out_repo.backend_name in [_r.name for _r in self._repo.remotes]: _logger.debug('Remote {0} already present in {2}'.format( colored(self._out_repo.backend_name, attrs=['bold']), self._out_repo.url, self._in_repo.color_name(self._name))) else: self._repo.create_remote(self._out_repo.backend_name, self._out_repo.url) - return self - @property def tags(self): _tags = [] for ref in self._repo.refs: if type(ref) == git.refs.tag.TagReference: _tags.append(str(ref)) return _tags @property def branches(self): _refs = [] for ref in self._repo.refs: if type(ref) != git.refs.tag.TagReference and\ ref.name != 'origin/HEAD': _refs.append(str(ref)) return _refs def push(self): class PushProgress(git.remote.RemoteProgress): def update(op_code, cur_count, max_count=None, message=''): print('update({0}, {1}, {2}, {3})'.format(op_code, cur_count, max_count, message)) _remote = self._repo.remote(name=self._out_repo.backend_name) _msg = 'Pushing repo {0} data to {1}'.format( self._in_repo.color_name(self._name), colored(self._out_repo.backend_name, attrs=['bold'])) _logger.info(_msg) if self._dry_run: dry_do(_msg) else: if len(self._repo.refs) == 0: raise RuntimeError('Repository is empty') for ref in self._repo.refs: if type(ref) == git.refs.remote.RemoteReference and\ ref.name != 'origin/HEAD': self._repo.create_head(ref.remote_head, commit=ref.name) _remote.push(all=True) _remote.push(tags=True) diff --git a/getmystuph/backends/repos/git_svn.py b/getmystuph/backends/repos/git_svn.py index 4d71be7..5dfb60c 100644 --- a/getmystuph/backends/repos/git_svn.py +++ b/getmystuph/backends/repos/git_svn.py @@ -1,128 +1,130 @@ # -*- coding: utf-8 -*- import os import re import git -import pysvn +import svn.remote import logging from ... import colored from ... import color_code from ...utils import get_password from .git import RepoGit __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) class RepoGitSvn(RepoGit): """This class handles the git svn""" def __enter__(self): - self._create_stage() + super().__enter__() _logger.info('Getting repo {0} [{1}] from svn server'.format( self._in_repo.color_name(self._name), self._url)) - def _get_login(realm, username, may_save): - password = get_password(self._in_repo.backend_name, - self._in_repo.username, - keyring=self._keyring) - return True, self._in_repo.username, password, False + _auth = {} + if (re.match('^https://', self._url)): + _auth['username'] = self._username + _auth['password'] = get_password(self._in_repo.backend_name, + self._in_repo.username, + keyring=self._keyring) + + self._svn_client = svn.remote.RemoteClient(self._url, **_auth) + return self - _svn_client = pysvn.Client() - _svn_client.callback_get_login = _get_login - _svn_logs = _svn_client.log(self._url) + def clone(self): + _svn_logs = self._svn_client.log_default() _svn_authors = set() for l in _svn_logs: - _svn_authors.add(l['author']) + _svn_authors.add(l.author) _authors = {} for _a in _svn_authors: _user = self._user_db.user_by_in_login(_a) if _user is not None: _authors[_a] = _user _logger.debug( 'Matching author {0} with "{name} <{email}>"'.format( colored(_a, color_code['user'], attrs=['bold']), **_user)) else: _authors[_a] = {'name': _a, 'email': '{0}@{1}'.format( _a, self._in_repo.backend_name)} _svn_authors_file = os.path.join(self.working_dir, 'getmystuph.authors') with open(_svn_authors_file, 'w') as afh: afh.write( '\n'.join( ['{0} = {name} <{email}>'.format(l, **a) for l, a in _authors.items()])) - _list = _svn_client.list(self._url, recurse=False) + _list = self._svn_client.list() _branches = False _trunk = False _tags = False for _file in _list: - _path = _file[0].repos_path.lstrip('/') - if re.match(r'branches$', _path): + _path = _file.rstrip('/') + if re.match(r'^branches$', _path): _branches = True - if re.match(r'trunk$', _path): + if re.match(r'^trunk$', _path): _trunk = True - if re.match(r'tags$', _path): + if re.match(r'^tags$', _path): _tags = True _stdlayout = False if _branches and _tags and _trunk: _logger.debug('Detected svn standard layout for {0}'.format( self._in_repo.color_name(self._name))) _stdlayout = True _logger.info('Cloning repo {0} [{1}] in {2}'.format( self._in_repo.color_name(self._name), self._url, colored(self.working_dir, attrs=['bold']))) if not os.path.isdir(os.path.join(self.working_dir, '.git')): _git = git.Git(self.working_dir) _git.svn('clone', self._url, self.working_dir, preserve_empty_dirs=True, authors_file=_svn_authors_file, stdlayout=_stdlayout) - super().__enter__() + super().clone() _tag_re = re.compile('origin/tags/(.*)') _trunk_re = re.compile('origin/trunk') for _ref in self._repo.refs: if type(_ref) == git.refs.remote.RemoteReference: _match = _tag_re.match(_ref.name) if _match is not None: _tag_name = _match.group(1) _logger.debug('Creating tag {0} from branch {1}'.format( _tag_name, _ref.name)) git.refs.tag.TagReference.create( self._repo, _tag_name, ref=_ref.name) _logger.debug('Deleting remote branch {1}'.format( _tag_name, _ref.name)) git.refs.remote.RemoteReference.delete( self._repo, _ref) elif _trunk_re.match(_ref.name): _logger.debug('Deleting trunk branch') git.refs.remote.RemoteReference.delete( self._repo, _ref) - return self diff --git a/getmystuph/backends/repos/svn.py b/getmystuph/backends/repos/svn.py index b873be5..0fabf79 100644 --- a/getmystuph/backends/repos/svn.py +++ b/getmystuph/backends/repos/svn.py @@ -1,127 +1,126 @@ import re -import pysvn +import svn.remote import logging import subprocess from ... import dry_do from ... import colored from ...repo import RepoQuery from ...utils import get_password __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) class RepoSvnSync(RepoQuery): """This class handles the common part on svn repositories, checkout, retreiving tags/branches """ def __enter__(self): super().__enter__() _logger.info('Getting repo {0} [{1}] from svn server'.format( self._in_repo.color_name(self._name), self._url)) - def _get_login(realm, username, may_save): - password = get_password(self._in_repo.backend_name, - self._in_repo.username, - keyring=self._keyring) - return True, self._in_repo.username, password, False + _auth = {} + if (re.match('^https://', self._url)): + _auth['username'] = self._username + _auth['password'] = get_password(self._in_repo.backend_name, + self._in_repo.username, + keyring=self._keyring) - self._svn_client = pysvn.Client() - self._svn_client.callback_get_login = _get_login + self._svn_client = svn.remote.RemoteClient(self._url, **_auth) return self + def clone(self): + #self._svn_client.checktou(self.working_dir) + pass + def _list_path(self, path): - _list = self._svn_client.list( - '{0}/{1}'.format(self._url.rstrip('/'), path), recurse=False) + _list = self._svn_client.list(rel_path=path) - _path_name_re = re.compile(r'^{0}/(.*)'.format(path)) _paths = [] for _file in _list: - _path = _file[0].repos_path.lstrip('/') - _match = _path_name_re.match(_path) - - if _match: - _paths.append(_match.group(1)) + _path = _file.rstrip('/') + _paths.append(_path) return _paths @property def branches(self): return self._list_path('branches') @property def tags(self): _tags = self._list_path('tags') return _tags def push(self): _msg = 'Synchronizing {0} [{1}] with {2} [{3}]'.format( self._in_repo.color_name(self._name), colored(self._in_repo.url, attrs=['bold']), self._out_repo.color_name(self._out_repo._name), colored(self._out_repo.url, attrs=['bold'])) _logger.info(_msg) if self._dry_run: dry_do(_msg) return self._out_repo.join_the_dark_side() _outputs = '.svnsync_{stage}_{repo}.{type}' _outputs_dict = {'stage': 'init', 'repo': self._in_repo._name} with open(_outputs.format(type='out', **_outputs_dict), "w") as out, \ open(_outputs.format(type='err', **_outputs_dict), "w") as err: _svn_sync_init = subprocess.Popen( [ 'svnsync', 'init', '--non-interactive', self._out_repo.url, self._in_repo.url ], stdout=out, stderr=err) _ret_code = _svn_sync_init.wait() if _ret_code != 0: _msg = 'Failed to initialize the svn synchronization of {0}' \ .format(self._in_repo.color_name(self._name)) _logger.error(_msg) raise RuntimeError(_msg) _outputs_dict['stage'] = 'sync' with open(_outputs.format(type='out', **_outputs_dict), "w") as out, \ open(_outputs.format(type='err', **_outputs_dict), "w") as err: _svn_sync = subprocess.Popen( [ 'svnsync', 'sync', '--non-interactive', self._out_repo.url ], stdout=out, stderr=err) _ret_code = _svn_sync.wait() if _ret_code != 0: _msg = 'Failed to synchronize {0}'.format( self._in_repo.color_name(self._name)) _logger.error(_msg) raise RuntimeError(_msg) def __exit__(self, *args, **kwargs): self._out_repo.join_the_good_side() super().__exit__(*args, **kwargs) diff --git a/getmystuph/importers/repo_importer.py b/getmystuph/importers/repo_importer.py index 454cd24..a5126cd 100644 --- a/getmystuph/importers/repo_importer.py +++ b/getmystuph/importers/repo_importer.py @@ -1,223 +1,226 @@ # -*- coding: utf-8 -*- import logging import copy from .. import export from .. import colored from .. import color_code from ..repo import Repo from .importer import Importer __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class RepoImporter(Importer): __default_import_scheme = {'type': 'git', 'permissions': {'scheme': 'import'}} def __init__(self, name, config, **kwargs): super().__init__(name, config, self.__default_import_scheme, **kwargs) _logger.debug( 'Initializing importer for Repo {0}' ' with configuration: {1}'.format(self._colored_name, self._config)) def transfer(self, name): _colored_name = colored(name, color_code['repo'], attrs=['bold']) _logger.info('Locking for repo: {0} ({1})'.format(_colored_name, self._colored_name)) _import_scheme = copy.copy(self._config['import-scheme']) _logger.debug(' --> repo info {0}'.format(colored(self._config, attrs=['bold']))) _config = copy.copy(self._config) _type = _config.pop('type', self.__default_import_scheme['type']) _in_repo = Repo(name=name, keyring=self._keyring, dry_run=self._dry_run, type=_type, **self._backend_in) _type = _import_scheme['type'] if _type == 'same': _type = _in_repo.repo_type _name = _import_scheme.pop('name', name).format( original_name=name) _out_repo = Repo(name=_name, keyring=self._keyring, dry_run=self._dry_run, type=_type, **self._backend_out) _queries = {'git': {'git': 'RepoGit', 'svn': 'RepoGitSvn'}, 'svn': {'svn': 'RepoSvnSync'}} _module = _out_repo.repo_type if _module != _in_repo.repo_type: _module = '_'.join([_module, _in_repo.repo_type]) _query_class = None if _type in _queries and _in_repo.repo_type in _queries[_type]: _class_name = _queries[_type][_in_repo.repo_type] _module = __import__( 'getmystuph.backends.repos.{0}'.format(_module), globals(), locals(), [_class_name], 0) _query_class = getattr(_module, _class_name) else: _msg = 'Cannot import a {0} repo in a {1} repo'.format( _type, _in_repo.repo_type) _logger.error(_msg) raise RuntimeError(_msg) _permissions_scheme = _import_scheme['permissions'] _projects = [] if 'project' in _permissions_scheme: if type(_permissions_scheme['project']) == list: _projects = _permissions_scheme['project'] else: _projects = [_permissions_scheme['project']] _out_repo.create(projects=_projects) _out_perms = Repo.Permissions(_out_repo) if _permissions_scheme['scheme'] == 'import': _in_perms = _in_repo.permissions _logger.debug("Replicating permissions {0}".format(_in_perms)) for _type in {'group', 'user'}: _perms_ug = getattr(_in_perms, '{0}s'.format(_type)) for _entity in _perms_ug: _in_id = _entity['id'] _out_id = getattr( self._user_db, _type)(_in_id, create=True) if _type == 'user' and \ _out_id == self._user_db.directory.whoami: _out_id = '_author_' if _out_id is not None: getattr(_out_perms, 'add_{0}'.format(_type))(_out_id, _entity['perm']) else: _logger.warning( 'No permissions to replicate for repository {0}'.format( _colored_name)) elif _permissions_scheme['scheme'] == 'project': if 'project' in _permissions_scheme: _gid = self._user_db.get_group_unique_id( _permissions_scheme['project']) if _gid is not None: _out_perms.groups = [ {'id': _gid, 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] else: _msg = str( 'The project {0} you specified in the ' + 'permissions of repo {1} does not exists' + ' in {2}').format( colored( _permissions_scheme['project'], color_code['group'], attrs=['bold']), _colored_name, self._user_db.directory.backend_name) _logger.error(_msg) raise RuntimeError(_msg) else: _msg = 'You should specify a project name in the ' + \ 'permissions of repo {0}' .format(_colored_name) + \ ' to be able to use the \'project\' import scheme' _logger.error(_msg) raise RuntimeError(_msg) elif _permissions_scheme['scheme'] == 'user': _out_perms.groups = [ {'id': '_author_', 'perm': Repo.EDIT + Repo.PUSH + Repo.VIEW}] elif _permissions_scheme['scheme'] == 'static': for _perm_type in ['edit', 'view', 'push']: if _perm_type not in _permissions_scheme: _msg = 'You should specify a \'{0}\' in the ' + \ 'permissions of repo {1} to be able to use the ' + \ '\'project\' import scheme'.format(_perm_type, _colored_name) _logger.error(_msg) raise RuntimeError(_msg) _equivalent = {'edit': Repo.EDIT, 'view': Repo.VIEW, 'push': Repo.PUSH} for _perm_type in _equivalent.keys(): if _perm_type in _permissions_scheme: _perm_list = _permissions_scheme[_perm_type] if type(_perm_list) is not list: _perm_list = [_perm_list] if _equivalent[_perm_type] is Repo.VIEW: _out_perms.anonymous = False _out_perms.remove_permission(_equivalent[_perm_type]) for _entity in _perm_list: if _entity == '_author_' or \ _entity == '_users_' or \ _entity == '_public_': _out_perms.add_user(_entity, _equivalent[_perm_type]) else: _id = self._user_db.directory.get_group_unique_id( _entity) if _id is not None: _out_perms.add_group(_id, _equivalent[_perm_type]) else: _logger.error( 'The project {0} was not found in {1}'.format( _entity, self._user_db.directory.backend_name)) if _out_perms is not None: _out_repo.set_permissions(_out_perms) if _query_class is not None: _out_repo.enable() with _query_class(_in_repo, out_repo=_out_repo, dry_run=self._dry_run, keyring=self._keyring, user_db=self._user_db) as _clone: + + _clone.clone() + _branches = _clone.branches _tags = _clone.tags for b in _branches: _logger.debug("Branch: {0}".format( colored(b, attrs=['bold']))) for t in _tags: _logger.debug("Tag: {0}".format( colored(t, attrs=['bold']))) _out_repo.wait_enabled() _clone.push() diff --git a/setup.py b/setup.py index 84b4c29..5952305 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ install_requires = [ 'keyring', 'ldap3', 'bs4', 'phabricator', 'pyyaml', 'gitpython' - 'pysvn' + 'svn' ]