diff --git a/getmystuph/backends/repos/git_svn.py b/getmystuph/backends/repos/git_svn.py index a2059ff..d653be3 100644 --- a/getmystuph/backends/repos/git_svn.py +++ b/getmystuph/backends/repos/git_svn.py @@ -1,126 +1,127 @@ # -*- coding: utf-8 -*- import os import re import git import pysvn -from .git import RepoGit from ... import colored from ... import color_code from ...utils import get_password +from .git import RepoGit + __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) class RepoGitSvn(RepoGit): """This class handles the git svn""" def __enter__(self): self._create_stage() _logger.info('Getting repo {0} [{1}] from svn server'.format( self._in_repo.color_name(self._name), self._url)) def _get_login(realm, username, may_save): password = get_password(self._in_repo.backend_name, self._in_repo.username, keyring=self._keyring) return True, self._in_repo.username, password, False _svn_client = pysvn.Client() _svn_client.callback_get_login = _get_login _svn_logs = _svn_client.log(self._url) _svn_authors = set() for l in _svn_logs: _svn_authors.add(l['author']) _authors = {} for _a in _svn_authors: _user = self._user_db.user_by_in_login(_a) if _user is not None: _authors[_a] = _user _logger.debug( 'Matching author {0} with "{name} <{email}>"'.format( colored(_a, color_code['user'], attrs=['bold']), **_user)) else: _authors[_a] = {'name': _a, 'email': '{0}@{1}'.format( _a, self._in_repo.backend_name)} _svn_authors_file = os.path.join(self.working_dir, 'getmystuph.authors') with open(_svn_authors_file, 'w') as afh: afh.write( '\n'.join( ['{0} = {name} <{email}>'.format(l, **a) for l, a in _authors.items()])) _list = _svn_client.list(self._url, recurse=False) _branches = False _trunk = False _tags = False for _file in _list: _path = _file[0].repos_path.lstrip('/') if re.match(r'branches$', _path): _branches = True if re.match(r'trunk$', _path): _trunk = True if re.match(r'tags$', _path): _tags = True _stdlayout = False if _branches and _tags and _trunk: _logger.debug('Detected svn standard layout for {0}'.format( self._in_repo.color_name(self._name))) _stdlayout = True _logger.info('Cloning repo {0} [{1}] in {2}'.format( self._in_repo.color_name(self._name), self._url, colored(self.working_dir, attrs=['bold']))) if not os.path.isdir(os.path.join(self.working_dir, '.git')): _git = git.Git(self.working_dir) _git.svn('clone', self._url, self.working_dir, preserve_empty_dirs=True, authors_file=_svn_authors_file, stdlayout=_stdlayout) super().__enter__() _tag_re = re.compile('origin/tags/(.*)') _trunk_re = re.compile('origin/trunk') for _ref in self._repo.refs: if type(_ref) == git.refs.remote.RemoteReference: _match = _tag_re.match(_ref.name) if _match is not None: _tag_name = _match.group(1) _logger.debug('Creating tag {0} from branch {1}'.format( _tag_name, _ref.name)) git.refs.tag.TagReference.create( self._repo, _tag_name, ref=_ref.name) _logger.debug('Deleting remote branch {1}'.format( _tag_name, _ref.name)) git.refs.remote.RemoteReference.delete( self._repo, _ref) elif _trunk_re.match(_ref.name): _logger.debug('Deleting trunk branch') git.refs.remote.RemoteReference.delete( self._repo, _ref) return self diff --git a/getmystuph/backends/repos/phabricator.py b/getmystuph/backends/repos/phabricator.py index 958e622..43e354a 100644 --- a/getmystuph/backends/repos/phabricator.py +++ b/getmystuph/backends/repos/phabricator.py @@ -1,245 +1,265 @@ # -*- coding: utf-8 -*- import logging import copy import re import time from ... import export from ... import dry_do from ...repo import Repo from ...utils import get_phabricator_instance from .. import color_phid __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class PhabRepo(Repo): def __init__(self, *args, host=None, username=None, token=None, **kwargs): super().__init__(username=username, **kwargs) options = copy.copy(kwargs) self._repo_type = options.pop('repo_type', 'git') # _create = options.pop('create', False) self._host = '{0}/api/'.format(host.rstrip('/')) self._server = None _server_re = re.compile(r'https?://(.*)') _match = _server_re.match(host.rstrip('/')) if _match is not None: self._server = _match.group(1) if self._server is None: raise RuntimeError( 'Cannot extract the server name for repo {0} from {1}'.format( self._colored_name, host)) self._phab = get_phabricator_instance(host=self._host, username=username, token=token) _data = self._phab.diffusion.repository.search( queryKey="all", constraints={"name": self._name, "types": [self.repo_type]})['data'] self._phab_id = None for _repo in _data: _repo_name = _repo['fields']['name'] if _repo_name == self._name: self._phab_id = _repo['phid'] self._id = _repo['id'] _logger.debug('Repositories {0} has id {1}'.format( self._colored_name, self._phab_id)) if self._phab_id is None: _logger.debug('Repositories {0} not in phabricator'.format( self._colored_name)) def create(self): if self._phab_id is not None: _msg = 'The repository {0}:{1} already exists'.format( self._colored_name, self._phab_id) _logger.error(_msg) return #raise RuntimeError(_msg) if self._dry_run: self._phab_id = "PHID-REPO-notarealrepo" else: _data = self._phab.diffusion.repository.edit( transactions=[{'type': 'name', 'value': self._name}, {'type': 'vcs', 'value': self.repo_type}]) self._creation_data = _data['object'] self._phab_id = self._creation_data['phid'] _msg = 'Created repository {0} id {1} [{2}]'.format( self._colored_name, color_phid(self._phab_id), self.repo_type) _logger.info(_msg) if self._dry_run: dry_do(_msg) self._id = 666 else: self._id = self._creation_data['id'] @property def url(self): if self.repo_type == 'git': self._url = 'git@{0}:/diffusion/{1}/{2}.git'.format( self._server, self._id, self._name) elif self.repo_type == 'svn': self._url = 'svn+ssh://git@{0}/diffusion/{1}/'.format( self._server, self._id) return self._url def enable(self): _msg = 'Activating repository {0} [{1}]'.format( self._colored_name, color_phid(self._phab_id)) _logger.info(_msg) if self._dry_run: dry_do(_msg) else: self._phab.diffusion.repository.edit( transactions=[{'type': 'status', 'value': 'active'}], objectIdentifier=self._phab_id) + def _dangerous_changes(self, state): + _msg = '{0}ctivating dangerous changes on repository {0} [{1}]'.format( + 'A' if state else 'De-a', + self._colored_name, color_phid(self._phab_id)) + _logger.info(_msg) + if self._dry_run: + dry_do(_msg) + return + + self._phab.diffusion.repository.edit( + transactions=[{'type': 'allowDangerousChanges', + 'value': state}], + objectIdentifier=self._phab_id) + + def join_the_dark_side(self): + self._dangerous_changes(True) + + def join_the_good_side(self): + self._dangerous_changes(False) + def wait_enabled(self, timeout=3600): _msg = 'Checking if {0} [{1}] is activated'.format( self._colored_name, color_phid(self._phab_id)) _logger.info(_msg) if self._dry_run: dry_do(_msg) else: _time = 0 while True: _data = self._phab.diffusion.repository.search( queryKey="all", constraints={"phids": [self._phab_id]})['data'] if not len(_data) == 1: raise RuntimeError('Cannot find the repo {0}'.format( self._colored_name)) _status = _data[0]['fields']['status'] _importing = _data[0]['fields']['isImporting'] if _status == 'active' and not _importing: return if _time > timeout: _logger.debug('Timeout reached before repo activation') raise RuntimeError('Timeout reached before repo activation') time.sleep(1) _time += 1 def set_permissions(self, permissions): _perms = {'edit': [], 'push': [], 'view': []} _equivalent = {'edit': Repo.EDIT, 'view': Repo.VIEW, 'push': Repo.PUSH} _phab_perms = {'edit': 'edit', 'view': 'view', 'push': 'policy.push'} _special_perms = {'_author_': 'obj.repository.author', '_users_': 'users', '_public_': 'public'} for _type in {'group', 'user'}: _perms_ug = getattr(permissions, '{0}s'.format(_type)) for _entity in _perms_ug: _id = _entity['id'] if _id in _special_perms: _id = _special_perms[_id] for _phab, _gen in _equivalent.items(): if _entity['perm'] & _gen: _perms[_phab].append(_id) if permissions.anonymous: _perms['view'] = ['public'] for _type in ['push', 'view', 'edit']: if _type not in _perms: continue _perms[_type] = list(set(_perms[_type])) if len(_perms[_type]) > 1: if 'public' in _perms[_type]: _perms[_type] = 'public' continue if 'users' in _perms[_type]: _perms[_type] = 'users' continue # create custom policy to replace the list regex = re.compile(r'PHID-([A-Z]{4})-.+') _lists = {'PROJ': [], 'USER': []} for _p in _perms[_type]: match = regex.match(_p) if match is not None: _lists[match.group(1)].append(_p) elif _p == 'obj.repository.author': _lists['USER'].append(self._directory.whoami) _policy = [ {"action": "allow", "rule": 'PhabricatorUsersPolicyRule', "value": _lists['USER']}, {"action": "allow", "rule": 'PhabricatorProjectsPolicyRule', "value": _lists['PROJ']}] _msg = 'Creating policy for users [{0}] and projects [{1}]'.format( ', '.join([color_phid(_id) for _id in _lists['USER']]), ', '.join([color_phid(_id) for _id in _lists['PROJ']])) _logger.debug(_msg) if self._dry_run: dry_do(_msg) _phid = 'PHID-PLCY-notapolicy' else: _data = self._phab.policy.create(objectType='REPO', default='deny', policy=_policy) _phid = _data['phid'] _logger.info('Replacing list {0} by policy {1}'.format( _perms[_type], _phid)) _perms[_type] = _phid else: _perms[_type] = _perms[_type][0] _msg = 'Setting \'{0}\' permissions for {1} to {2}:'.format( _type, self._colored_name, color_phid(_perms[_type])) _logger.info(_msg) if not self._dry_run: self._phab.diffusion.repository.edit( transactions=[{'type': _phab_perms[_type], 'value': _perms[_type]}], objectIdentifier=self._phab_id) else: dry_do(_msg) diff --git a/getmystuph/backends/repos/svn.py b/getmystuph/backends/repos/svn.py index 34051c0..443a9a3 100644 --- a/getmystuph/backends/repos/svn.py +++ b/getmystuph/backends/repos/svn.py @@ -1,31 +1,123 @@ -# -*- coding: utf-8 -*- -import os import re -import svn +import pysvn import logging +import subprocess -from ... import colored -from ... import color_code from ... import dry_do +from ... import colored from ...repo import RepoQuery +from ...utils import get_password __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) -class RepoSVNSync(RepoQuery): - """This class handles the common part on git repositories, cloning, - retreiving tags/branches doing subtrees +class RepoSvnSync(RepoQuery): + """This class handles the common part on svn repositories, checkout, + retreiving tags/branches """ def __enter__(self): + super().__enter__() + + _logger.info('Getting repo {0} [{1}] from svn server'.format( + self._in_repo.color_name(self._name), + self._url)) + + def _get_login(realm, username, may_save): + password = get_password(self._in_repo.backend_name, + self._in_repo.username, + keyring=self._keyring) + return True, self._in_repo.username, password, False + + self._svn_client = pysvn.Client() + self._svn_client.callback_get_login = _get_login + + return self + + def _list_path(self, path): + _list = self._svn_client.list( + '{0}/{1}'.format(self._url.rstrip('/'), path), recurse=False) + + _path_name_re = re.compile(r'^{0}/(.*)'.format(path)) + _paths = [] + for _file in _list: + _path = _file[0].repos_path.lstrip('/') + _match = _path_name_re.match(_path) + + if _match: + _paths.append(_match.group(1)) + + return _paths + + @property + def branches(self): + return self._list_path('branches') + + @property + def tags(self): + _tags = self._list_path('tags') + return _tags + + def push(self): + _msg = 'Synchronizing {0} [{1}] with {2} [{3}]'.format( + self._in_repo.color_name(self._name), + colored(self._in_repo.url, attrs=['bold']), + self._out_repo.color_name(self._out_repo._name), + colored(self._out_repo.url, attrs=['bold'])) + + _logger.info(_msg) + + if self._dry_run: + dry_do(_msg) + return + + self._out_repo.join_the_dark_side() + + _outputs = '.svnsync_{stage}_{repo}.{type}' + _outputs_dict = {'stage': 'init', + 'repo': self._in_repo._name} + with open(_outputs.format(type='out', **_outputs_dict),"w") as out, \ + open(_outputs.format(type='err', **_outputs_dict),"w") as err: + _svn_sync_init = subprocess.Popen(['svnsync', 'init', + '--non-interactive', + self._out_repo.url, + self._in_repo.url], + stdout=out, + stderr=err) + + _ret_code = _svn_sync_init.wait() + if _ret_code != 0: + _msg = 'Failed to initialize the svn synchronization of {0}'.format( + self._in_repo.color_name(self._name)) + _logger.error(_msg) + raise RuntimeError(_msg) + + _outputs_dict['stage'] = 'sync' + with open(_outputs.format(type='out', **_outputs_dict),"w") as out, \ + open(_outputs.format(type='err', **_outputs_dict),"w") as err: + _svn_sync = subprocess.Popen(['svnsync', 'sync', + '--non-interactive', + self._out_repo.url], + stdout=out, + stderr=err) + + _ret_code = _svn_sync.wait() + if _ret_code != 0: + _msg = 'Failed to synchronize {0}'.format( + self._in_repo.color_name(self._name)) + _logger.error(_msg) + raise RuntimeError(_msg) + def __exit__(self, *args, **kwargs): + self._out_repo.join_the_good_side() + super().__exit__(*args, **kwargs) diff --git a/getmystuph/repo.py b/getmystuph/repo.py index a2870ad..62a1c55 100644 --- a/getmystuph/repo.py +++ b/getmystuph/repo.py @@ -1,244 +1,252 @@ # -*- coding: utf-8 -*- import copy import logging import os import tempfile from . import export from . import colored from . import color_code from .backends import _get_class from .directory import Directory __author__ = "Nicolas Richart" __copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ "de Lausanne) - SCITAS (Scientific IT and Application " \ "Support)" __credits__ = ["Nicolas Richart"] __license__ = "BSD" __version__ = "0.1" __maintainer__ = "Nicolas Richart" __email__ = "nicolas.richart@epfl.ch" _logger = logging.getLogger(__name__) @export class Repo(object): '''Interface class to define for your backend''' VIEW = 0x1 PUSH = 0x2 EDIT = 0x4 _repo_backends = dict() def __new__(cls, *args, **kwargs): """ Factory constructor depending on the chosen backend """ option = copy.copy(kwargs) backend = option.pop('backend', None) repo_type = option.pop('type', 'git') _class = _get_class(repo_type, backend) return super(Repo, cls).__new__(_class) def __init__(self, name, *args, **kwargs): self._name = name self._colored_name = self.color_name(name) options = copy.copy(kwargs) self._username = options.pop('username', None) self._type = options.pop('type', None) self._dry_run = options.pop("dry_run", False) self._backend_name = options.pop("backend", None) self._directory = options.pop('directory', None) if self._directory is None: self._directory = Directory(type='directory', backend=self._backend_name, username=self._username, dry_run=self._dry_run, **options) def enable(self): pass @property def backend_name(self): return self._backend_name @property def repo_type(self): return self._type def color_name(self, name): return colored(name, color_code['repo'], attrs=['bold']) @property def directory(self): return self._directory class Permissions(object): def __init__(self, repo): self.__groups = [] self.__users = [] self.__anonymous = False self.__repo = repo @property def groups(self): return self.__groups @groups.setter def groups(self, groups_perms): self.__groups = copy.copy(groups_perms) def add_group(self, group_id, perm): self.__groups.append({'id': group_id, 'perm': perm}) def add_user(self, user_id, perm): self.__users.append({'id': user_id, 'perm': perm}) def remove_permission(self, perm): _lists = {'group': self.__groups, 'user': self.__users} for _type in _lists.keys(): for _entity in _lists[_type]: _entity['perm'] = \ _entity['perm'] ^ (_entity['perm'] & perm) def user_perm(self, _id): for _user in self.__users: if _user['id'] == _id: return _user['perm'] return 0 @property def users(self): return self.__users @users.setter def users(self, users_perms): self.__users = copy.copy(users_perms) @property def anonymous(self): return self.__anonymous @anonymous.setter def anonymous(self, anonymous): self.__anonymous = anonymous @property def all_users(self): _users = [u['id'] for u in self.__users] _directory = self.__repo.directory for g in self._groups: _users.extend(_directory.get_users_from_group(g['id'])) return set(_users) def __repr__(self): return ''.format( self.__groups, self.__users, self.__anonymous) def set_permissions(self, permissions): pass @property def permissions(self): ''' Returns a dictionary of permissions of the form: {'groups': [{'id': id, 'perm': perm, ...}, ...], 'users': [{'id': id, 'perm': perm, ...}, ...], 'anonymous': True/False} perm should be read, write, admin, or None ''' return self.Permissions(self) @property def name(self): return self._name @property def url(self): return self._url @property def username(self): return self._username def get_query(self): if self._type == 'git': from .repo_backends import RepoGit return RepoGit(self) else: raise RuntimeError( 'No backend for \'{0}\' implemented yet'.format(self._type)) class RepoQuery(object): class debug_mktemp: def __init__(self, name): self._path = os.path.join(os.environ['TMPDIR'], 'getmystuph', name) try: os.makedirs(self._path) except FileExistsError: pass @property def name(self): return self._path def cleanup(self): pass def __init__(self, repo, **kwargs): self._in_repo = repo self._name = repo.name self._url = repo.url self._username = repo.username self._dry_run = kwargs.pop('dry_run', False) self._keyring = kwargs.pop('keyring', None) self._out_repo = kwargs.pop('out_repo', None) self._user_db = kwargs.pop('user_db', None) self._stage_path = None def __enter__(self): if self._stage_path is None: self._create_stage() return self def _create_stage(self): self._stage_path = tempfile.TemporaryDirectory( prefix=self._name + '-') #self._stage_path = RepoQuery.debug_mktemp(self._name) _logger.debug('Creating stage folder {0} for repo {1}'.format( colored(self.working_dir, attrs=['bold']), self._in_repo.color_name(self._name))) def __exit__(self, *arg, **kwargs): _logger.debug('Cleaning staged folder {0}'.format( colored(self.working_dir, attrs=['bold']))) self._stage_path.cleanup() - def add_remote(self, out_repo): + def join_the_dark_side(self): + '''Allow nasty changes if required''' + pass + + def join_the_good_side(self): + '''Disallow nasty changes if required''' pass @property def tags(self): return [] @property def branches(self): return [] @property def working_dir(self): return self._stage_path.name + + def push(self): + pass