diff --git a/.arclint b/.arclint new file mode 100644 index 0000000..55b08f7 --- /dev/null +++ b/.arclint @@ -0,0 +1,9 @@ +{ + "linters": { + "python-flake": { + "type": "flake8", + "include": "(\\.py$)", + "exclude": "(__init__\\.py$)" + } + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/.logging.conf b/.logging.conf new file mode 100644 index 0000000..eded55a --- /dev/null +++ b/.logging.conf @@ -0,0 +1,23 @@ +version: 1 +formatters: + simple: + format: '%(name)s - %(levelname)s - %(message)s' +handlers: + console: + class: logging.StreamHandler + level: DEBUG + formatter: simple + stream: ext://sys.stdout +loggers: + getmystuph: + level: DEBUG + handlers: [console] + propagate: yes + getmystuph.epfl: + level: INFO + handlers: [console] + propagate: no + getmystuph.utils: + level: INFO + handlers: [console] + propagate: no diff --git a/example.yaml b/example.yaml new file mode 100644 index 0000000..501bbf5 --- /dev/null +++ b/example.yaml @@ -0,0 +1,54 @@ +use_keyring: true + +phabricator: + username: richart-test + host: https://scitassrv18.epfl.ch/api/ + token: cli-n5dutb2wv26ivcpo66yvb3sbk64g + +global: + backend: epfl + username: richart + groups: + import-scheme: + type: sub-project # or project + project: test_import # only for sub-project type + name: test_{orig_name} +# repositories: +# import-scheme: +# type: same | git | svn +# partial-import: /path +# branches: [] # if not specified all are imported +# tags: [] # if not specified all are imported +# policies: +# if anonymous access view will be public +# type: create-separate-groups # create 3 groups +# type: groups # use existing groups +# type: default # importer only +# type: best-effort # try its best to match +# names: lsms-{repo}-{policy} +# names: +# view: lsms +# push: lsms +# edit: hpc-lsms + +groups: + hpc-lsms: + + lsms-unit: + import-scheme: + name: lsms + + +repositories: + iohelper: + backend: epfl + type: git + import-scheme: + branches: [master] + tags: [] + + test-interface: + type: svn + import-scheme: all + create-groups: + all-in-one: my_project diff --git a/getmystuph/__init__.py b/getmystuph/__init__.py new file mode 100644 index 0000000..ce437ad --- /dev/null +++ b/getmystuph/__init__.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +''' + @package getmystuph + @file __init__.py + @copyright BSD + @author Nicolas Richart + + @section COPYRIGHT + Copyright (©) 2015 EPFL (Ecole Polytechnique Fédérale de Lausanne) + SCITAS - Scientific IT and Application Support + + This file is part of getmystuph +''' +import sys as __gms_sys + +def export(definition): + """ + Decorator to export definitions from sub-modules to the top-level package + + :param definition: definition to be exported + :return: definition + """ + __module = __gms_sys.modules[definition.__module__] + __pkg = __gms_sys.modules[__module.__package__] + __pkg.__dict__[definition.__name__] = definition + + if '__all__' not in __pkg.__dict__: + __pkg.__dict__['__all__'] = [] + + __pkg.__all__.append(definition.__name__) + + return definition + +try: + from termcolor import colored +except ImportError: + # noinspection PyUnusedLocal + def colored(string, *args, **kwargs): + return string +__all__ = ['colored'] + +from . import directory +from . import repo + +from . import directory_backends +from . import repo_backends + +from . import epfl +import logging + +_logger = logging.getLogger(__name__) + +from .utils import _register_backend + +_register_backend( + 'c4science', { + 'git': {'module': 'repo_backends.phabricator', + 'class': 'PhabRepo'}, + 'svn': {'module': 'repo_backends.phabricator', + 'class': 'PhabRepo'}, + 'directory': {'module': 'directory_backends.phabricator', + 'class': 'PhabDirectory'} + }) + +def get_password(service, username, keyring=None): + service = 'getmystuph/' + service + if keyring: + _print_service = colored('{0}@{1}'.format(username, service), + 'blue') + _logger.debug('Try to retrieve password from keyring \'{0}\''.format(_print_service)) # noqa: E501 + + keyring_passwd = keyring.get_password(service, username) + if keyring_passwd is None: + _logger.debug('Password for \'{0}\' not in keyring'.format(_print_service)) # noqa: E501 + + keyring_passwd = getpass.getpass( + "Password for {0}@{1}: ".format(username, service)) + store = ask_question("Do you want to store your password " + + "in the system keyring ?") + if store: + _logger.debug('Adding password for \'{0}\' in keyring'.format(_print_service)) # noqa: E501 + + self._keyring.set_password(service, + username, + keyring_passwd) + else: + _logger.warning('To avoid this message to reappear, ' + + 'remove the \'use_keyring\' from the ' + + 'configuration file.') + else: + _logger.debug('Password for \'{0}\' found in keyring'.format(_print_service)) # noqa: E501 + + return keyring_passwd + else: + getpass_passwd = getpass.getpass( + "Password for {0}@{1}: ".format(username, service)) + return getpass_passwd diff --git a/getmystuph/directory.py b/getmystuph/directory.py new file mode 100644 index 0000000..6295dcc --- /dev/null +++ b/getmystuph/directory.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +import copy +import logging + +from . import export +from .utils import _get_class + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + + +@export +class Directory(object): + def __new__(cls, **kwargs): + """ + Factory constructor depending on the chosen backend + """ + option = copy.copy(kwargs) + backend = option.pop('backend', None) + _class = _get_class('directory', backend) + + return super(Directory, cls).__new__(_class) + + def __init__(self, **kwargs): + pass + + def is_valid_user(self, id): + pass + + def is_valid_group(self, id): + pass + + def get_users_from_group(self, id): + pass + + def get_group_unique_id(self, name): + return '' + + def get_user_unique_id(self, email): + return '' + + def get_group_name(self, id): + return '' + + def get_user_name(self, id): + return '' + + def get_user_email(self, id): + return '' + + def search_users(self, list_emails): + _res = {} + for _mail in list_emails: + _res[_mail] = self.get_user_unique_id(_mail) + return _res + + def create_group(self, name): + raise PermissionError('Groups cannot be created in this directory') + + def set_group_users(self, gid, uids): + raise PermissionError('Groups cannot be modified in this directory') diff --git a/getmystuph/directory_backends/__init__.py b/getmystuph/directory_backends/__init__.py new file mode 100644 index 0000000..f636b14 --- /dev/null +++ b/getmystuph/directory_backends/__init__.py @@ -0,0 +1,2 @@ +from . import ldap +from . import phabricator diff --git a/getmystuph/directory_backends/ldap.py b/getmystuph/directory_backends/ldap.py new file mode 100644 index 0000000..331f943 --- /dev/null +++ b/getmystuph/directory_backends/ldap.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +from .. import export +from .. import Directory +import ldap3 as ldap + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + + +@export +class LDAPDirectory(Directory): + def __init__(self, uri, *args, **kwargs): + self.__ldap_basedn = kwargs.pop('basedn', '') + self.__ldap_scope = kwargs.pop('scope', ldap.SUBTREE) + self.__ldap_user_unique_id = kwargs.pop('uidNumber', 'uidNumber') + self.__ldap_user_gecos = kwargs.pop('gecos', 'gecos') + self.__ldap_user_id = kwargs.pop('uid', 'uid') + self.__ldap_user_email = kwargs.pop('email', 'email') + self.__ldap_user_filter = kwargs.pop('user_filter', '(&(objectClass=posixAccount)({attr}={value}))') # NOQA: ignore=E501 + self.__ldap_user_group_attrs = kwargs.pop('user_group_attrs', 'memberOf') # NOQA: ignore=E501 + + self.__ldap_group_unique_id = kwargs.pop('gidNumber', 'gidNumber') + self.__ldap_group_id = kwargs.pop('gid', 'cn') + self.__ldap_group_filter = kwargs.pop('group_filter', '(&(objectClass=posixGroup)({attr}={value}))') # NOQA: ignore=E501 + self.__ldap_group_member_filter = kwargs.pop('group_member_filter', 'uidNumber') # NOQA: ignore=E501 + self.__ldap_group_user_attrs = kwargs.pop('group_user_attrs', 'memberUid') # NOQA: ignore=E501 + + super(LDAPDirectory, self).__init__(*args, **kwargs) + self.__ldap_uri = uri + + self.__server = ldap.Server(self.__ldap_uri) + self.__ldap = ldap.Connection(self.__server, auto_bind=True) + + def __get_one(self, fltr, attr): + """get the first ldap entry of attribute (attr) for a given + filter (fltr)""" + return self.__get_all(fltr, attr)[0] + + def __get_one_attr(self, fltr, attr): + """get the first ldap entry of attribute (attr) for a given + filter (fltr)""" + _res = self.__get_all(fltr, attr) + if len(_res) != 0: + return _res[0][attr].value + return '' + + def __get_all(self, fltr, attr): + """get all the ldap attributes entries (attr) for a given + filter (fltr)""" + + if type(attr) is not list: + attrs = [attr] + else: + attrs = attr + + _res = self.__ldap.search(search_base=self.__ldap_basedn, + search_scope=self.__ldap_scope, + search_filter=fltr, + attributes=attrs) + + if _res: + return self.__ldap.entries + else: + return [] + + def is_valid_user(self, id): + _res = self.__get_all( + self.__ldap_user_filter.format( + attr=self.__ldap_user_unique_id, + value=id), + self.__ldap_user_unique_id + ) + return len(_res) != 0 + + def is_valid_group(self, id): + _res = self.__get_one( + self.__ldap_user_filter.format( + attr=self.__ldap_group_unique_id, + value=id), + self.__ldap_group_unique_id + ) + return len(_res) != 0 + + def get_users_from_group(self, id): + _users = [] + _members = self.__get_one_attr( + self.__ldap_group_filter.format( + attr=self.__ldap_group_unique_id, + value=id), + self.__ldap_group_user_attrs + ) + + if self.__ldap_group_member_filter != self.__ldap_user_unique_id: + for m in _members: + _filter = \ + self.__ldap_user_filter.format( + attr=self.__ldap_group_member_filter, + value=m) + + _id = self.__get_one_attr( + _filter, + self.__ldap_user_unique_id, + ) + + if _id: + _users.append(_id) + else: + for m in _members: + if self.is_valid_user(m): + _users.append(m) + + return _users + + def get_group_unique_id(self, name): + return self.__get_one_attr( + self.__ldap_group_filter.format( + attr=self.__ldap_group_id, + value=name), + self.__ldap_group_unique_id) + + def get_user_unique_id(self, email): + return self.__get_one_attr( + self.__ldap_user_filter.format( + attr=self.__ldap_user_email, + value=email), + self.__ldap_user_unique_id) + + def get_group_name(self, id): + return self.__get_one_attr( + self.__ldap_group_filter.format( + attr=self.__ldap_group_unique_id, + value=id), + self.__ldap_group_id) + + def get_user_name(self, id): + return self.__get_one_attr( + self.__ldap_user_filter.format( + attr=self.__ldap_user_unique_id, + value=id), + self.__ldap_user_gecos) + + def get_user_email(self, id): + return self.__get_one_attr( + self.__ldap_user_filter.format( + attr=self.__ldap_user_unique_id, + value=id), + self.__ldap_user_email) diff --git a/getmystuph/directory_backends/phabricator.py b/getmystuph/directory_backends/phabricator.py new file mode 100644 index 0000000..4d04ef2 --- /dev/null +++ b/getmystuph/directory_backends/phabricator.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +import logging +from .. import colored +from phabricator import Phabricator + +from .. import export +# from . import colored +from .. import Directory + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + + +@export +class PhabDirectory(Directory): + def __init__(self, *args, **kwargs): + self._phab = kwargs.pop('phabricator', None) + if not self._phab: + self._phab = Phabricator() + + def is_valid_user(self, id): + return self.get_user_name(id) != '' + + def is_valid_group(self, id): + return self.get_group_name(id) != '' + + def get_users_from_group(self, id): + _res = self._phab.project.query(phid=[id]) + if _res['data']: + return _res[0]['data'][id]['members'] + return '' + + def get_group_unique_id(self, name): + _res = self._phab.project.query(names=[name]) + if _res['data']: + return list(_res['data'].keys())[0] + return '' + + def get_user_unique_id(self, email): + _res = self._phab.user.query(emails=[email]) + if _res: + return _res[0]['phid'] + return '' + + def get_group_name(self, gid): + _res = self._phab.project.query(phid=[gid]) + if _res['data']: + return _res[0]['data'][gid]['name'] + return '' + + def get_user_name(self, uid): + _res = self._phab.user.query(phid=[uid]) + if _res: + return _res[0]['realName'] + return '' + + def get_user_email(self, uid): + raise RuntimeError("This information is not accessible") + + def create_group(self, name, members=[]): + _logger.debug('Creating group {0}'.format(colored(name, + 'red', + attrs=['bold']))) + _res = self._phab.project.create(name=name, members=members) + return _res['phid'] + + def set_group_users(self, gid, uids): + _logger.debug('Setting users {0} as members of group {1}' + .format(colored(uids, attrs=['bold']), + colored(gid, 'red', attrs=['bold']))) + transactions = [{"type": "members.set", "value": uids}] + self._phab.project.edit(transactions=transactions, + objectIdentifier=gid) + + def create_subgroup(self, name, pgid, members=None): + _logger.debug('Creating group {0} as a subgroup of {1}' + .format(colored(name, 'red', attrs=['bold']), + colored(pgid, attrs=['bold']))) + transactions = [{"type": "parent", "value": pgid}, + {"type": "name", "value": name}] + if members is not None: + transactions.append({"type": "members.set", "value": members}) + + _res = self._phab.project.edit(transactions=transactions) + return _res['object']['phid'] diff --git a/getmystuph/epfl/__init__.py b/getmystuph/epfl/__init__.py new file mode 100644 index 0000000..4576d29 --- /dev/null +++ b/getmystuph/epfl/__init__.py @@ -0,0 +1,18 @@ +# keep this file +__all__ = [] + +from . import directory +from . import repo + + +from ..utils import _register_backend +_register_backend( + 'epfl', { + 'git': {'module': 'epfl.repo', + 'class': 'RepoGitEPFL'}, + 'svn': {'module': 'epfl.repo', + 'class': 'RepoSvnEPFL'}, + 'directory': {'module': 'epfl.directory', + 'class': 'EPFLDirectory'} + } +) diff --git a/getmystuph/epfl/directory.py b/getmystuph/epfl/directory.py new file mode 100644 index 0000000..461ca8f --- /dev/null +++ b/getmystuph/epfl/directory.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +from .. import export +from ..directory_backends import LDAPDirectory +import ldap3 + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + + +@export +class EPFLDirectory(LDAPDirectory): + + def __init__(self, **kwargs): + args = { + 'basedn': 'o=epfl,c=ch', + 'scope': ldap3.SUBTREE, + 'uidNumber': 'uniqueIdentifier', + 'gecos': 'gecos', + 'uid': 'uid', + 'email': 'mail', + 'user_filter': '(&(objectClass=posixAccount)({attr}={value}))', + 'user_group_attrs': 'memberOf', + 'gidNumber': 'uniqueIdentifier', + 'gid': 'cn', + 'group_filter': '(&(objectClass=groupOfNames)({attr}={value}))', + 'group_member_filter': 'uniqueIdentifier', + 'group_user_attrs': 'memberUniqueId', + } + + super(EPFLDirectory, self).__init__('ldaps://scoldap.epfl.ch', **args) + + def get_switch_aai_id(self, _id): + return '{0}@epfl.ch'.format(_id) diff --git a/getmystuph/epfl/repo.py b/getmystuph/epfl/repo.py new file mode 100644 index 0000000..5df5ab3 --- /dev/null +++ b/getmystuph/epfl/repo.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +from bs4 import BeautifulSoup +import re +import copy +import logging + +from .. import export +from .. import colored +from .. import Repo +from .tequila import TequilaGet + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + + +class RepoEPFL(Repo): + ''' + Description of a repostitory on {svn git}.epfl.ch + ''' + + _LIST_REPOS = '{root}/repository/my.go' + _MANAGE_REPO = '{root}/repository/manage.go?id={id}' + _PERMISSION_URL = '{root}/objectRole/list.go?objectId={id}' + _REPO_REGEX = '/polyrepo/private/repository/manage\.go\?id=([0-9]+)' + + _PERMISSIONS = {'Reader': Repo.READ, + 'Contributor': Repo.WRITE, + 'Administrator': Repo.ADMIN} + + def __init__(self, name, *args, **kwargs): + super(RepoEPFL, self).__init__(name, *args, **kwargs) + + option = copy.copy(kwargs) + repo_id = option.pop('id', None) + tequila_ctx = option.pop('tequila_ctx', None) + + _logger.info( + "Getting a Tequilla context for user \'{0}\'".format( + colored(self._username, 'blue'))) + + self.__tequila_ctx = self._tequila_ctx(tequila_ctx=tequila_ctx, + **kwargs) + + if repo_id is None: + (repo, info) = self.list_repositories( + tequila_ctx=self.__tequila_ctx) + + if name in repo: + self._id = info[name]['id'] + else: + _logger.error('The repo {0} was not found in your' + + ' list of repositories'.format( + self._colored_name)) + raise RuntimeError('The repo {0} was not found in your list ' + + 'of repositories'.format(name)) + else: + self._id = repo_id + + @property + def permissions(self): + '''Get the group and user permissions on the repository''' + + _logger.info('Retrieving list of permissions' + + ' for repositories {0}'.format(self._colored_name)) + _html_resp = self.__tequila_ctx.get( + self._PERMISSION_URL.format(root=self._ROOT_URL, + id=self._id)) + _html_soup = BeautifulSoup(_html_resp.text, 'html.parser') + + _anonymous_perm = _html_soup.find( + 'input', {'id': 'anonymousAccess'}).has_attr('checked') + _logger.debug(' anonymous access: {0}'.format(_anonymous_perm)) + + _permissions = Repo.Permissions(self) + _permissions._anonymous = _anonymous_perm + + _permissions_tmp = { + 'groups': [], + 'users': [], + } + + _group_regex = re.compile('([US][0-9]+)') + _list_soup = _html_soup.find( + 'form', {'name': 'lister'}).find_all('tr') + + for _tr in _list_soup: + _tds = _tr.find_all('td') + if not _tds: + continue + + _perm_txt = _tds[-2].text.strip() + _perm = self._PERMISSIONS[_perm_txt] + _id_td = _tds[-1] + _ug_id = _id_td.text + _is_group = _group_regex.match(_ug_id) + _name = '' + _color = 'blue' + if _is_group: + _perm_type = 'groups' + if _logger.getEffectiveLevel() == logging.DEBUG: + _name = self.directory.get_group_name(_ug_id) + _color = 'green' + else: + _perm_type = 'users' + if _logger.getEffectiveLevel() == logging.DEBUG: + _name = self.directory.get_user_name(_ug_id) + + _permissions_tmp[_perm_type].append({'id': _ug_id, 'perm': _perm}) + + _logger.debug(' {0}: {1} [{2}] -> {3} [{4}]'.format( + _perm_type, + colored(_name, _color), + colored(_ug_id, attrs=['bold']), + _perm_txt.lower(), + _perm + )) + + _permissions._groups = _permissions_tmp['groups'] + _permissions._users = _permissions_tmp['users'] + return _permissions + + @classmethod + def _tequila_ctx(cls, tequila_ctx=None, **kwargs): + if tequila_ctx is None: + return TequilaGet( + cls._LIST_REPOS.format(root=cls._ROOT_URL), **kwargs) + else: + return tequila_ctx + + @classmethod + def list_repositories(cls, tequila_ctx=None, **kwargs): + _logger.info("Retrieving the list of repositories") + + _repos = [] + _extra_info = {} + + _tequila_ctx = cls._tequila_ctx(tequila_ctx=tequila_ctx, **kwargs) + + _html_resp = _tequila_ctx.get( + cls._LIST_REPOS.format(root=cls._ROOT_URL) + ) + + _html_soup = BeautifulSoup(_html_resp.text, 'html.parser') + _list_soup = _html_soup.find('tbody') + + _id_regex = re.compile(cls._REPO_REGEX) + for _link in _list_soup.find_all('a'): + _repo = _link.get_text() + _repos.append(_repo) + + _repo_link = _link.get('href') + _match = _id_regex.match(_repo_link) + if _match: + _id = _match.group(1) + _extra_info[_repo] = {'id': _id} + + _logger.debug(" List of repositories:") + for repo in _repos: + _logger.debug(" {0} -- {1}".format( + Repo.color_name(repo), + colored(_extra_info[repo], attrs=['bold']))) + return (_repos, _extra_info) + + +@export +class RepoGitEPFL(RepoEPFL): + _ROOT_URL = 'https://git.epfl.ch/polyrepo/private' + + def __init__(self, name, *args, **kwargs): + super(RepoGitEPFL, self).__init__(name, *args, **kwargs) + self._url = 'https://{0}@git.epfl.ch/repo/{1}.git'.format( + self._username, name) + + +@export +class RepoSvnEPFL(RepoEPFL): + _ROOT_URL = 'https://svn.epfl.ch/polyrepo/private' + + def __init__(self, name, *args, **kwargs): + super(RepoSvnEPFL, self).__init__(name, *args, **kwargs) + self._url = 'https://{0}@svn.epfl.ch/svn/{1}'.format( + self._username, name) diff --git a/getmystuph/epfl/tequila.py b/getmystuph/epfl/tequila.py new file mode 100644 index 0000000..8102921 --- /dev/null +++ b/getmystuph/epfl/tequila.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +''' + This code is adapted from the conde of Antoine Albertelli, found here: + https://github.com/antoinealb/python-tequila +''' + +import getpass +import requests +from bs4 import BeautifulSoup +import logging +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + +logging.getLogger('requests').setLevel(logging.WARNING) + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support) \n\n" \ + "Copyright (c) 2013, Antoine Albertelli" +__credits__ = ["Antoine Albertelli", "Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + + +class TequilaGet(object): + """Get url that needs tequila authentiication""" + TEQUILA_LOGIN_POST = "https://tequila.epfl.ch/cgi-bin/tequila/login" + __session = None + + class TequilaError(RuntimeError): + """ + Exception thrown in case of Tequila error. + """ + pass + + def __init__(self, url, username, password=None, **kwargs): + """ + Explicitly login into the tequila service, this will create + a new tequila session. + :raise TequilaError: + """ + self.__session = requests.session() + + resp = self.__session.get(url) + if resp.status_code != 200: + raise TequilaGet.TequilaError('Cannot access {0}'.format(url)) + + parsed_url = urlparse.urlsplit(resp.url) + dict_query = urlparse.parse_qs(parsed_url.query) + sesskey = dict_query['requestkey'][0] + + if password is None: + password = getpass.getpass( + 'Tequilla password of user {0}: '.format(username)) + + payload = dict() + payload["requestkey"] = sesskey + payload["username"] = username + payload["password"] = password + + resp = self.__session.post(self.TEQUILA_LOGIN_POST, + verify=True, data=payload) + if resp.status_code != 200: + raise TequilaGet.TequilaError("Tequila didn't return a 200 code") + + soup = BeautifulSoup(resp.text, 'html.parser') + error = soup.find('font', color='red', size='+1') + if error: + # Grab the tequila error if any + raise TequilaGet.TequilaError(error.string) + + def get(self, url): + """Get an url in a authenticated session""" + resp = self.__session.get(url) + if resp.status_code != 200: + raise TequilaGet.TequilaError() + return resp diff --git a/getmystuph/repo.py b/getmystuph/repo.py new file mode 100644 index 0000000..87b3542 --- /dev/null +++ b/getmystuph/repo.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +import copy +import logging +import tempfile + +from . import export +from . import colored +from .utils import _get_class +from . import Directory + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + + +@export +class Repo(object): + '''Interface class to define for your backend''' + + READ = 1 + WRITE = 2 + ADMIN = 4 + + _repo_backends = dict() + + def __new__(cls, *args, **kwargs): + """ + Factory constructor depending on the chosen backend + """ + option = copy.copy(kwargs) + backend = option.pop('backend', None) + repo_type = option.pop('type', 'git') + + _class = _get_class(repo_type, backend) + + return super(Repo, cls).__new__(_class) + + def __init__(self, name, *args, **kwargs): + self._colored_name = self.color_name(name) + self._name = name + option = copy.copy(kwargs) + self._username = option.pop('username', None) + self._type = option.pop('type', None) + + _directory_kwargs = option.pop('directory', {}) + self._directory = Directory(backend=kwargs['backend'], + **_directory_kwargs) + + @classmethod + def color_name(cls, name): + return colored(name, 'red', attrs=['bold']) + + @property + def directory(self): + return self._directory + + class Permissions(object): + def __init__(self, repo): + self._groups = None + self._users = None + self._anonymous = False + self._repo = repo + + @property + def groups(self): + return self._groups + + @property + def users(self): + return self._users + + @property + def anonymous(self): + return self._anonymous + + @property + def all_users(self): + _users = [u['id'] for u in self._users] + _directory = self._repo.directory + for g in self._groups: + _users.extend(_directory.get_users_from_group(g['id'])) + + return set(_users) + + def __repr__(self): + return ''.format( + self._groups, self._users, self._anonymous) + + @property + def permissions(self): + ''' + Returns a dictionary of permissions of the form: + {'groups': [{'id': id, 'perm': perm, ...}, ...], + 'users': [{'id': id, 'perm': perm, ...}, ...], + 'anonymous': True/False} + + perm should be read, write, admin, or None + ''' + return self.Permissions(self) + + def get_query(self): + if self._type == 'git': + from .repo_backends import RepoGit + return RepoGit(self._name, self._url, self._username) + else: + raise RuntimeError( + 'No backend for \'{0}\' implemented yet'.format(self._type)) + + +class RepoQuery(object): + def __init__(self, name, url, username): + self._name = name + self._url = url + self._username = username + + def __enter__(self): + def debug_mktemp(name): + _path = '/tmp/richart/{0}'.format(name) + import os + try: + os.mkdir(_path) + except FileExistsError: + pass + return _path + + self._stage_path = tempfile.TemporaryDirectory( + prefix=self._name + '-') + # self._stage_path = debug_mktemp(self._name) + + _logger.debug('Creating stage folder {0} for repo {1}'.format( + colored(self.working_dir, attrs=['bold']), + Repo.color_name(self._name))) + + def __exit__(self, *arg, **kwargs): + _logger.debug('Cleaning staged folder {0}'.format( + colored(self.working_dir, attrs=['bold']))) + + self._stage_path.cleanup() + + def list_tags(self): + return [] + + def list_branches(self): + return [] + + @property + def working_dir(self): + return self._stage_path.name + # return self._stage_path diff --git a/getmystuph/repo_backends/__init__.py b/getmystuph/repo_backends/__init__.py new file mode 100644 index 0000000..932ffe7 --- /dev/null +++ b/getmystuph/repo_backends/__init__.py @@ -0,0 +1 @@ +from . import git diff --git a/getmystuph/repo_backends/git.py b/getmystuph/repo_backends/git.py new file mode 100644 index 0000000..6b7f046 --- /dev/null +++ b/getmystuph/repo_backends/git.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +import os +import git +import logging + + +from .. import colored +from ..repo import Repo, RepoQuery + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + + +class RepoGit(RepoQuery): + """This class handles the common part on git repositories, cloning, + retreiving tags/branches doing subtrees + """ + + def __enter__(self): + super(RepoGit, self).__enter__() + + _logger.info('Cloning repo {0} [{1}] in {2}'.format( + Repo.color_name(self._name), + self._url, + colored(self.working_dir, attrs=['bold']))) + + if not os.path.isdir(os.path.join(self.working_dir, '.git')): + self._repo = git.Repo.clone_from(self._url, + self.working_dir) + else: + _logger.warning('Repo {0} is already cloned in {1}'.format( + Repo.color_name(self._name), + colored(self.working_dir, attrs=['bold']))) + self._repo = git.Repo(self.working_dir) + + def list_tags(self): + _tags = [] + for ref in self._repo.refs: + if type(ref) == git.refs.tag.TagReference: + _tags.append(ref.name) + return _tags + + def list_branches(self): + _refs = [] + for ref in self._repo.refs: + if type(ref) == git.refs.remote.RemoteReference and\ + ref.name != 'origin/HEAD': + _refs.append(ref.name) + return _refs diff --git a/getmystuph/repo_backends/phabricator.py b/getmystuph/repo_backends/phabricator.py new file mode 100644 index 0000000..df0e609 --- /dev/null +++ b/getmystuph/repo_backends/phabricator.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import logging +from phabricator import Phabricator + +from .. import export +# from . import colored +from .. import Repo + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + + +@export +class PhabRepo(Repo): + def __init__(self, *args, **kwargs): + self._phab = kwargs.pop('phabricator', None) + if not self._phab: + self._phab = Phabricator() diff --git a/getmystuph/utils.py b/getmystuph/utils.py new file mode 100644 index 0000000..7f9fde0 --- /dev/null +++ b/getmystuph/utils.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +import logging + +__author__ = "Nicolas Richart" +__copyright__ = "Copyright (C) 2016, EPFL (Ecole Polytechnique Fédérale " \ + "de Lausanne) - SCITAS (Scientific IT and Application " \ + "Support)" +__credits__ = ["Nicolas Richart"] +__license__ = "BSD" +__version__ = "0.1" +__maintainer__ = "Nicolas Richart" +__email__ = "nicolas.richart@epfl.ch" + +_logger = logging.getLogger(__name__) + +__repo_backends = {} + + +def _register_backend(name, backends): + __repo_backends[name] = backends + + +def _get_class(_type, backend): + if not backend or backend not in __repo_backends: + _logger.error("{0} not a known backend".format(backend)) + raise TypeError("{0} not a known backend".format(backend)) + + if _type not in __repo_backends[backend]: + _logger.error(("{0} is not a known type for the " + + "backend {1}").format(_type, backend)) + raise TypeError(("{0} is not a known type for the " + + "backend {1}").format(_type, backend)) + + module_info = __repo_backends[backend][_type] + + _logger.debug("Importing module {0}".format(module_info['module'])) + module = __import__('getmystuph.' + module_info['module'], + globals(), + locals(), + [module_info['class']], 0) + + _class = getattr(module, module_info['class']) + return _class diff --git a/getmystuph_in_phab.py b/getmystuph_in_phab.py new file mode 100755 index 0000000..e123bca --- /dev/null +++ b/getmystuph_in_phab.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 + +import argparse +import yaml +import keyring +import getpass +import collections +from phabricator import Phabricator + +import getmystuph +from getmystuph import colored + +import logging +import logging.config + +try: + with open('.logging.conf', 'r') as fh: + config = yaml.load(fh) + logging.config.dictConfig(config) +except: + logging.basicConfig(level=logging.ERROR) + +_logger = logging.getLogger('getmystuph').getChild('main') + + +def ask_question(question, possible_answer=None, default='y'): + if possible_answer is None: + possible_answer = {'y': True, 'n': False} + + answers = '/'.join([k if not k == default else k.upper() + for k in possible_answer.keys()]) + answer = None + while answer not in possible_answer.keys(): + answer = input('{0} ({1})? '.format(question, answers)) + answer = answer.lower() + if answer == '': + return possible_answer[default] + + return possible_answer[answer] + + +class Importer(object): + """ + Parses a YAML configuration file: + """ + _config = None + + def __init__(self, args): + self._config = yaml.load(args.config) + _logger.debug('ConfigurationParser: {0}'.format(self._config)) + + if 'phabricator' not in self._config: + # this will use default infos from ~/.arcrc + self._config['phabricator'] = {} + + try: + self._phab = Phabricator(**self._config['phabricator']) + self._phab.update_interfaces() + # this request is just to make an actual connection + self._phab.user.whoami() + except Exception as e: + _logger.error( + 'Could not connect to phabricator, either give the' + + ' connection information in the \'phabricator\' ' + + 'section of the config file:\n' + + ' phabricator:\n' + + ' username: richart\n' + + ' host: https://c4science.ch\n' + + ' token: cli-g3amff25kdpnnv2tqvigmr4omnn7\n') + raise e + + self._keyring = None + if 'use_keyring' in self._config and self._config['use_keyring']: + self._keyring = keyring.get_keyring() + + self._imported_groups = {} + self._cached_users = {} + + def _complete_users(self, directory, users): + _phab_directory = getmystuph.Directory(phabricator=self._phab, + backend='c4science') + _users = [] + for _user in users: + if _user not in self._cached_users: + _user_info = {'id': _user} + _mail = directory.get_user_email(_user) + _user_info['email'] = _mail + _user_info['name'] = directory.get_user_name(_user) + _phid = _phab_directory.get_user_unique_id(_mail) + if _phid: + _user_info['phid'] = _phid + self._cached_users[_user] = _user_info + else: + _user_info = self._cached_users[_user] + _users.append(_user_info) + return _users + + def _import_group(self, name, info): + colored_name = colored(name, 'red', attrs=['bold']) + _logger.info('Locking for group: {0}'.format(colored_name)) + _logger.debug(' --> group info {0}'.format(colored(info, + attrs=['bold']))) + + directory = getmystuph.Directory(**info) + gid = directory.get_group_unique_id(name) + if gid == '': + _logger.error('{0} is not a valid group in the directory {1}' + .format(colored_name, directory)) + raise ValueError('{0} is not a valid group in the directory {1}' + .format(colored_name, directory)) + + _logger.debug(' --> group id {0} -> {1}' + .format(colored_name, + colored(gid, attrs=['bold']))) + + _users = directory.get_users_from_group(gid) + _users = self._complete_users(directory, _users) + + if 'import-scheme' in info: + + def _create_group(name, directory): + _logger.debug('Checking phid for group {0}'.format(name)) + phid = directory.get_group_unique_id(name) + + if phid != '': + _logger.debug('Group {0} -> {1}'.format(name, phid)) + _logger.warning( + '{0} already exists in c4science try to update it' + .format(name)) + return (phid, False) + else: + phid = directory.create_group(name) + return (phid, True) + + _phab_directory = getmystuph.Directory(phabricator=self._phab, + backend='c4science') + + users_phids = [u['phid'] for u in _users if 'phid' in u] + + import_scheme = info['import-scheme'] + phab_name = name + if 'name' in import_scheme: + phab_name = import_scheme['name'].format(orig_name=name) + + if 'type' not in import_scheme or \ + import_scheme['type'] not in ['project', 'sub-project']: + msg = "You should specify a type of " + \ + "import-scheme for group {0}".format(colored_name) + _logger.error(msg) + raise ValueError(msg) + + if import_scheme['type'] == 'project': + phid, newly_created = _create_group( + phab_name, _phab_directory, members=users_phids) + + elif import_scheme['type'] == 'sub-project': + if 'project' not in import_scheme: + _msg = 'To create {0} as a subproject you ' + \ + 'have to specify a parent project' \ + .format(colored_name) + _logger.error(_msg) + raise ValueError(_msg) + project_name = import_scheme['project'] + pphid, newly_created = _create_group(project_name, + _phab_directory) + + phid = _phab_directory.get_group_unique_id(phab_name) + if phid != '': + _logger.warning( + '{0} already exists in c4science try to update it' + .format(phab_name)) + newly_created = False + else: + phid = _phab_directory.create_subgroup( + phab_name, pphid, members=users_phids) + newly_created = True + + if not newly_created: + _phab_directory.set_group_users(phid, users_phids) + + def _import_repository(self, name, info): + _logger.info('Getting repo {0}'.format( + getmystuph.Repo.color_name(name))) + _logger.debug(' --> repo info {0}'.format(colored(info, + attrs=['bold']))) + password = getmystuph.get_password(info['backend'], + info['username'], + keyring=self._keyring) + + repo = getmystuph.Repo(name, password=password, **info) + + _permissions = repo.permissions + _logger.debug("Permissions for repo {0}: {1}".format( + repo.color_name(name), + colored(_permissions, attrs=['bold']))) + _users = self._complete_users(repo.directory, + _permissions.all_users) + for u in _users: + print(u) + + if 'import_scheme' in info and \ + info['import_scheme'] != 'all': + try: + query = repo.get_query() + with query: + print(query.list_branches()) + print(query.list_tags()) + except: + _logger.warning( + "No fine grain operation possible for repo {0}".format( + repo.color_name(name))) + + def __get_full_info(self, info, _type): + if info is None: + info = {} + + if 'global' in self._config: + global_conf = self._config['global'] + if _type in global_conf: + for key, value in global_conf[_type].items(): + if key not in info: + info[key] = value + elif type(value) == dict: + info[key] = dict(value, **info[key]) + + import_always = ['backend', 'username'] + for key in import_always: + if key in global_conf and key not in info: + info[key] = global_conf[key] + + if 'username' not in info: + info['username'] = getpass.getuser() + return info + + def import_all(self): + methods = collections.OrderedDict( + [('groups', '_import_group'), + ('repositories', '_import_repository')]) + for _type in methods.keys(): + if _type in self._config: + all_info = self._config[_type] + if type(all_info) == list: + all_info = {key: {} for key in all_info} + + for name, info in all_info.items(): + info = self.__get_full_info(info, _type) + getattr(self, methods[_type])(name, info) + + +# Set up cli arguments +parser = argparse.ArgumentParser( + description='Import projects into c4science' +) + +parser.add_argument( + '--config', + help='configuration file (YAML)', + type=argparse.FileType(), + required=True +) + + +args = parser.parse_args() + +imp = Importer(args) +imp.import_all() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a12d7fd --- /dev/null +++ b/setup.py @@ -0,0 +1,8 @@ +install_requires = [ + 'keyring', + 'ldap3', + 'bs4', + 'phabricator', + 'yaml', + 'git' +]