diff --git a/.arcconfig b/.arcconfig index def1b2165..346ed1e50 100644 --- a/.arcconfig +++ b/.arcconfig @@ -1,7 +1,6 @@ { "repository.callsign" : "AKAPRIV", "phabricator.uri" : "https://c4science.ch/", - "load" : [ ".linters/clang-format-linter" ], "history.immutable" : true, "projects": "akantu-developers" } diff --git a/Jenkinsfile b/Jenkinsfile index 109217210..12267cfe1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,145 +1,158 @@ pipeline { parameters {string(defaultValue: '', description: 'api-token', name: 'API_TOKEN') string(defaultValue: '', description: 'buildable phid', name: 'BUILD_TARGET_PHID') string(defaultValue: '', description: 'Commit id', name: 'COMMIT_ID') string(defaultValue: '', description: 'Diff id', name: 'DIFF_ID') } options { disableConcurrentBuilds() } environment { PHABRICATOR_HOST = 'https://c4science.ch/api/' PYTHONPATH = sh returnStdout: true, script: 'echo ${WORKSPACE}/test/ci/script/' BLA_VENDOR = 'OpenBLAS' OMPI_MCA_plm = 'isolated' OMPI_MCA_btl = 'tcp,self' } agent { dockerfile { filename 'Dockerfile' dir 'test/ci' additionalBuildArgs '--tag akantu-environment' } } stages { + stage('Lint') { + steps { + sh 'arc lint --output json | jq . -srcM > lint.json' + } + post { + always { + sh """ + ./test/ci/scripts/hbm send-arc-lint -f lint.json + rm lint.json + """ + } + } + } stage('Configure') { steps { sh """ mkdir -p build cd build cmake -DAKANTU_COHESIVE_ELEMENT:BOOL=TRUE \ -DAKANTU_IMPLICIT:BOOL=TRUE \ -DAKANTU_PARALLEL:BOOL=TRUE \ -DAKANTU_PYTHON_INTERFACE:BOOL=TRUE \ -DAKANTU_TESTS:BOOL=TRUE .. """ } post { failure { deleteDir() } } } stage('Compile') { steps { sh 'make -C build/src || true' } } stage ('Warnings gcc') { steps { warnings(consoleParsers: [[parserName: 'GNU Make + GNU C Compiler (gcc)']]) } } stage('Compile python') { steps { sh 'make -C build/python || true' } } stage('Compile tests') { steps { sh 'make -C build/test || true' } } stage('Tests') { steps { sh ''' rm -rf build/gtest_reports cd build/ #source ./akantu_environement.sh ctest -T test --no-compress-output || true ''' } post { always { script { def TAG = sh returnStdout: true, script: 'head -n 1 < build/Testing/TAG' def TAG_ = TAG.trim() if (fileExists("build/Testing/${TAG}/Test.xml")) { sh "cp build/Testing/${TAG}/Test.xml CTestResults.xml" } } } } } } post { always { step([$class: 'XUnitBuilder', thresholds: [ [$class: 'SkippedThreshold', failureThreshold: '0'], [$class: 'FailedThreshold', failureThreshold: '0']], tools: [ [$class: 'CTestType', pattern: 'CTestResults.xml', skipNoTestFiles: true] ]]) step([$class: 'XUnitBuilder', thresholds: [ [$class: 'SkippedThreshold', failureThreshold: '100'], [$class: 'FailedThreshold', failureThreshold: '0']], tools: [ [$class: 'GoogleTestType', pattern: 'build/gtest_reports/**', skipNoTestFiles: true] ]]) createArtifact("./CTestResults.xml") } success { passed() } failure { emailext( body: '''${SCRIPT, template="groovy-html.template"}''', mimeType: 'text/html', subject: "[Jenkins] ${currentBuild.fullDisplayName} Failed", recipientProviders: [[$class: 'CulpritsRecipientProvider']], to: 'akantu-admins@akantu.ch', replyTo: 'akantu-admins@akantu.ch', attachLog: true, compressLog: false) failed() } } } def failed() { sh "./test/ci/scripts/hbm failed" } def passed() { sh "./test/ci/scripts/hbm passed" } def createArtifact(artefact) { sh "./test/ci/scripts/hbm send-uri -k 'Jenkins URI' -u ${BUILD_URL} -l 'View Jenkins result'" sh "./test/ci/scripts/hbm send-ctest-results -f ${artefact}" } diff --git a/test/ci/scripts/harbomaster/__init__.py b/test/ci/scripts/harbomaster/__init__.py index 9ef5e7842..b88afb730 100644 --- a/test/ci/scripts/harbomaster/__init__.py +++ b/test/ci/scripts/harbomaster/__init__.py @@ -1,25 +1,26 @@ # for the module import sys as __hbm_sys def export(definition): """ Decorator to export definitions from sub-modules to the top-level package :param definition: definition to be exported :return: definition """ __module = __hbm_sys.modules[definition.__module__] __pkg = __hbm_sys.modules[__module.__package__] __pkg.__dict__[definition.__name__] = definition if '__all__' not in __pkg.__dict__: __pkg.__dict__['__all__'] = [] __pkg.__all__.append(definition.__name__) return definition from . import ctestresults +from . import arclint from . import hbm diff --git a/test/ci/scripts/harbomaster/arclint.py b/test/ci/scripts/harbomaster/arclint.py new file mode 100644 index 000000000..5da15b1cb --- /dev/null +++ b/test/ci/scripts/harbomaster/arclint.py @@ -0,0 +1,61 @@ +import json +from .results import Results +from . import export + +@export +class ARCLintJson: + STATUS = {'passed': Results.PASS, + 'failed': Results.FAIL} + + def __init__(self, filename): + self._file = open(filename, "r") + self._json = json.load(self._file) + + def __iter__(self): + self._last_path_lints = [] + self._lints = iter(self._json) + return self + + def __next__(self): + class Lint: + def __init__(self, path, json): + self.json = json + self.json['path'] = path + if 'name' in json and 'code' in json and \ + json['name'] == json['code']: + self.json['name'] = json['description'] + del self.json['description'] + + def __getattr__(self, name): + if name == 'json': + return self.json + elif name in self.json: + return self.json[name] + else: + return None + + def __str__(self): + return f'{self.path} => {self.name}' + + lint = None + while not lint: + if len(self._last_path_lints) > 0: + lint = Lint(self._last_path, + self._last_path_lints.pop(0)) + break + + json = next(self._lints) + if type(json) != dict: + raise RuntimeError("Wrong input type for the linter processor") + + self._last_path = list(json.keys())[0] + self._last_path_lints = json[self._last_path] + + return lint + + + def __exit__(self, exc_type, exc_value, traceback): + self._file.close() + + def __enter__(self): + return self diff --git a/test/ci/scripts/harbomaster/ctestresults.py b/test/ci/scripts/harbomaster/ctestresults.py index a15a7a782..a087924e3 100644 --- a/test/ci/scripts/harbomaster/ctestresults.py +++ b/test/ci/scripts/harbomaster/ctestresults.py @@ -1,51 +1,47 @@ import xml.etree.ElementTree as xml_etree from .results import Results from . import export @export class CTestResults: STATUS = {'passed': Results.PASS, 'failed': Results.FAIL} def __init__(self, filename): self._file = open(filename, "r") self._etree = xml_etree.parse(self._file) self._root = self._etree.getroot() self.test_format = 'CTest' def __iter__(self): self._tests = iter(self._root.findall('./Testing/Test')) return self def __next__(self): class Test: def __init__(self, element): self.name = element.find('Name').text self.path = element.find('FullName').text self.status = CTestResults.STATUS[element.attrib['Status']] self.duration = float(element.find("./Results/NamedMeasurement[@name='Execution Time']/Value").text) self.reason = None if self.status == Results.FAIL: self.reason = element.find("./Results/NamedMeasurement[@name='Exit Code']/Value").text if self.reason == "Timeout": self.status = Results.BROKEN else: - self.reason = "{0} with exit code [{1}]".format( + self.reason = "{0} with exit code [{1}]\nSTDOUT:\n{2}".format( self.reason, - element.find("./Results/NamedMeasurement[@name='Exit Value']/Value").text) + element.find("./Results/NamedMeasurement[@name='Exit Value']/Value").text, + '\n'.join((el.text for el in element.findall("./Results/Measurement/Value"))), + ) - def __str__(self): - return f'{self._name}: {self._status} in {self._duration}' - test = next(self._tests) - #print(test.find('Name')) - #while not test.find('Name'): - # test = next(self._tests) - + return Test(test) def __exit__(self, exc_type, exc_value, traceback): self._file.close() def __enter__(self): return self diff --git a/test/ci/scripts/harbomaster/hbm.py b/test/ci/scripts/harbomaster/hbm.py index f02f8723b..259ac8fd6 100644 --- a/test/ci/scripts/harbomaster/hbm.py +++ b/test/ci/scripts/harbomaster/hbm.py @@ -1,115 +1,134 @@ from phabricator import Phabricator import yaml from . import export from .results import Results def get_phabricator_instance(ctx=None): _phab = None try: _host = None _username = None _token = None if ctx: _host = ctx.pop('HOST', None) _username = ctx.pop('USERNAME', None) _token = ctx.pop('API_TOKEN', None) _phab = Phabricator(host=_host, username=_username, token=_token) _phab.update_interfaces() # this request is just to make an actual connection _phab.user.whoami() except Exception as e: print('Could not connect to phabricator, either give the' + ' connection with the default configuration of arc' + ' or in the backend configuration of the configuration' + ' file:\n' + ' in/out:\n' + ' username: mylogin\n' + ' host: https://c4science.ch/\n' + ' token: cli-g3amff25kdpnnv2tqvigmr4omnn7\n') raise e return _phab @export class Harbormaster: STATUS = {Results.PASS: 'pass', Results.FAIL: 'fail', Results.BROKEN: 'broken', Results.SKIP: 'skip', Results.UNSTABLE: 'unsound'} def __init__(self, **kwargs): ctx = kwargs['ctx'] self.__phid = ctx['BUILD_TARGET_PHID'] self.__phab = get_phabricator_instance(**kwargs) def _send_message(self, results): self.__phab.harbormaster.sendmessage(buildTargetPHID=self.__phid, type=self.STATUS[results]) def send_unit_tests(self, tests): _unit_tests = [] _format = tests.test_format _list_of_failed = {} try: _yaml = open(".tests_previous_state", 'r') _previously_failed = yaml.load(_yaml) if not _previously_failed: _previously_failed = {} _yaml.close() except OSError: _previously_failed = {} for _test in tests: status = self.STATUS[_test.status] if _test.name in _previously_failed and \ (_previously_failed[_test.name] == self.STATUS[_test.status] or \ (_previously_failed[_test.name] == 'unsound' and _test.status != Results.PASS)): status = 'unsound' _test_dict = { 'name': _test.name, 'result': status, 'format': _format } if _test.duration: _test_dict['duration'] = _test.duration if _test.path: _test_dict['path'] = _test.path if _test.reason: - _test_dict['detail'] = _test.reason - _unit_tests.append(_test_dict) + _test_dict['details'] = _test.reason + if status != 'pass': _list_of_failed[_test.name] = status - + _unit_tests.append(_test_dict) + with open(".tests_previous_state", 'w+') as _cache_file: yaml.dump(_list_of_failed, _cache_file, default_flow_style=False) _msg = {'buildTargetPHID': self.__phid, 'type': 'work', 'unit':_unit_tests} self.__phab.harbormaster.sendmessage(**_msg) + def send_lint(self, linter_processor): + _lints = [] + for lint in linter_processor: + _lint = {} + for key in ['code', 'name', 'severity', 'path', 'line', + 'char', 'description']: + val = getattr(lint, key) + if val: + _lint[key] = val + + _lints.append(_lint) + + _msg = {'buildTargetPHID': self.__phid, + 'type': 'work', + 'lint':_lints} + + self.__phab.harbormaster.sendmessage(**_msg) + def send_uri(self, key, uri, name): self.__phab.harbormaster.createartifact(buildTargetPHID=self.__phid, artifactType='uri', artifactKey=name, artifactData={ 'uri': uri, 'name': name, 'ui.external': True }) def passed(self): self._send_message(Results.PASS) def failed(self): self._send_message(Results.FAIL) diff --git a/test/ci/scripts/hbm b/test/ci/scripts/hbm index 4eed91740..22482da8c 100755 --- a/test/ci/scripts/hbm +++ b/test/ci/scripts/hbm @@ -1,48 +1,56 @@ #!/usr/bin/env python3 import click import harbomaster @click.group() @click.option('-a', '--api-token', default=None, envvar='API_TOKEN') @click.option('-h', '--host', default=None, envvar='PHABRICATOR_HOST') @click.option('-b', '--build-target-phid', envvar='BUILD_TARGET_PHID') @click.pass_context def hbm(ctx, api_token, host, build_target_phid): ctx.obj['API_TOKEN'] = api_token ctx.obj['HOST'] = host ctx.obj['BUILD_TARGET_PHID'] = build_target_phid @hbm.command() @click.option('-f', '--filename') @click.pass_context def send_ctest_results(ctx, filename): _hbm = harbomaster.Harbormaster(ctx=ctx.obj) with harbomaster.CTestResults(filename) as tests: _hbm.send_unit_tests(tests) +@hbm.command() +@click.option('-f', '--filename') +@click.pass_context +def send_arc_lint(ctx, filename): + _hbm = harbomaster.Harbormaster(ctx=ctx.obj) + with harbomaster.ARCLintJson(filename) as tests: + _hbm.send_lint(tests) + @hbm.command() @click.option('-k', '--key') @click.option('-u', '--uri') @click.option('-l', '--label') @click.pass_context def send_uri(ctx, key, uri, label): _hbm = harbomaster.Harbormaster(ctx=ctx.obj) _hbm.send_uri(key, uri, label) @hbm.command() @click.pass_context def passed(ctx): _hbm = harbomaster.Harbormaster(ctx=ctx.obj) _hbm.passed() @hbm.command() @click.pass_context def failed(ctx): _hbm = harbomaster.Harbormaster(ctx=ctx.obj) _hbm.failed() if __name__ == '__main__': hbm(obj={})