diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40c6641c9..f49003d93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,445 +1,445 @@ stages: - configure - build - test - deploy .docker_build: image: 'docker:19.03.11' stage: .pre services: - docker:19.03.11-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled DOCKER_HOST: tcp://docker:2376 DOCKER_TLS_CERTDIR: "/certs" before_script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - cd test/ci/${IMAGE_NAME}/ - docker build -t registry.gitlab.com/akantu/akantu/${IMAGE_NAME} . - docker push registry.gitlab.com/akantu/akantu/${IMAGE_NAME} docker build:debian-testing: variables: IMAGE_NAME: debian:testing extends: .docker_build rules: - changes: - test/ci/debian:testing/Dockerfile docker build:ubuntu-lts: variables: IMAGE_NAME: ubuntu:lts extends: .docker_build rules: - changes: - test/ci/ubuntu:lts/Dockerfile .configure: stage: configure except: - tags variables: BLA_VENDOR: 'Generic' script: - cmake -E make_directory build - cd build - cmake -DAKANTU_COHESIVE_ELEMENT:BOOL=TRUE -DAKANTU_IMPLICIT:BOOL=TRUE -DAKANTU_PARALLEL:BOOL=TRUE -DAKANTU_STRUCTURAL_MECHANICS:BOOL=TRUE -DAKANTU_HEAT_TRANSFER:BOOL=TRUE -DAKANTU_DAMAGE_NON_LOCAL:BOOL=TRUE -DAKANTU_PHASE_FIELD:BOOL=TRUE -DAKANTU_PYTHON_INTERFACE:BOOL=TRUE -DAKANTU_CONTACT_MECHANICS:BOOL=TRUE -DAKANTU_EXAMPLES:BOOL=TRUE -DAKANTU_BUILD_ALL_EXAMPLES:BOOL=TRUE -DAKANTU_TESTS:BOOL=TRUE -DAKANTU_RUN_IN_DOCKER:BOOL=TRUE -DAKANTU_TEST_EXAMPLES:BOOL=${TEST_EXAMPLES} -DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE} .. - cp compile_commmands.json .. artifacts: when: on_success paths: - build - compile_commands.json expire_in: 10h .build: stage: build script: - cmake --build build/src > >(tee -a build-${output}-out.log) 2> >(tee -a build-${output}-err.log >&2) - cmake --build build/python > >(tee -a build-${output}-out.log) 2> >(tee -a build-${output}-err.log >&2) - cmake --build build/test/ > >(tee -a build-${output}-out.log) 2> >(tee -a build-${output}-err.log >&2) - cmake --build build/examples > >(tee -a build-${output}-out.log) 2> >(tee -a build-${output}-err.log >&2) artifacts: when: on_success paths: - build/ - build-${output}-err.log - compile_commands.json exclude: - build/**/*.o expire_in: 10h .tests: stage: test script: - cd build - ctest -T test --output-on-failure --no-compress-output --timeout 1800 after_script: - cd build - tag=$(head -n 1 < Testing/TAG) - if [ -e Testing/${tag}/Test.xml ]; then - xsltproc -o ./juint.xml ${CI_PROJECT_DIR}/test/ci/ctest2junit.xsl Testing/${tag}/Test.xml; - fi - if [ ${CMAKE_BUILD_TYPE} = "Coverage" ]; then - gcovr --xml --gcov-executable "${GCOV_EXECUTABLE}" --output coverage.xml --object-directory ${CI_PROJECT_DIR}/build --root ${CI_PROJECT_DIR} -s || true - fi artifacts: when: always paths: - build/juint.xml - build/coverage.xml reports: junit: - build/juint.xml cobertura: - build/coverage.xml # ------------------------------------------------------------------------------ .cache_build: variables: CCACHE_BASEDIR: ${CI_PROJECT_DIR}/ CCACHE_DIR: ${CI_PROJECT_DIR}/.ccache #CCACHE_NOHASHDIR: 1 #CCACHE_COMPILERCHECK: content cache: key: ${output}_${BUILD_TYPE} policy: pull-push paths: - .ccache/ - third-party/google-test - third-party/pybind11 before_script: - ccache --zero-stats || true after_script: - ccache --show-stats || true # ------------------------------------------------------------------------------ .image_debian_testing: image: registry.gitlab.com/akantu/akantu/debian:testing .image_ubuntu_lts: image: registry.gitlab.com/akantu/akantu/ubuntu:lts # ------------------------------------------------------------------------------ .compiler_gcc: variables: CC: /usr/lib/ccache/gcc CXX: /usr/lib/ccache/g++ FC: gfortran GCOV_EXECUTABLE: gcov .compiler_clang: variables: CC: /usr/lib/ccache/clang CXX: /usr/lib/ccache/clang++ FC: gfortran GCOV_EXECUTABLE: llvm-cov gcov .build_coverage: variables: TEST_EXAMPLES: 'FALSE' BUILD_TYPE: 'Coverage' .build_release: variables: TEST_EXAMPLES: 'TRUE' BUILD_TYPE: 'Release' .build_valgrind: variables: TEST_EXAMPLES: 'FALSE' BUILD_TYPE: 'Valgrind' # ------------------------------------------------------------------------------ .debian_testing_gcc: variables: output: debian_testing_gcc extends: - .compiler_gcc - .image_debian_testing - .cache_build .debian_testing_clang: variables: output: debian_testing_clang extends: - .compiler_clang - .image_debian_testing - .cache_build .ubuntu_lts_gcc: variables: output: ubuntu_lts_gcc extends: - .compiler_gcc - .image_ubuntu_lts - .cache_build # ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------ configure:debian_testing_gcc: extends: - .debian_testing_gcc - .build_coverage - .configure build:debian_testing_gcc: extends: - .debian_testing_gcc - .build_coverage - .build needs: - job: configure:debian_testing_gcc test:debian_testing_gcc: extends: - .debian_testing_gcc - .build_coverage - .tests needs: - job: build:debian_testing_gcc # ------------------------------------------------------------------------------ configure:debian_testing_gcc_valgrind: extends: - .debian_testing_gcc - .build_valgrind - .configure build:debian_testing_gcc_valgrind: extends: - .debian_testing_gcc - .build_valgrind - .build needs: - job: configure:debian_testing_gcc_valgrind test:debian_testing_gcc_valgrind: extends: - .debian_testing_gcc - .build_valgrind - .tests needs: - job: build:debian_testing_gcc_valgrind # ------------------------------------------------------------------------------ configure:debian_testing_clang: extends: - .debian_testing_clang - .build_coverage - .configure build:debian_testing_clang: extends: - .debian_testing_clang - .build_coverage - .build needs: - job: configure:debian_testing_clang test:debian_testing_clang: extends: - .debian_testing_clang - .build_coverage - .tests needs: - job: build:debian_testing_clang # ------------------------------------------------------------------------------ configure:ubuntu_lts_gcc: extends: - .ubuntu_lts_gcc - .build_release - .configure build:ubuntu_lts_gcc: extends: - .ubuntu_lts_gcc - .build_release - .build needs: - job: configure:ubuntu_lts_gcc test:ubuntu_lts_gcc: extends: - .ubuntu_lts_gcc - .build_release - .tests needs: - job: build:ubuntu_lts_gcc # ------------------------------------------------------------------------------ cq:code_quality: stage: test image: docker:19.03.12 allow_failure: true services: - docker:19.03.12-dind variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24" needs: [] script: - export SOURCE_CODE=$PWD - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage function propagate_env_vars() { CURRENT_ENV=$(printenv) for VAR_NAME; do echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " done } - docker pull --quiet "$CODE_QUALITY_IMAGE" - | docker run --rm \ $(propagate_env_vars \ SOURCE_CODE \ TIMEOUT_SECONDS \ CODECLIMATE_DEBUG \ CODECLIMATE_DEV \ REPORT_STDOUT \ REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ CODECLIMATE_PREFIX \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ "$CODE_QUALITY_IMAGE" /code artifacts: paths: - gl-code-quality-report.json expire_in: 1 week needs: - job: build:debian_testing_clang rules: - if: '$CODE_QUALITY_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' .clang_tools: stage: test extends: - .debian_testing_clang before_script: - if [ 'x${CI_MERGE_REQUEST_ID}' != 'x' ]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - git diff --name-only $CI_COMMIT_SHA $CI_MERGE_REQUEST_TARGET_BRANCH_NAME > file_list - FILE_LIST_ARG='-f file_list' - fi needs: - job: build:debian_testing_clang allow_failure: true rules: - if: '$CODE_QUALITY_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' cq:clang_tidy: extends: - .clang_tools script: - test/ci/scripts/cq -x third-party -x extra-packages + -x pybind11 + -x test ${FILE_LIST_ARG} clang-tidy -p ${CI_PROJECT_DIR}/build > gl-clang-tidy-report.json artifacts: paths: - gl-clang-tidy-report.json cq:clang_format: extends: - .clang_tools script: - test/ci/scripts/cq -x third-party -x extra-packages clang-format -p ${CI_PROJECT_DIR}/build > gl-clang-format-report.json artifacts: paths: - gl-clang-format-report.json cq:compilation_warnings: stage: test image: python:latest script: - pip install warning-parser termcolor Click - ls build-*-err.log - test/ci/scripts/cq -x third-party -x extra-packages - -x pybind11 - -x test warnings build-*-err.log > gl-warnings-report.json needs: - job: build:debian_testing_clang - job: build:debian_testing_gcc - job: build:ubuntu_lts_gcc artifacts: paths: - gl-warnings-report.json cq:merge_code_quality: stage: deploy extends: - .debian_testing_clang script: - jq -Ms '[.[][]]' gl-*-report.json | tee gl-codequality.json | jq -C needs: - job: cq:code_quality - job: cq:clang_tidy - job: cq:clang_format - job: cq:compilation_warnings artifacts: paths: - gl-codequality.json artifacts: reports: codequality: [gl-codequality.json] # ------------------------------------------------------------------------------ pages: stage: deploy extends: - .debian_testing_gcc script: - cd build - cmake -DAKANTU_DOCUMENTATION=ON .. - cmake --build . -t sphinx-doc - mv doc/dev-doc/html ../public needs: - job: build:debian_testing_gcc artifacts: paths: - public only: - master diff --git a/test/ci/scripts/codequality/issue_generator.py b/test/ci/scripts/codequality/issue_generator.py index 3835fa10d..95efb400f 100644 --- a/test/ci/scripts/codequality/issue_generator.py +++ b/test/ci/scripts/codequality/issue_generator.py @@ -1,122 +1,123 @@ #!/usr/bin/env python3 from . import print_debug, print_info import hashlib import os import re import copy class IssueGenerator: """Interface for the issue generators""" def __init__(self, **kwargs): self._files = kwargs.pop('file_list', []) excludes = kwargs.pop('excludes', None) if excludes is None: excludes = [] extensions = kwargs.pop('extensions', None) if extensions is None: extensions = ['.cc', '.hh'] self._extensions = [ re.compile(r"\{}$".format(extension)) for extension in extensions ] self._exclude_patterns = [ re.compile(exclude) for exclude in excludes ] self._issues = {} self._filter_file_list() def _filter_file_list(self): file_list = copy.copy(self._files) self._files = [] for filename in file_list: filename = os.path.relpath(filename) need_exclude = self._need_exclude(filename) if need_exclude: print_debug(f'exluding file: {filename}') continue print_info(f'adding file: {filename}') self._files.append(filename) def _need_exclude(self, filename): need_exclude = False for pattern in self._exclude_patterns: match = pattern.search(filename) need_exclude |= bool(match) match_extension = False for extension in self._extensions: match = extension.search(filename) match_extension |= bool(match) need_exclude |= not match_extension return need_exclude def add_issue(self, unfmt_issue): """add an issue to the list if not already present""" issue = self._format_issue(unfmt_issue) filepath = issue['location']['path'] if self._need_exclude(filepath): return if issue['fingerprint'] in self._issues: return self._issues[issue['fingerprint']] = issue @property def issues(self): """get the list of registered issues""" return list(self._issues.values()) def _format_issue(self, unfmt_issue): filepath = os.path.relpath(unfmt_issue['file']) issue = { 'type': 'issue', 'check_name': unfmt_issue['name'], - 'description': unfmt_issue['description'], + 'description': (f"[{unfmt_issue['name']}]" + \ + "{unfmt_issue['description']}"), 'location': { "path": filepath, "lines": { "begin": unfmt_issue['line'], "end": unfmt_issue['line'], }, "positions": { "begin": { "line": unfmt_issue['line'], "column": unfmt_issue['column'], }, 'end': { "line": unfmt_issue['line'], "column": unfmt_issue['column'], }, }, }, } if 'end_line' in unfmt_issue: issue['location']['positions']['end'] = { "line": unfmt_issue['end_line'], "column": unfmt_issue['column'], } issue['location']['lines']['end'] = unfmt_issue['end_line'] issue['fingerprint'] = hashlib.md5( '{file}:{line}:{column}:{type}'.format( file=filepath, line=unfmt_issue['line'], column=unfmt_issue['column'], type=unfmt_issue['name']).encode()).hexdigest() issue['categories'], issue['severity'] = \ self._get_classifiaction(unfmt_issue) print_debug(issue) return issue diff --git a/test/ci/scripts/codequality/issue_generator_clang_format.py b/test/ci/scripts/codequality/issue_generator_clang_format.py index 8a507d179..a604668fd 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_format.py +++ b/test/ci/scripts/codequality/issue_generator_clang_format.py @@ -1,54 +1,54 @@ #!/usr/bin/env python3 from . import print_debug, print_info from .issue_generator_clang_tool import ClangToolIssueGenerator import os import re import copy import difflib import subprocess class ClangFormatIssueGenerator(ClangToolIssueGenerator): """issue generator for clang format""" def __init__(self, **kwargs): kwargs['clang_tool_executable'] = kwargs.pop('clang_format_executable', 'clang-format') super().__init__('clang-format', **kwargs) def _get_classifiaction(self, issue): return (['Style'], 'info') def generate_issues(self): issue = {} for filename in self._files: with open(filename, 'r') as fh: unformated_file = fh.readlines() command = copy.copy(self._command) command.append(filename) formated_file = list(self._run_command(command)) # diffs = difflib.unified_diff(unformated_file, formated_file, n=0) # print(diffs) # for diff in diffs: # print(diff, end='') s = difflib.SequenceMatcher(None, unformated_file, formated_file) for tag, i1, i2, j1, j2 in s.get_opcodes(): if tag != 'equal': diff = list( difflib.unified_diff( unformated_file[i1:i2], formated_file[j1:j2])) issue = { - 'name': tag, + 'name': f'clang-format:{tag}', 'description': ''.join(diff[3:]), 'file': filename, 'line': i1, 'column': 1, 'end_line': i2, } self.add_issue(issue) diff --git a/test/ci/scripts/codequality/issue_generator_clang_tidy.py b/test/ci/scripts/codequality/issue_generator_clang_tidy.py index f2a154183..6e4fa37c4 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_tidy.py +++ b/test/ci/scripts/codequality/issue_generator_clang_tidy.py @@ -1,102 +1,105 @@ #!/usr/bin/env python3 from . import print_debug, print_info from .issue_generator_clang_tool import ClangToolIssueGenerator import os import re import copy import json import subprocess class ClangTidyIssueGenerator(ClangToolIssueGenerator): """issue generator for clang tidy""" # 7-bit C1 ANSI sequences ANSI_ESCAPE = re.compile(r''' \x1B # ESC (?: # 7-bit C1 Fe (except CSI) [@-Z\\-_] | # or [ for CSI, followed by a control sequence \[ [0-?]* # Parameter bytes [ -/]* # Intermediate bytes [@-~] # Final byte ) ''', re.VERBOSE) ISSUE_PARSE = re.compile(r'(?P.*\.(cc|hh)):(?P[0-9]+):(?P[0-9]+): (warning|error): (?P.*) \[(?P.*)\]') # NOQA pylint: disable=line-too-long CLASSIFICATIONS = { 'bugprone': { 'categories': ['Bug Risk'], 'severity': 'major', }, 'modernize': { 'categories': ['Clarity', 'Compatibility', 'Style'], 'severity': 'info' }, 'mpi': { 'categories': ['Bug Risk', 'Performance'], 'severity': 'critical', }, 'openmp': { 'categories': ['Bug Risk', 'Performance'], 'severity': 'critical', }, 'performance': { 'categories': ['Performance'], 'severity': 'minor', }, 'readability': { 'categories': ['Clarity', 'Style'], 'severity': 'info' }, } def __init__(self, **kwargs): kwargs['clang_tool_executable'] = kwargs.pop('clang_tidy_executable', 'clang-tidy') super().__init__('clang-tidy', need_compiledb=True, **kwargs) def _get_classifiaction(self, issue): - type_ = issue['name'] + type_ = issue['type'] categories = ['Bug Risk'] severity = 'blocker' if type_ in self.CLASSIFICATIONS: categories = self.CLASSIFICATIONS[type_]['categories'] severity = self.CLASSIFICATIONS[type_]['severity'] elif type_[0] == 'clang': if type_[1] == 'diagnostic': categories = ['Bug Risk'] severity = 'blocker' elif type_[1] == 'analyzer': categories = ['Bug Risk'] severity = 'major' return (categories, severity) def generate_issues(self): issue = {} for filename in self._files: command = copy.copy(self._command) command.append(filename) for line in self._run_command(command): line = line.rstrip() match = self.ISSUE_PARSE.match(line) if match: if len(issue) != 0: self.add_issue(issue) issue = match.groupdict() + issue['type'] = issue['name'] + issue['name'] = f"clang-tidy:{issue['name']}" + print_debug(f'[clang-tidy] new issue: {line}') elif issue: if 'content' in issue: issue['content'].append(line) print_debug(f'[clang-tidy] more extra content: {line}') else: issue['content'] = [line] print_debug(f'[clang-tidy] extra content: {line}') if len(issue) != 0: self.add_issue(issue) diff --git a/test/ci/scripts/codequality/issue_generator_warnings.py b/test/ci/scripts/codequality/issue_generator_warnings.py index c09ec72dc..b39eb3461 100644 --- a/test/ci/scripts/codequality/issue_generator_warnings.py +++ b/test/ci/scripts/codequality/issue_generator_warnings.py @@ -1,81 +1,81 @@ #!/usr/bin/env python3 """clang-tidy2code-quality.py: Conversion of clang-tidy output 2 code-quality""" __author__ = "Nicolas Richart" __credits__ = [ "Nicolas Richart ", ] __copyright__ = "Copyright (©) 2018-2021 EPFL (Ecole Polytechnique Fédérale" \ " de Lausanne) Laboratory (LSMS - Laboratoire de Simulation" \ " en Mécanique des Solides)" __license__ = "LGPLv3" from . import print_debug, print_info from .issue_generator import IssueGenerator import re import sys import warning_parser as warn class WarningsIssueGenerator(IssueGenerator): ''' Main class to run and convert the results of clang-tidy to the code-quality format ''' CLASSIFICATIONS = { 'gcc': { 'uninitialized': { 'categories': ['Bug Risk'], 'severity': 'major', }, 'sign-compare': { 'categories': ['Bug Risk'], 'severity': 'minor' }, }, } def __init__(self, **kwargs): super().__init__(**kwargs) files = kwargs.pop('files') self._input_files = {} compiler_re = re.compile(".*build.*(gcc|clang)-err\.log") for _file in files: match = compiler_re.search(_file) if match: self._input_files[match.group(1)] = _file else: print_info(f"Skipped {_file}, could not determine compiler") def generate_issues(self): '''parse warning files''' for compiler, _file in self._input_files.items(): warnings = warn.get_warnings(_file, compiler) for warning in warnings: issue = { - 'name': warning.get_category(), + 'name': f'warning:{compiler}-{warning.get_category()}', 'description': warning.get_message(), 'file': warning.get_filepath(), 'line': warning.get_line(), 'column': warning.get_column(), 'raw': warning, } self.add_issue(issue) def _get_classifiaction(self, warning): categories = ['Clarity'] severity = 'info' if warning.get_tool() in self.CLASSIFICATIONS: classifications = self.CLASSIFICATIONS[warning.get_tool()] if warning.get_category() in classifications: cat = warning.get_category() categories = classifications[cat]['categories'] severity = classifications[cat]['severity'] return (categories, severity)