diff --git a/test/ci/scripts/codequality/issue_generator.py b/test/ci/scripts/codequality/issue_generator.py index 95efb400f..5f26d87fe 100644 --- a/test/ci/scripts/codequality/issue_generator.py +++ b/test/ci/scripts/codequality/issue_generator.py @@ -1,123 +1,123 @@ #!/usr/bin/env python3 from . import print_debug, print_info import hashlib import os import re import copy class IssueGenerator: """Interface for the issue generators""" def __init__(self, **kwargs): self._files = kwargs.pop('file_list', []) excludes = kwargs.pop('excludes', None) if excludes is None: excludes = [] extensions = kwargs.pop('extensions', None) if extensions is None: extensions = ['.cc', '.hh'] self._extensions = [ re.compile(r"\{}$".format(extension)) for extension in extensions ] self._exclude_patterns = [ re.compile(exclude) for exclude in excludes ] self._issues = {} self._filter_file_list() def _filter_file_list(self): file_list = copy.copy(self._files) self._files = [] for filename in file_list: filename = os.path.relpath(filename) need_exclude = self._need_exclude(filename) if need_exclude: print_debug(f'exluding file: {filename}') continue print_info(f'adding file: {filename}') self._files.append(filename) def _need_exclude(self, filename): need_exclude = False for pattern in self._exclude_patterns: match = pattern.search(filename) need_exclude |= bool(match) match_extension = False for extension in self._extensions: match = extension.search(filename) match_extension |= bool(match) need_exclude |= not match_extension return need_exclude def add_issue(self, unfmt_issue): """add an issue to the list if not already present""" issue = self._format_issue(unfmt_issue) filepath = issue['location']['path'] if self._need_exclude(filepath): return if issue['fingerprint'] in self._issues: return self._issues[issue['fingerprint']] = issue @property def issues(self): """get the list of registered issues""" return list(self._issues.values()) def _format_issue(self, unfmt_issue): filepath = os.path.relpath(unfmt_issue['file']) issue = { 'type': 'issue', 'check_name': unfmt_issue['name'], - 'description': (f"[{unfmt_issue['name']}]" + \ - "{unfmt_issue['description']}"), + 'description': ( + f'''[{unfmt_issue['name']}] {unfmt_issue['description']}'''), 'location': { "path": filepath, "lines": { "begin": unfmt_issue['line'], "end": unfmt_issue['line'], }, "positions": { "begin": { "line": unfmt_issue['line'], "column": unfmt_issue['column'], }, 'end': { "line": unfmt_issue['line'], "column": unfmt_issue['column'], }, }, }, } if 'end_line' in unfmt_issue: issue['location']['positions']['end'] = { "line": unfmt_issue['end_line'], "column": unfmt_issue['column'], } issue['location']['lines']['end'] = unfmt_issue['end_line'] issue['fingerprint'] = hashlib.md5( '{file}:{line}:{column}:{type}'.format( file=filepath, line=unfmt_issue['line'], column=unfmt_issue['column'], type=unfmt_issue['name']).encode()).hexdigest() issue['categories'], issue['severity'] = \ self._get_classifiaction(unfmt_issue) print_debug(issue) return issue diff --git a/test/ci/scripts/codequality/issue_generator_clang_format.py b/test/ci/scripts/codequality/issue_generator_clang_format.py index a604668fd..296f86c7b 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_format.py +++ b/test/ci/scripts/codequality/issue_generator_clang_format.py @@ -1,54 +1,54 @@ #!/usr/bin/env python3 from . import print_debug, print_info from .issue_generator_clang_tool import ClangToolIssueGenerator import os import re import copy import difflib import subprocess class ClangFormatIssueGenerator(ClangToolIssueGenerator): """issue generator for clang format""" def __init__(self, **kwargs): kwargs['clang_tool_executable'] = kwargs.pop('clang_format_executable', 'clang-format') super().__init__('clang-format', **kwargs) def _get_classifiaction(self, issue): return (['Style'], 'info') def generate_issues(self): issue = {} for filename in self._files: with open(filename, 'r') as fh: unformated_file = fh.readlines() command = copy.copy(self._command) command.append(filename) formated_file = list(self._run_command(command)) # diffs = difflib.unified_diff(unformated_file, formated_file, n=0) # print(diffs) # for diff in diffs: # print(diff, end='') s = difflib.SequenceMatcher(None, unformated_file, formated_file) for tag, i1, i2, j1, j2 in s.get_opcodes(): if tag != 'equal': diff = list( difflib.unified_diff( unformated_file[i1:i2], formated_file[j1:j2])) issue = { - 'name': f'clang-format:{tag}', + 'name': f'''clang-format:{tag}''', 'description': ''.join(diff[3:]), 'file': filename, 'line': i1, 'column': 1, 'end_line': i2, } self.add_issue(issue) diff --git a/test/ci/scripts/codequality/issue_generator_clang_tidy.py b/test/ci/scripts/codequality/issue_generator_clang_tidy.py index 6e4fa37c4..0ad5412ee 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_tidy.py +++ b/test/ci/scripts/codequality/issue_generator_clang_tidy.py @@ -1,105 +1,105 @@ #!/usr/bin/env python3 from . import print_debug, print_info from .issue_generator_clang_tool import ClangToolIssueGenerator import os import re import copy import json import subprocess class ClangTidyIssueGenerator(ClangToolIssueGenerator): """issue generator for clang tidy""" # 7-bit C1 ANSI sequences ANSI_ESCAPE = re.compile(r''' \x1B # ESC (?: # 7-bit C1 Fe (except CSI) [@-Z\\-_] | # or [ for CSI, followed by a control sequence \[ [0-?]* # Parameter bytes [ -/]* # Intermediate bytes [@-~] # Final byte ) ''', re.VERBOSE) ISSUE_PARSE = re.compile(r'(?P.*\.(cc|hh)):(?P[0-9]+):(?P[0-9]+): (warning|error): (?P.*) \[(?P.*)\]') # NOQA pylint: disable=line-too-long CLASSIFICATIONS = { 'bugprone': { 'categories': ['Bug Risk'], 'severity': 'major', }, 'modernize': { 'categories': ['Clarity', 'Compatibility', 'Style'], 'severity': 'info' }, 'mpi': { 'categories': ['Bug Risk', 'Performance'], 'severity': 'critical', }, 'openmp': { 'categories': ['Bug Risk', 'Performance'], 'severity': 'critical', }, 'performance': { 'categories': ['Performance'], 'severity': 'minor', }, 'readability': { 'categories': ['Clarity', 'Style'], 'severity': 'info' }, } def __init__(self, **kwargs): kwargs['clang_tool_executable'] = kwargs.pop('clang_tidy_executable', 'clang-tidy') super().__init__('clang-tidy', need_compiledb=True, **kwargs) def _get_classifiaction(self, issue): type_ = issue['type'] categories = ['Bug Risk'] severity = 'blocker' if type_ in self.CLASSIFICATIONS: categories = self.CLASSIFICATIONS[type_]['categories'] severity = self.CLASSIFICATIONS[type_]['severity'] elif type_[0] == 'clang': if type_[1] == 'diagnostic': categories = ['Bug Risk'] severity = 'blocker' elif type_[1] == 'analyzer': categories = ['Bug Risk'] severity = 'major' return (categories, severity) def generate_issues(self): issue = {} for filename in self._files: command = copy.copy(self._command) command.append(filename) for line in self._run_command(command): line = line.rstrip() match = self.ISSUE_PARSE.match(line) if match: if len(issue) != 0: self.add_issue(issue) issue = match.groupdict() issue['type'] = issue['name'] - issue['name'] = f"clang-tidy:{issue['name']}" + issue['name'] = f'''clang-tidy:{issue['name']}''' print_debug(f'[clang-tidy] new issue: {line}') elif issue: if 'content' in issue: issue['content'].append(line) print_debug(f'[clang-tidy] more extra content: {line}') else: issue['content'] = [line] print_debug(f'[clang-tidy] extra content: {line}') if len(issue) != 0: self.add_issue(issue)