diff --git a/test/ci/debian:bullseye/Dockerfile b/test/ci/debian:bullseye/Dockerfile index c58571b90..1fed2d6d7 100644 --- a/test/ci/debian:bullseye/Dockerfile +++ b/test/ci/debian:bullseye/Dockerfile @@ -1,38 +1,39 @@ FROM debian:bullseye # library dependencies RUN apt -qq update && apt -qq -y install \ g++ gfortran clang cmake \ openmpi-bin libmumps-dev libscotch-dev \ libboost-dev libopenblas-dev \ libeigen3-dev \ python3 python3-dev python3-numpy python3-scipy python3-mpi4py \ && rm -rf /var/lib/apt/lists/* # for documentation RUN apt -qq update && apt -qq -y install \ python3-sphinx \ python3-sphinxcontrib.bibtex \ python3-sphinx-rtd-theme \ python3-breathe \ python3-git python3-jinja2 \ doxygen graphviz \ && rm -rf /var/lib/apt/lists/* # for ci RUN apt -qq update && apt -qq -y install \ gmsh python3-pytest python3-click python3-termcolor \ - ccache clang-format python3-flake8 python3-pip clang-tidy \ + python3-flake8 python3-pip python3-tqdm \ + ccache clang-format clang-tidy \ curl git xsltproc jq \ gcovr llvm binutils \ ninja-build \ && rm -rf /var/lib/apt/lists/* RUN pip3 install warning_parser COPY .openmpi /root/.openmpi # for debug RUN apt -qq update && apt -qq -y install \ gdb valgrind \ && rm -rf /var/lib/apt/lists/* diff --git a/test/ci/scripts/codequality/issue_generator_clang_format.py b/test/ci/scripts/codequality/issue_generator_clang_format.py index 683bd55d8..30dcedc00 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_format.py +++ b/test/ci/scripts/codequality/issue_generator_clang_format.py @@ -1,59 +1,52 @@ #!/usr/bin/env python3 from .issue_generator_clang_tool import ClangToolIssueGenerator import copy import difflib - class ClangFormatIssueGenerator(ClangToolIssueGenerator): """issue generator for clang format""" def __init__(self, **kwargs): kwargs['clang_tool_executable'] = kwargs.pop('clang_format_executable', 'clang-format') super().__init__('clang-format', **kwargs) def _get_classifiaction(self, issue): return (['Style'], 'info') - def generate_issues(self): - issue = {} - for filename in self._files: - with open(filename, 'r') as fh: - unformated_file = fh.readlines() - - command = copy.copy(self._command) - command.append(filename) - formated_file = list(self._run_command(command)) - - # diffs = difflib.unified_diff(unformated_file, formated_file, n=0) - - # print(diffs) - # for diff in diffs: - # print(diff, end='') - - s = difflib.SequenceMatcher(None, unformated_file, formated_file) - for tag, i1, i2, j1, j2 in s.get_opcodes(): - if tag == 'equal': - continue - # diff = list( - # difflib.unified_diff( - # unformated_file[i1:i2], - # formated_file[j1:j2])) - description = '' - if tag == 'delete': - description = f'```suggestion:-0+{i2-i1}\n```' - if tag == 'insert': - description = f'''```suggestion:-0+0\n{''.join(unformated_file[i1:i2])}{''.join(formated_file[j1:j2])}```''' # noqa - if tag == 'replace': - description = f'''```suggestion:-0+{i2-i1}\n{''.join(formated_file[j1:j2])}```''' # noqa - - issue = { - 'name': f'''clang-format:{tag}''', - 'description': ''.join(description), - 'file': filename, - 'line': i1 + 1, # lines start at 1 not 0 - 'column': 1, - 'end_line': i2 + 1, - } - self.add_issue(issue) + + def _generate_issues_for_file(self, filename): + with open(filename, 'r') as fh: + unformated_file = fh.readlines() + + command = copy.copy(self._command) + command.append(filename) + if self._current_file is not None: + self._current_file.set_description_str(f"Current file: {filename}") + + formated_file = list(self._run_command(command)) + + issues = [] + s = difflib.SequenceMatcher(None, unformated_file, formated_file) + for tag, i1, i2, j1, j2 in s.get_opcodes(): + description = '' + if tag == 'equal': + continue + if tag == 'delete': + description = f'```suggestion:-0+{i2-i1-1}\n```' + if tag == 'insert': + description = f'''```suggestion:-0+0\n{''.join(unformated_file[i1:i2])}{''.join(formated_file[j1:j2])}```''' # noqa + if tag == 'replace': + description = f'''```suggestion:-0+{i2-i1-1}\n{''.join(formated_file[j1:j2])}```''' # noqa + + issue = { + 'name': f'''clang-format:{tag}''', + 'description': ''.join(description), + 'file': filename, + 'line': i1 + 1, # lines start at 1 not 0 + 'column': 1, + 'end_line': i2 + 1, + } + issues.append(issue) + return issues diff --git a/test/ci/scripts/codequality/issue_generator_clang_tidy.py b/test/ci/scripts/codequality/issue_generator_clang_tidy.py index 0ad5412ee..7575facc5 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_tidy.py +++ b/test/ci/scripts/codequality/issue_generator_clang_tidy.py @@ -1,105 +1,111 @@ #!/usr/bin/env python3 from . import print_debug, print_info from .issue_generator_clang_tool import ClangToolIssueGenerator import os import re import copy import json import subprocess +import multiprocessing as mp class ClangTidyIssueGenerator(ClangToolIssueGenerator): """issue generator for clang tidy""" # 7-bit C1 ANSI sequences ANSI_ESCAPE = re.compile(r''' \x1B # ESC (?: # 7-bit C1 Fe (except CSI) [@-Z\\-_] | # or [ for CSI, followed by a control sequence \[ [0-?]* # Parameter bytes [ -/]* # Intermediate bytes [@-~] # Final byte ) ''', re.VERBOSE) ISSUE_PARSE = re.compile(r'(?P.*\.(cc|hh)):(?P[0-9]+):(?P[0-9]+): (warning|error): (?P.*) \[(?P.*)\]') # NOQA pylint: disable=line-too-long CLASSIFICATIONS = { 'bugprone': { 'categories': ['Bug Risk'], 'severity': 'major', }, 'modernize': { 'categories': ['Clarity', 'Compatibility', 'Style'], 'severity': 'info' }, 'mpi': { 'categories': ['Bug Risk', 'Performance'], 'severity': 'critical', }, 'openmp': { 'categories': ['Bug Risk', 'Performance'], 'severity': 'critical', }, 'performance': { 'categories': ['Performance'], 'severity': 'minor', }, 'readability': { 'categories': ['Clarity', 'Style'], 'severity': 'info' }, } def __init__(self, **kwargs): kwargs['clang_tool_executable'] = kwargs.pop('clang_tidy_executable', 'clang-tidy') super().__init__('clang-tidy', need_compiledb=True, **kwargs) + self._num_threads = min(2, mp.cpu_count()) def _get_classifiaction(self, issue): type_ = issue['type'] categories = ['Bug Risk'] severity = 'blocker' if type_ in self.CLASSIFICATIONS: categories = self.CLASSIFICATIONS[type_]['categories'] severity = self.CLASSIFICATIONS[type_]['severity'] elif type_[0] == 'clang': if type_[1] == 'diagnostic': categories = ['Bug Risk'] severity = 'blocker' elif type_[1] == 'analyzer': categories = ['Bug Risk'] severity = 'major' return (categories, severity) - def generate_issues(self): + def _generate_issues_for_file(self, filename): + command = copy.copy(self._command) + command.append(filename) + if self._current_file is not None: + self._current_file.set_description_str(f"Current file: {filename}") + + issues = [] issue = {} - for filename in self._files: - command = copy.copy(self._command) - command.append(filename) - for line in self._run_command(command): - line = line.rstrip() - match = self.ISSUE_PARSE.match(line) - if match: - if len(issue) != 0: - self.add_issue(issue) - issue = match.groupdict() - issue['type'] = issue['name'] - issue['name'] = f'''clang-tidy:{issue['name']}''' + for line in self._run_command(command): + line = line.rstrip() + match = self.ISSUE_PARSE.match(line) + if match: + if len(issue) != 0: + issues.append(issue) + issue = match.groupdict() + issue['type'] = issue['name'] + issue['name'] = f'''clang-tidy:{issue['name']}''' - print_debug(f'[clang-tidy] new issue: {line}') - elif issue: - if 'content' in issue: - issue['content'].append(line) - print_debug(f'[clang-tidy] more extra content: {line}') - else: - issue['content'] = [line] - print_debug(f'[clang-tidy] extra content: {line}') + #print_debug(f'[clang-tidy] new issue: {line}') + elif issue: + if 'content' in issue: + issue['content'].append(line) + #print_debug(f'[clang-tidy] more extra content: {line}') + else: + issue['content'] = [line] + #print_debug(f'[clang-tidy] extra content: {line}') if len(issue) != 0: - self.add_issue(issue) + issues.append(issue) + return issues diff --git a/test/ci/scripts/codequality/issue_generator_clang_tool.py b/test/ci/scripts/codequality/issue_generator_clang_tool.py index b1ef4d100..f0f6c05c9 100644 --- a/test/ci/scripts/codequality/issue_generator_clang_tool.py +++ b/test/ci/scripts/codequality/issue_generator_clang_tool.py @@ -1,74 +1,98 @@ #!/usr/bin/env python3 from . import print_debug, print_info from .issue_generator import IssueGenerator import os import re import copy import json import subprocess +import multiprocessing as mp +from multiprocessing.pool import ThreadPool as Pool +import tqdm class ClangToolIssueGenerator(IssueGenerator): """issue generator for clang tidy""" # 7-bit C1 ANSI sequences ANSI_ESCAPE = re.compile(r''' \x1B # ESC (?: # 7-bit C1 Fe (except CSI) [@-Z\\-_] | # or [ for CSI, followed by a control sequence \[ [0-?]* # Parameter bytes [ -/]* # Intermediate bytes [@-~] # Final byte ) ''', re.VERBOSE) def __init__(self, tool, **kwargs): self._tool = tool opts = copy.copy(kwargs) super().__init__(**kwargs) compiledb_path = opts.pop('compiledb_path') arguments = opts.pop('arguments', None) clang_tool = opts.pop('clang_tool_executable', tool) self._command = [clang_tool] if 'need_compiledb' in opts and opts['need_compiledb']: self._command.extend(['-p', compiledb_path]) if arguments is not None: self._command.extend(arguments) if len(self._files) == 0 and compiledb_path: self._get_files_from_compile_db(compiledb_path) + self._current_file = None + self._num_threads = mp.cpu_count() + def _get_files_from_compile_db(self, compiledb_path): file_list = [] with open(os.path.join( compiledb_path, 'compile_commands.json'), 'r') as compiledb_fh: compiledb = json.load(compiledb_fh) for entry in compiledb: file_list.append(entry['file']) self._files = file_list self._filter_file_list() def _run_command(self, command): - print_info(f'''[{self._tool}] command: {' '.join(command)}''') + #print_info(f'''[{self._tool}] command: {' '.join(command)}''') popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True) for stdout_line in iter(popen.stdout.readline, ""): clean_line = self.ANSI_ESCAPE.sub('', stdout_line) yield clean_line popen.stdout.close() return_code = popen.wait() if return_code: print_debug( f"[{self._tool}] {command} ReturnCode {return_code}") + + def generate_issues(self): + issues = [] + self._current_file = tqdm.tqdm(total=0, position=1, + bar_format='{desc}') + with Pool(self._num_threads) as p: + list_of_lists = list(tqdm.tqdm( + p.imap(self._generate_issues_for_file, + self._files), + position=0, + desc="Files: ", + total=len(self._files))) + + for list_ in list_of_lists: + issues.extend(list_) + + for issue in issues: + self.add_issue(issue)