diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbce0c2..a950eb5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,286 +1,288 @@ stages: - docker - lint - build - test - deploy variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG GIT_SUBMODULE_STRATEGY: recursive BUILD_DIR: build-release MAIN_BRANCH: master cache: key: "$CI_COMMIT_REF_SLUG" # ------------------------------------------------------------------------------ .docker_build: stage: docker image: docker:19.03.12 services: - docker:19.03.12-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled DOCKER_HOST: tcp://docker:2376 DOCKER_TLS_CERTDIR: "/certs" DEFAULT_IMAGE: $CI_REGISTRY_IMAGE:$CI_DEFAULT_BRANCH before_script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - docker pull $DEFAULT_IMAGE-$IMAGE_NAME || true - docker build --cache-from $DEFAULT_IMAGE-$IMAGE_NAME -t $IMAGE_TAG-$IMAGE_NAME -f $DOCKERFILE . - docker push $IMAGE_TAG-$IMAGE_NAME docker build:debian: variables: IMAGE_NAME: debian-stable DOCKERFILE: tests/ci/docker/debian.mpi extends: .docker_build docker build:manylinux: variables: IMAGE_NAME: manylinux DOCKERFILE: tests/ci/docker/manylinux extends: .docker_build docker build:cuda: variables: IMAGE_NAME: cuda DOCKERFILE: tests/ci/docker/ubuntu_lts.cuda extends: .docker_build # ------------------------------------------------------------------------------ .debian_stable: variables: output: ${CI_COMMIT_REF_SLUG}-debian-stable image: ${IMAGE_TAG}-debian-stable .manylinux: variables: output: ${CI_COMMIT_REF_SLUG}-manylinux image: ${IMAGE_TAG}-manylinux .cuda: variables: output: ${CI_COMMIT_REF_SLUG}-cuda image: ${IMAGE_TAG}-cuda # ------------------------------------------------------------------------------ lint: stage: lint extends: - .debian_stable allow_failure: true variables: CLANG_PATCH: clang_format.patch FLAKE_ERRORS: flake8_lint.txt script: - git remote remove upstream || true - git remote add upstream "$CI_PROJECT_URL" - git fetch upstream - '[ -z "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ] && HEAD="$CI_COMMIT_BEFORE_SHA" || HEAD="upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"' - echo "$HEAD" - git-clang-format --diff "$HEAD" | tee $CLANG_PATCH - grep "no modified files to format" $CLANG_PATCH || grep "clang-format did not modify any files" $CLANG_PATCH - flake8 python/ | tee $FLAKE_ERRORS - '[ ! -s $FLAKE_ERRORS ]' artifacts: when: on_failure paths: - $CLANG_PATCH - $FLAKE_ERRORS .build: stage: build variables: COMPILE_LOG: compilation.log script: - scons 2>&1 | tee $COMPILE_LOG artifacts: when: always paths: &build_artifacts - build-setup.conf - $COMPILE_LOG - $BUILD_DIR - config.log - "*.json" .ccache: variables: CCACHE_BASEDIR: $CI_PROJECT_DIR/$BUILD_DIR CCACHE_DIR: $CI_PROJECT_DIR/.ccache CCACHE_NOHASDIR: 1 CCACHE_COMPILERCHECK: content CXX: /usr/lib/ccache/g++ OMPI_CXX: ${CXX} cache: key: ${output} policy: pull-push paths: - .ccache after_script: - ccache --show-stats || true build:mpi: extends: - .build - .debian_stable - .ccache before_script: - ccache --zero-stats || true - bear -- scons build_tests=True use_googletest=True build_python=True py_exec=python3 use_mpi=True backend=omp fftw_threads=omp verbose=True -h .build:cuda: extends: - .build - .cuda before_script: - scons build_tests=True use_googletest=True build_python=True py_exec=python3 use_mpi=False backend=cuda fftw_threads=none verbose=True -h # ------------------------------------------------------------------------------ -.hidden: - clang-tidy: - stage: test - dependencies: - - build:mpi - allow_failure: true - extends: .debian_stable - variables: - CLANG_TIDY_FILE: clang_tidy.txt - script: - - run-clang-tidy -extra-arg=-UTAMAAS_USE_MPI -header-filter="build-release/.*" -quiet src tests python > $CLANG_TIDY_FILE || true - - '! grep "warning:" $CLANG_TIDY_FILE' - artifacts: - when: on_failure - paths: - - $CLANG_TIDY_FILE +static-checks: + stage: test + dependencies: + - build:mpi + allow_failure: true + extends: .debian_stable + variables: + CLANG_TIDY_FILE: clang_tidy.txt + MYPY_FILE: mypy.txt + script: + - run-clang-tidy -extra-arg=-UTAMAAS_USE_MPI -header-filter="build-release/.*" -quiet src tests python > $CLANG_TIDY_FILE || true + - '[ -d build-release/python ] && ( mypy --ignore-missing-import build-release/python > $MYPY_FILE || true ) || true' + - '! grep "warning:" $CLANG_TIDY_FILE && ! grep "error:" $MYPY_FILE' + artifacts: + when: on_failure + paths: + - $CLANG_TIDY_FILE + - $MYPY_FILE test:mpi: stage: test dependencies: - build:mpi extends: .debian_stable variables: PYTHONPATH: $CI_PROJECT_DIR/$BUILD_DIR/python TESTS: $BUILD_DIR/tests JUNITXML: results.xml TESTS_LOG: tests.log script: - ls $PYTHONPATH - python3 -c "import sys; print(sys.path)" - python3 -m pytest -vvv --last-failed --durations=0 --junitxml=$JUNITXML $TESTS 2>&1 | tee $TESTS_LOG after_script: - python3 -m pytest --cache-show || true artifacts: when: always paths: - $JUNITXML - $TESTS_LOG reports: junit: - $JUNITXML cache: key: ${output}-pytest policy: pull-push paths: - .pytest_cache # ------------------------------------------------------------------------------ .protected_refs: extends: .manylinux rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG =~ /^v.*/' wheels: stage: build extends: - .protected_refs - .ccache before_script: - ccache --zero-stats || true script: - ./tests/ci/build_wheels.sh artifacts: paths: - dist/wheelhouse # ------------------------------------------------------------------------------ .deploy_wheels: stage: deploy extends: .protected_refs dependencies: - wheels script: python -m twine upload --verbose --skip-existing dist/wheelhouse/* package:gitlab: extends: .deploy_wheels variables: TWINE_USERNAME: gitlab-ci-token TWINE_PASSWORD: ${CI_JOB_TOKEN} TWINE_REPOSITORY_URL: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi package:pypi: extends: .deploy_wheels variables: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${PYPI_TOKEN} rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' package:zenodo: stage: deploy extends: .debian_stable dependencies: [] # do not download artifacts => clean tree variables: TAMAAS_RECORD: 5713675 script: - scons archive - 'TAMAAS_VERSION="$(./INFOS.py --version)"' - ./tests/ci/zenodo.py --record ${TAMAAS_RECORD} --token ${ZENODO_TOKEN} --version ${TAMAAS_VERSION} CHANGELOG.md COPYING README.md tamaas-${TAMAAS_VERSION}.tar.gz rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' package:docker: stage: deploy extends: .docker_build variables: DOCKERFILE: Dockerfile script: - '[ "$CI_COMMIT_REF_SLUG" = "$MAIN_BRANCH" ] && IMAGE="$CI_REGISTRY_IMAGE:latest" || IMAGE="$IMAGE_TAG"' - docker pull $IMAGE || true - docker build --cache-from $IMAGE --build-arg USE_MPI=True -t $IMAGE -f $DOCKERFILE . - docker push $IMAGE diff --git a/tests/ci/docker/debian.mpi b/tests/ci/docker/debian.mpi index b6398de..9027af1 100644 --- a/tests/ci/docker/debian.mpi +++ b/tests/ci/docker/debian.mpi @@ -1,50 +1,51 @@ FROM debian:stable-slim MAINTAINER Lucas Frérot # Add contrib and non-free RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list # Core dependencies RUN apt-get -qq update && apt-get install -qq -y --no-install-recommends \ g++ \ git \ libboost-dev \ libthrust-dev \ libopenmpi-dev \ libfftw3-dev \ libfftw3-mpi-dev \ libgtest-dev \ pybind11-dev \ scons \ && rm -rf /var/lib/apt/lists/* # Python dependencies RUN apt-get -qq update && apt-get install -qq -y --no-install-recommends \ python3-dev \ python3-pip \ python3-numpy \ python3-scipy \ python3-h5py \ python3-netcdf4 \ python3-mpi4py \ && rm -rf /var/lib/apt/lists/* RUN python3 -m pip install pytest uvw # Documentation dependencies RUN apt-get -qq update && apt-get install -qq -y --no-install-recommends \ python3-sphinx \ python3-breathe \ python3-sphinx-rtd-theme \ doxygen \ && rm -rf /var/lib/apt/lists/* # Linting etc. RUN apt-get -qq update && apt-get install -qq -y --no-install-recommends \ ccache \ clang-format \ clang-tidy \ bear \ python3-requests \ + mypy \ && rm -rf /var/lib/apt/lists/* RUN python3 -m pip install flake8