diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a950eb5..9f21ee8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,288 +1,289 @@ stages: - docker - lint - build - test - deploy variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG GIT_SUBMODULE_STRATEGY: recursive BUILD_DIR: build-release MAIN_BRANCH: master cache: key: "$CI_COMMIT_REF_SLUG" # ------------------------------------------------------------------------------ .docker_build: stage: docker image: docker:19.03.12 services: - docker:19.03.12-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled DOCKER_HOST: tcp://docker:2376 DOCKER_TLS_CERTDIR: "/certs" DEFAULT_IMAGE: $CI_REGISTRY_IMAGE:$CI_DEFAULT_BRANCH before_script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - docker pull $DEFAULT_IMAGE-$IMAGE_NAME || true - docker build --cache-from $DEFAULT_IMAGE-$IMAGE_NAME -t $IMAGE_TAG-$IMAGE_NAME -f $DOCKERFILE . - docker push $IMAGE_TAG-$IMAGE_NAME docker build:debian: variables: IMAGE_NAME: debian-stable DOCKERFILE: tests/ci/docker/debian.mpi extends: .docker_build docker build:manylinux: variables: IMAGE_NAME: manylinux DOCKERFILE: tests/ci/docker/manylinux extends: .docker_build docker build:cuda: variables: IMAGE_NAME: cuda DOCKERFILE: tests/ci/docker/ubuntu_lts.cuda extends: .docker_build # ------------------------------------------------------------------------------ .debian_stable: variables: output: ${CI_COMMIT_REF_SLUG}-debian-stable image: ${IMAGE_TAG}-debian-stable .manylinux: variables: output: ${CI_COMMIT_REF_SLUG}-manylinux image: ${IMAGE_TAG}-manylinux .cuda: variables: output: ${CI_COMMIT_REF_SLUG}-cuda image: ${IMAGE_TAG}-cuda # ------------------------------------------------------------------------------ lint: stage: lint extends: - .debian_stable allow_failure: true variables: CLANG_PATCH: clang_format.patch FLAKE_ERRORS: flake8_lint.txt script: - git remote remove upstream || true - git remote add upstream "$CI_PROJECT_URL" - git fetch upstream - '[ -z "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ] && HEAD="$CI_COMMIT_BEFORE_SHA" || HEAD="upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"' - echo "$HEAD" - git-clang-format --diff "$HEAD" | tee $CLANG_PATCH - grep "no modified files to format" $CLANG_PATCH || grep "clang-format did not modify any files" $CLANG_PATCH - flake8 python/ | tee $FLAKE_ERRORS - '[ ! -s $FLAKE_ERRORS ]' artifacts: when: on_failure paths: - $CLANG_PATCH - $FLAKE_ERRORS .build: stage: build variables: COMPILE_LOG: compilation.log script: - scons 2>&1 | tee $COMPILE_LOG artifacts: when: always paths: &build_artifacts - build-setup.conf - $COMPILE_LOG - $BUILD_DIR - config.log - "*.json" .ccache: variables: CCACHE_BASEDIR: $CI_PROJECT_DIR/$BUILD_DIR CCACHE_DIR: $CI_PROJECT_DIR/.ccache CCACHE_NOHASDIR: 1 CCACHE_COMPILERCHECK: content CXX: /usr/lib/ccache/g++ OMPI_CXX: ${CXX} cache: key: ${output} policy: pull-push paths: - .ccache after_script: - ccache --show-stats || true build:mpi: extends: - .build - .debian_stable - .ccache before_script: - ccache --zero-stats || true - bear -- scons build_tests=True use_googletest=True build_python=True py_exec=python3 use_mpi=True backend=omp fftw_threads=omp verbose=True -h .build:cuda: extends: - .build - .cuda before_script: - scons build_tests=True use_googletest=True build_python=True py_exec=python3 use_mpi=False backend=cuda fftw_threads=none verbose=True -h # ------------------------------------------------------------------------------ static-checks: stage: test dependencies: - build:mpi allow_failure: true extends: .debian_stable variables: CLANG_TIDY_FILE: clang_tidy.txt MYPY_FILE: mypy.txt script: + - rm src/core/fftw/fftw_mpi_engine.cpp tests/test_mpi.cpp - run-clang-tidy -extra-arg=-UTAMAAS_USE_MPI -header-filter="build-release/.*" -quiet src tests python > $CLANG_TIDY_FILE || true - '[ -d build-release/python ] && ( mypy --ignore-missing-import build-release/python > $MYPY_FILE || true ) || true' - - '! grep "warning:" $CLANG_TIDY_FILE && ! grep "error:" $MYPY_FILE' + - '! grep -E "(warning|error)" $CLANG_TIDY_FILE && ! grep "error" $MYPY_FILE' artifacts: when: on_failure paths: - $CLANG_TIDY_FILE - $MYPY_FILE test:mpi: stage: test dependencies: - build:mpi extends: .debian_stable variables: PYTHONPATH: $CI_PROJECT_DIR/$BUILD_DIR/python TESTS: $BUILD_DIR/tests JUNITXML: results.xml TESTS_LOG: tests.log script: - ls $PYTHONPATH - python3 -c "import sys; print(sys.path)" - python3 -m pytest -vvv --last-failed --durations=0 --junitxml=$JUNITXML $TESTS 2>&1 | tee $TESTS_LOG after_script: - python3 -m pytest --cache-show || true artifacts: when: always paths: - $JUNITXML - $TESTS_LOG reports: junit: - $JUNITXML cache: key: ${output}-pytest policy: pull-push paths: - .pytest_cache # ------------------------------------------------------------------------------ .protected_refs: extends: .manylinux rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG =~ /^v.*/' wheels: stage: build extends: - .protected_refs - .ccache before_script: - ccache --zero-stats || true script: - ./tests/ci/build_wheels.sh artifacts: paths: - dist/wheelhouse # ------------------------------------------------------------------------------ .deploy_wheels: stage: deploy extends: .protected_refs dependencies: - wheels script: python -m twine upload --verbose --skip-existing dist/wheelhouse/* package:gitlab: extends: .deploy_wheels variables: TWINE_USERNAME: gitlab-ci-token TWINE_PASSWORD: ${CI_JOB_TOKEN} TWINE_REPOSITORY_URL: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi package:pypi: extends: .deploy_wheels variables: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${PYPI_TOKEN} rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' package:zenodo: stage: deploy extends: .debian_stable dependencies: [] # do not download artifacts => clean tree variables: TAMAAS_RECORD: 5713675 script: - scons archive - 'TAMAAS_VERSION="$(./INFOS.py --version)"' - ./tests/ci/zenodo.py --record ${TAMAAS_RECORD} --token ${ZENODO_TOKEN} --version ${TAMAAS_VERSION} CHANGELOG.md COPYING README.md tamaas-${TAMAAS_VERSION}.tar.gz rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' package:docker: stage: deploy extends: .docker_build variables: DOCKERFILE: Dockerfile script: - '[ "$CI_COMMIT_REF_SLUG" = "$MAIN_BRANCH" ] && IMAGE="$CI_REGISTRY_IMAGE:latest" || IMAGE="$IMAGE_TAG"' - docker pull $IMAGE || true - docker build --cache-from $IMAGE --build-arg USE_MPI=True -t $IMAGE -f $DOCKERFILE . - docker push $IMAGE