diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0879531..0237bb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,263 +1,265 @@ stages: - docker - lint - build - test - deploy variables: IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG GIT_SUBMODULE_STRATEGY: recursive BUILD_DIR: build-release + MAIN_BRANCH: master cache: key: "$CI_COMMIT_REF_SLUG" # ------------------------------------------------------------------------------ .docker_build: stage: docker image: docker:19.03.12 services: - docker:19.03.12-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled DOCKER_HOST: tcp://docker:2376 DOCKER_TLS_CERTDIR: "/certs" DEFAULT_IMAGE: $CI_REGISTRY_IMAGE:$CI_DEFAULT_BRANCH before_script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - docker pull $DEFAULT_IMAGE-$IMAGE_NAME || true - docker build --cache-from $DEFAULT_IMAGE-$IMAGE_NAME -t $IMAGE_TAG-$IMAGE_NAME -f $DOCKERFILE . - docker push $IMAGE_TAG-$IMAGE_NAME docker build:debian: variables: IMAGE_NAME: debian-stable DOCKERFILE: tests/ci/docker/debian.mpi extends: .docker_build docker build:manylinux: variables: IMAGE_NAME: manylinux DOCKERFILE: tests/ci/docker/manylinux extends: .docker_build docker build:cuda: variables: IMAGE_NAME: cuda DOCKERFILE: tests/ci/docker/ubuntu_lts.cuda extends: .docker_build # ------------------------------------------------------------------------------ .debian_stable: variables: output: ${CI_COMMIT_REF_SLUG}-debian-stable image: ${IMAGE_TAG}-debian-stable .manylinux: variables: output: ${CI_COMMIT_REF_SLUG}-manylinux image: ${IMAGE_TAG}-manylinux .cuda: variables: output: ${CI_COMMIT_REF_SLUG}-cuda image: ${IMAGE_TAG}-cuda # ------------------------------------------------------------------------------ lint: stage: lint extends: - .debian_stable allow_failure: true variables: CLANG_PATCH: clang_format.patch FLAKE_ERRORS: flake8_lint.txt script: - git remote remove upstream || true - git remote add upstream "$CI_PROJECT_URL" - git fetch upstream - '[ -z "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ] && HEAD="$CI_COMMIT_BEFORE_SHA" || HEAD="upstream/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"' - echo "$HEAD" - git-clang-format --diff "$HEAD" | tee $CLANG_PATCH - grep "no modified files to format" $CLANG_PATCH || grep "clang-format did not modify any files" $CLANG_PATCH - flake8 python/ | tee $FLAKE_ERRORS - '[ ! -s $FLAKE_ERRORS ]' artifacts: when: on_failure paths: - $CLANG_PATCH - $FLAKE_ERRORS .build: stage: build variables: COMPILE_LOG: compilation.log script: - scons 2>&1 | tee $COMPILE_LOG artifacts: when: always paths: &build_artifacts - build-setup.conf - $COMPILE_LOG - $BUILD_DIR - config.log - "*.json" .ccache: variables: CCACHE_BASEDIR: $CI_PROJECT_DIR/$BUILD_DIR CCACHE_DIR: $CI_PROJECT_DIR/.ccache CCACHE_NOHASDIR: 1 CCACHE_COMPILERCHECK: content CXX: /usr/lib/ccache/g++ OMPI_CXX: ${CXX} cache: key: ${output} policy: pull-push paths: - .ccache after_script: - ccache --show-stats || true build:mpi: extends: - .build - .debian_stable - .ccache before_script: - ccache --zero-stats || true - bear -- scons build_tests=True use_googletest=True build_python=True py_exec=python3 use_mpi=True backend=omp fftw_threads=omp verbose=True -h build:cuda: extends: - .build - .cuda before_script: - scons build_tests=True use_googletest=True build_python=True py_exec=python3 use_mpi=False backend=cuda fftw_threads=none verbose=True -h # ------------------------------------------------------------------------------ clang-tidy: stage: test dependencies: - build:mpi allow_failure: true extends: .debian_stable variables: CLANG_TIDY_FILE: clang_tidy.txt script: - run-clang-tidy -header-filter="build-release/.*" -quiet src tests python > $CLANG_TIDY_FILE || true - '! grep "warning:" $CLANG_TIDY_FILE' artifacts: when: on_failure paths: - $CLANG_TIDY_FILE test:mpi: stage: test dependencies: - build:mpi extends: .debian_stable variables: PYTHONPATH: $CI_PROJECT_DIR/$BUILD_DIR/python TESTS: $BUILD_DIR/tests JUNITXML: results.xml TESTS_LOG: tests.log script: - ls $PYTHONPATH - python3 -c "import sys; print(sys.path)" - python3 -m pytest -vvv --last-failed --durations=0 --junitxml=$JUNITXML $TESTS 2>&1 | tee $TESTS_LOG after_script: - python3 -m pytest --cache-show || true artifacts: when: always paths: - $JUNITXML - $TESTS_LOG reports: junit: - $JUNITXML cache: key: ${output}-pytest policy: pull-push paths: - .pytest_cache # ------------------------------------------------------------------------------ .protected_refs: extends: .manylinux rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG =~ /^v.*/' wheels: stage: build extends: - .protected_refs - .ccache before_script: - ccache --zero-stats || true script: - ./tests/ci/build_wheels.sh artifacts: paths: - dist/wheelhouse # ------------------------------------------------------------------------------ .deploy_wheels: stage: deploy extends: .protected_refs dependencies: - wheels script: python -m twine upload --verbose dist/wheelhouse/* package:gitlab: extends: .deploy_wheels variables: TWINE_USERNAME: gitlab-ci-token TWINE_PASSWORD: ${CI_JOB_TOKEN} TWINE_REPOSITORY_URL: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi package:pypi: extends: .deploy_wheels variables: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${PYPI_TOKEN} rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' package:docker: stage: deploy extends: .docker_build variables: - IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG DOCKERFILE: Dockerfile script: + - '[ "$CI_COMMIT_REF_SLUG" = "$MAIN_BRANCH" ] && IMAGE="$CI_REGISTRY_IMAGE:latest" + || IMAGE="$IMAGE_TAG"' - docker pull $IMAGE || true - docker build --cache-from $IMAGE --build-arg USE_MPI=True -t $IMAGE -f $DOCKERFILE . - docker push $IMAGE diff --git a/CHANGELOG.md b/CHANGELOG.md index 909780e..63670ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,237 +1,246 @@ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) for final versions and [PEP440](https://www.python.org/dev/peps/pep-0440/) in case intermediate versions need to be released (e.g. development version `2.2.3.dev1` or release candidates `2.2.3rc1`), or individual commits are packaged. ## Unreleased ### Added - Now using `clang-format`, `clang-tidy` and `flake8` for linting +- Pre-built Tamaas images can be pulled with `docker pull + registry.gitlab.com/tamaas/tamaas` +- Added a `--version` flag to the `tamaas` command ### Changed - The root `Dockerfile` now compiles Tamaas, so using Tamaas in Docker is easier. - The model attributes dumped to Numpy files are now written in a JSON-formatted string to avoid unsafe loading/unpickling of objects. - Removed the `build_doc` build option: now the doc targets are automatically added if the dependencies are met, and built if `scons doc` is called. - Removed the `use_googletest` build option: if tests are built and gtest is present, the corresponding tests will be built + +### Fixed + +- The command `tamaas plot` gracefully fails if Matplotlib is missing +- Better heuristic to guess which fields are defined on boundary in dumpers + (still not perfect and may give false negatives) ## v2.3.0 -- 2021-06-15 ### Added - Added `read()` method to dumpers to create a model from a dump file - `getClusters()` can be called in MPI contact with partial contact maps - Added a JSON encoder class for models and a JSON dumper - CUDA compatibility is re-established, but has not been tested - Docstrings in the Python bindings for many classes/methods ### Changed - Tamaas version numbers are now managed by [versioneer](https://github.com/python-versioneer/python-versioneer). This means that Git tags prefixed with `v` (e.g. `v2.2.3`) carry meaning and determine the version. When no tag is set, versioneer uses the last tag, specifies the commit short hash and the distance to the last tag (e.g. `2.2.2+33.ge314b0e`). This version string is used in the compiled library, the `setup.py` script and the `__version__` variable in the python module. - Tamaas migrated to [GitLab](https://gitlab.com/tamaas/tamaas) - Continuous delivery has been implemented: - the `master` branch will now automatically build and publish Python wheels to `https://gitlab.com/api/v4/projects/19913787/packages/pypi/simple`. These "nightly" builds can be installed with: pip install \ --extra-index-url https://gitlab.com/api/v4/projects/19913787/packages/pypi/simple \ tamaas - version tags pushed to `master` will automatically publish the wheels to [PyPI](https://pypi.org/project/tamaas/) ### Deprecated - The `finalize()` function is now deprecated, since it is automatically called when the process terminates - Python versions 3.5 and below are not supported anymore ### Fixed - Fixed a host of dump read/write issues when model type was not `volume_*d`. Dumper tests are now streamlined and systematic. - Fixed a bug where `Model::solveDirichlet` would not compute correctly - Fixed a bug where `Statistics::contact` would not normalize by the global number of surface points ## v2.2.2 -- 2021-04-02 ### Added - Entry-point `tamaas` defines a grouped CLI for `examples/pipe_tools`. Try executing `tamaas surface -h` from the command-line! ### Changed - `CXXFLAGS` are now passed to the linker - Added this changelog - Using absolute paths for environmental variables when running `scons test` - Reorganized documentation layout - Gave the build system a facelift (docs are now generated directly with SCons instead of a Makefile) ### Deprecated - Python 2 support is discontinued. Version `v2.2.1` is the last PyPi build with a Python 2 wheel. - The scripts in `examples/pipe_tools` have been replaced by the `tamaas` command ### Fixed - `UVWDumper` no longer imports `mpi4py` in sequential - Compiling with different Thrust/FFTW backends ## v2.2.1 -- 2021-03-02 ### Added - Output registered fields and dumpers in `print(model)` - Added `operator[]` to the C++ model class (for fields) - Added `traction` and `displacement` properties to Python model bindings - Added `operators` property to Python model bindings, which provides a dict-like access to registered operators - Added `shape` and `spectrum` to properties to Python surface generator bindings - Surface generator constructor accepts surface global shape as argument - Choice of FFTW thread model ### Changed - Tests use `/tmp` for temporary files - Updated dependency versions (Thrust, Pybind11) ### Deprecated - Most `get___()` and `set___()` in Python bindings have been deprecated. They will generate a `DeprecationWarning`. ### Removed - All legacy code ## v2.2.0 -- 2020-12-31 ### Added - More accurate function for computation of contact area - Function to compute deviatoric of tensor fields - MPI implementation - Convenience `hdf5toVTK` function - Readonly properties `shape`, `global_shape`, `boundary_shape` on model to give shape information ### Changed - Preprocessor defined macros are prefixed with `TAMAAS_` - Moved `tamaas.to_voigt` to `tamaas.compute.to_voigt` ### Fixed - Warning about deprecated constructors with recent GCC versions - Wrong computation of grid strides - Wrong computation of grid sizes in views ## v2.1.4 -- 2020-08-07 ### Added - Possibility to generate a static `libTamaas` - C++ implementation of DFSANE solver - Allowing compilation without OpenMP ### Changed - NetCDF dumper writes frames to a single file ### Fixed - Compatibility with SCons+Python 3 ## v2.1.3 -- 2020-07-27 ### Added - Version number to `TamaasInfo` ### Changed - Prepending root directory when generating archive ## v2.1.2 -- 2020-07-24 This release changes some core internals related to discrete Fourier transforms for future MPI support. ### Added - Caching `CXXFLAGS` in SCons build - SCons shortcut to create code archive - Test of the elastic-plastic contact solver - Paraview data dumper (`.pvd` files) - Compression for UVW dumper - `__contains__` and `__iter__` Python bindings of model - Warning message of possible overflow in Kelvin ### Changed - Simplified `tamaas_info.cpp`, particularly the diff part - Using a new class `FFTEngine` to manage discrete Fourier transforms. Plans are re-used as much as possible with different data with the same shape. This is in view of future MPI developments - Redirecting I/O streams in solve functions so they can be used from Python (e.g. in Jupyter notebooks) - Calling `initialize()` and `finalize()` is no longer necessary ### Fixed - Convergence issue with non-linear solvers - Memory error in volume potentials ## v2.1.1 -- 2020-04-22 ### Added - SCons shortcut to run tests ### Fixed - Correct `RPATH` for shared libraries - Issues with SCons commands introduced in v2.1.0 - Tests with Python 2.7 ## v2.1.0 -- 2020-04-17 ### Added - SCons shortcuts to build/install Tamaas and its components - Selection of integration method for Kelvin operator - Compilation option to remove the legacy part of Tamaas - NetCDF dumper ### Fixed - Link bug with clang - NaNs in Kato saturated solver ## v2.0.0 -- 2019-11-11 First public release. Contains relatively mature elastic-plastic contact code. diff --git a/doc/sphinx/source/quickstart.rst b/doc/sphinx/source/quickstart.rst index 30a6a40..88d8e29 100644 --- a/doc/sphinx/source/quickstart.rst +++ b/doc/sphinx/source/quickstart.rst @@ -1,237 +1,237 @@ Quickstart ---------- Here is a quick introduction to get you started with Tamaas. Installation from PyPI ^^^^^^^^^^^^^^^^^^^^^^ If you have a Linux system, you can simply run ``python3 -m pip install tamaas``. This installs the latest stable version of Tamaas from `PyPI `_. You can get the latest cutting-edge build of Tamaas with: .. code-block:: bash python3 -m pip install \ --extra-index-url https://gitlab.com/api/v4/projects/19913787/packages/pypi/simple \ tamaas .. note:: To limit the number of statically linked dependencies in the wheel package, the builds that can be installed via PyPI or the GitLab package registery do not include parallelism or architecture specific optimizations. If you want to execute Tamaas in parallel, or want to improve performance, it is highly recommanded that you compile from source with the instructions below. Docker image ^^^^^^^^^^^^ Docker images are automatically built and pused to the Gitlab registry. To get the latest image simply run: .. code-block:: bash - docker pull registry.gitlab.com/tamaas/tamaas:master + docker pull registry.gitlab.com/tamaas/tamaas # to rename for convenience - docker tag registry.gitlab.com/tamaas/tamaas:master tamaas + docker tag registry.gitlab.com/tamaas/tamaas tamaas Then you can run scripts directly: .. code-block:: bash docker run -v $PWD:/app -t tamaas python3 /app/your_script.py # or docker run tamaas tamaas surface --sizes 16 16 --cutoffs 4 4 8 --hurst 0.8 | docker run -i tamaas tamaas contact 0.1 > results.npy The ``Dockerfile`` at the root of the Tamaas repository can be used to build an image containing Tamaas with a full set of dependencies and customize the build options. Simply run: .. code-block:: bash docker build -t tamaas . .. tip:: The following arguments can be passed to docker to customize the Tamaas build (with the ``--build-arg`` flag for ``docker build``): - ``BACKEND``: parallelism model for loops - ``FFTW_THREADS``: parallelism model for FFTW3 - ``USE_MPI``: compile an MPI-parallel version of Tamaas See below for explanations. A `singularity `_ container can be created from the docker image with: .. code-block:: bash - singularity build tamaas.sif docker://registry.gitlab.com/tamaas/tamaas:master + singularity build tamaas.sif docker://registry.gitlab.com/tamaas/tamaas Installation from source ^^^^^^^^^^^^^^^^^^^^^^^^ First make sure the following dependencies are installed for Tamaas: - a **C++ compiler** with full **C++14** and **OpenMP** support - **SCons** (python build system) - **FFTW3** - **thrust** (1.9.2+) - **boost** (pre-processor) - **python 3+** with **numpy** - **pybind11** (included as submodule) - **expolit** (included as submodule) Optional dependencies are: - an MPI implementation - **FFTW3** with MPI/threads/OpenMP (your pick) support - **scipy** (for nonlinear solvers) - **uvw**, **h5py**, **netCDF4** (for dumpers) - **googletest** and **pytest** (for tests) - **Doxygen** and **Sphinx** (for documentation) .. tip:: On a HPC environment, use the following variables to specify where the dependencies are located: - ``FFTW_ROOT`` - ``THRUST_ROOT`` - ``BOOST_ROOT`` - ``PYBIND11_ROOT`` - ``GTEST_ROOT`` .. note:: You can use the provided Dockerfile to build an image with the external dependencies installed. You should first clone the git repository with the submodules that are dependencies to tamaas (`pybind11 `_ and `googletest `_):: git clone --recursive https://gitlab.com/tamaas/tamaas.git The build system uses `SCons `_. In order to compile Tamaas with the default options:: scons After compiling a first time, you can edit the compilation options in the file ``build-setup.conf``, or alternatively supply the options directly in the command line: .. code-block:: bash scons option=value [...] To get a list of **all** build options and their possible values, you can run ``scons -h``. You can run ``scons -H`` to see the SCons-specific options (among them ``-j n`` executes the build with ``n`` threads and ``-c`` cleans the build). Note that the build is aware of the ``CXX`` and ``CXXFLAGS`` environment variables. Once compiled, you can install the python module with: .. code-block:: bash scons install prefix=/your/prefix The above command automatically calls ``pip(3)`` if it is installed, and falls back on ``setuptools`` otherwise. If you do not want to install Tamaas, you can use the following command:: scons dev This creates a symlink in ``~/.local/lib//site-packages`` and is equivalent to ``pip(3) install -e`` or ``./setup.py develop``. You can check that everything is working fine with: .. code-block:: bash python3 -c 'import tamaas; print(tamaas)' Important build options ....................... Here are a selected few important compilation options: ``build_type`` Controls the type of build, which essentially changes the optimisation level (``-O0`` for ``debug``, ``-O2`` for ``profiling`` and ``-O3`` for ``release``) and the amount of debug information. Default type is ``release``. Accepted values are: - ``release`` - ``debug`` - ``profiling`` ``CXX`` Compiler (uses the environment variable ``CXX`` by default). ``CXXFLAGS`` Compiler flags (uses the environment variable ``CXXFLAGS`` by default). Useful to add tuning flags (e.g. ``-march=native -mtune=native`` for GCC or ``-xHOST`` for Intel), or additional optimisation flags (e.g. ``-flto`` for link-time optimization). ``backend`` Controls the Thrust parallelism backend. Defaults to ``omp`` for OpenMP. Accepted values are: - ``omp``: OpenMP - ``cpp``: Sequential - ``tbb`` (unsupported): Threading Building Blocks ``fftw_threads`` Controls the FFTW thread model. Defaults to ``omp`` for OpenMP. Accepted values are: - ``omp``: OpenMP - ``threads``: PThreads - ``none``: Sequential ``use_mpi`` Activates MPI-parallelism (boolean value). More details on the above options can be found in :doc:`performance`. Building the docs ^^^^^^^^^^^^^^^^^ Documentation is built using ``scons doc``. Make sure to have the correct dependencies installed (they are already provided in the Docker image). Running the contact pipe tools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Once installed, the python package provides a `tamaas` command, which defines three subcommands that can be used to explore the mechanics of elastic rough contact: - ``surface`` generates randomly rough surfaces (see :doc:`rough_surfaces`) - ``contact`` solves an elastic contact problem with a surface read from ``stdin`` (see :doc:`contact`) - ``plot`` plots the surface tractions and displacements read from ``stdin`` Here's a sample command line for you to try out: .. code-block:: bash tamaas surface --cutoffs 2 4 64 --size 512 512 --hurst 0.8 | tamaas contact 1 | tamaas plot Check out the help of each command (e.g. ``tamaas surface -h``) for a description of the arguments. Running the tests ^^^^^^^^^^^^^^^^^ You need to activate the ``build_tests`` option to compile the tests: .. code-block:: bash scons build_tests=True Tests can then be run with the ``scons test`` command. Make sure you have `pytest `_ installed.