diff --git a/blackdynamite/.dockerignore b/blackdynamite/.dockerignore new file mode 100644 index 0000000..c1021cc --- /dev/null +++ b/blackdynamite/.dockerignore @@ -0,0 +1,4 @@ +data +notebooks +.renku +.git diff --git a/blackdynamite/.gitignore b/blackdynamite/.gitignore new file mode 100644 index 0000000..55c37d3 --- /dev/null +++ b/blackdynamite/.gitignore @@ -0,0 +1,339 @@ + +# Created by https://www.gitignore.io/api/macos,python,R,linux,vim,emacs,visualstudiocode,intellij +# Edit at https://www.gitignore.io/?templates=macos,python,R,linux,vim,emacs,visualstudiocode,intellij + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +# JetBrains templates +**___jb_tmp___ + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Linux ### + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history + +# End of https://www.gitignore.io/api/macos,python,R,linux,vim,emacs,visualstudiocode,intellij + +# Renku +.renku.lock +.renku/tmp +.renku/cache +.bd \ No newline at end of file diff --git a/blackdynamite/.gitlab-ci.yml b/blackdynamite/.gitlab-ci.yml new file mode 100644 index 0000000..910df3b --- /dev/null +++ b/blackdynamite/.gitlab-ci.yml @@ -0,0 +1,17 @@ +variables: + GIT_STRATEGY: fetch + GIT_SSL_NO_VERIFY: "true" + GIT_LFS_SKIP_SMUDGE: 1 + +stages: + - build + +image_build: + stage: build + image: docker:stable + before_script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY + script: | + CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7) + docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 . + docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 diff --git a/blackdynamite/.renku/renku.ini b/blackdynamite/.renku/renku.ini new file mode 100644 index 0000000..5a3d87d --- /dev/null +++ b/blackdynamite/.renku/renku.ini @@ -0,0 +1,2 @@ +[renku "interactive"] +default_url = /lab diff --git a/blackdynamite/.renkulfsignore b/blackdynamite/.renkulfsignore new file mode 100644 index 0000000..63b50a9 --- /dev/null +++ b/blackdynamite/.renkulfsignore @@ -0,0 +1,23 @@ +# WARNING: editing this file will impact the behavior of renku commands +# by telling renku to *not track* listed files with git LFS. +# +# By default, renku commands track all files above the renku configured lfs_threshold +# with git LFS to prevent accidentally checking in large files (default=100kb). +# You can update this threshold by calling 'renku config lfs_threshold kb'. +# +# Use this file to specify particular paths to *not track* with LFS regardless of +# their size. The syntax is the same as .gitignore (https://git-scm.com/docs/gitignore). +# Remember: files listed here, when generated during renku commands, will add to +# the size of your repo, and could slow things down. +# +# See https://github.com/SwissDataScienceCenter/renku-project-template for proper +# usage. + +# Warning: removing these lines will check source files (e.g. generated ipynb files from papermill) into LFS and they +# will no longer be displayed in the renku UI +*.ipynb +*.py +*.r +*.md +*.rst +.bd \ No newline at end of file diff --git a/blackdynamite/Dockerfile b/blackdynamite/Dockerfile new file mode 100644 index 0000000..47a1c4d --- /dev/null +++ b/blackdynamite/Dockerfile @@ -0,0 +1,50 @@ +# For finding latest versions of the base image see +# https://github.com/SwissDataScienceCenter/renkulab-docker +ARG RENKU_BASE_IMAGE=renku/renkulab-py:3.9-0.12.0 +FROM ${RENKU_BASE_IMAGE} + +# Uncomment and adapt if code is to be included in the image +# COPY src /code/src + +# Uncomment and adapt if your R or python packages require extra linux (ubuntu) software +# e.g. the following installs apt-utils and vim; each pkg on its own line, all lines +# except for the last end with backslash '\' to continue the RUN line +# +# USER root +# RUN apt-get update && \ +# apt-get install -y --no-install-recommends \ +# apt-utils \ +# vim +# USER ${NB_USER} + +# install the python dependencies +COPY requirements.txt environment.yml /tmp/ +RUN conda env update -q -f /tmp/environment.yml && \ + /opt/conda/bin/pip install -r /tmp/requirements.txt && \ + conda clean -y --all && \ + conda env export -n "root" + +# RENKU_VERSION determines the version of the renku CLI +# that will be used in this image. To find the latest version, +# visit https://pypi.org/project/renku/#history. +ARG RENKU_VERSION={{ __renku_version__ | default("1.6.0") }} + +######################################################## +# Do not edit this section and do not add anything below + +# Install renku from pypi or from github if it's a dev version +RUN if [ -n "$RENKU_VERSION" ] ; then \ + source .renku/venv/bin/activate ; \ + currentversion=$(renku --version) ; \ + if [ "$RENKU_VERSION" != "$currentversion" ] ; then \ + pip uninstall renku -y ; \ + gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(rc[[:digit:]]\+\)*\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\4/p") ; \ + if [ -n "$gitversion" ] ; then \ + pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\ + else \ + pip install --force renku==${RENKU_VERSION} ;\ + fi \ + fi \ + fi + +######################################################## diff --git a/blackdynamite/README.md b/blackdynamite/README.md new file mode 100644 index 0000000..8467cc1 --- /dev/null +++ b/blackdynamite/README.md @@ -0,0 +1,12 @@ +# {{ name }} +{% if __project_description__ %} +{{ __project_description__ }} +{% endif %} +## Introduction + +This is a BlackDynamite project (Using Renku) + +## Working with the project + +To be written + diff --git a/blackdynamite/bd.yaml b/blackdynamite/bd.yaml new file mode 100644 index 0000000..127c38e --- /dev/null +++ b/blackdynamite/bd.yaml @@ -0,0 +1,25 @@ +--- + +study: bd_study + +job: + param1: int + param2: float + param3: str + +run: + compiler: str + +job_space: + param1: 10 + param2: [3.14, 1., 2.] + param3: 'toto' + +run_space: + compiler: 'gcc' + +config_files: + - doIt.py + +exec_file: launch.sh + diff --git a/blackdynamite/data/.gitkeep b/blackdynamite/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/blackdynamite/doIt.py b/blackdynamite/doIt.py new file mode 100644 index 0000000..30a911a --- /dev/null +++ b/blackdynamite/doIt.py @@ -0,0 +1,31 @@ +#!/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import BlackDynamite as BD +import numpy as np + +myrun, myjob = BD.getRunFromScript() + +print(myjob) + +myrun.start() + +for step in range(0, 10): + _quantity = myrun.id*step + myrun.pushScalarQuantity(_quantity, step, "ekin", is_integer=False) + myrun.pushScalarQuantity(_quantity*2, step, "epot", is_integer=False) + myrun.pushVectorQuantity( + np.random.random(10), step, "random_vect", is_integer=False) + +myrun.finish() diff --git a/blackdynamite/launch.sh b/blackdynamite/launch.sh new file mode 100644 index 0000000..f3f0b57 --- /dev/null +++ b/blackdynamite/launch.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo 'here is the job' +echo __BLACKDYNAMITE__id__ +echo __BLACKDYNAMITE__param1__ +echo __BLACKDYNAMITE__param2__ +echo __BLACKDYNAMITE__param3__ + +python3 ./doIt.py diff --git a/blackdynamite/notebooks/.gitkeep b/blackdynamite/notebooks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/blackdynamite/post_treatment.py b/blackdynamite/post_treatment.py new file mode 100755 index 0000000..07fca53 --- /dev/null +++ b/blackdynamite/post_treatment.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +################################################################ +import BlackDynamite as BD +import matplotlib.pyplot as plt +################################################################ +# basic connection +parser = BD.BDParser() +params = parser.parseBDParameters() +mybase = BD.Base(**params) +################################################################ + + +# function to plot things (user's job) +def plot(run_list): + for r, j in run_list: + step, ekin = r.getScalarQuantity('ekin') + if ekin is None: + continue + print(j) + list_files = r.listFiles() + print(list_files) + fname = r.getFile(list_files[3]) + print(fname + ':') + _file = open(fname) + print(_file.read()) + plt.plot(step, ekin, 'o-', + label='$p_2 = {0}$'.format(j['param2'])) + + vect = r.getVectorQuantity('random_vect', 1) + print(vect) + plt.legend(loc='best') + plt.show() +################################################################ + + +# selecting some runs +runSelector = BD.RunSelector(mybase) +run_list = runSelector.selectRuns(params) +plot(run_list) + +# selecting some other runs +params['constraints'] = ['run_name ~ test', 'state = FINISHED', 'param2 > 1'] +run_list = runSelector.selectRuns(params) +plot(run_list) diff --git a/blackdynamite/requirements.txt b/blackdynamite/requirements.txt new file mode 100644 index 0000000..6c9cc72 --- /dev/null +++ b/blackdynamite/requirements.txt @@ -0,0 +1 @@ +--extra-index-url https://gitlab.com/ganciaux/blackdynamite.git blackdynamite