diff --git a/slides.ipynb b/slides.ipynb index 86c007a..3299a28 100644 --- a/slides.ipynb +++ b/slides.ipynb @@ -1,1494 +1,1509 @@ { "cells": [ { "cell_type": "markdown", "metadata": { "ein.tags": "worksheet-0", "slideshow": { "slide_type": "slide" } }, "source": [ "# So, you want to go open Science ?\n", "## Version, colaborate, share, test and document !\n", "\n", "### Agenda\n", "- Git in 30 seconds\n", "- The GitFlow methodology\n", "- Git remotes and having public/public part of a repository\n", "- Github/Gitlab pages, C4science wiki\n", "- Continuous testing, integration and deployment (CI/CD)\n", "\n", "### Resources\n", "- Slides \n", "- Handout \n", "\n", "### Credits\n", "Nicolas Richart and Jean-Baptiste Aubort - https://scitas.epfl.ch" ] }, { "cell_type": "markdown", "metadata": { "ein.tags": "worksheet-0", "slideshow": { "slide_type": "slide" } }, "source": [ "## Git in 30 seconds" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": { "autoscroll": false, "ein.hycell": false, "ein.tags": "worksheet-0", "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "cd ..\n", "rm -rf so-workshop-test" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": { "autoscroll": false, "ein.hycell": false, "ein.tags": "worksheet-0", "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into 'so-workshop-test'...\n", - "remote: Counting objects: 64, done.\u001b[K\n", - "remote: Compressing objects: 100% (39/39), done.\u001b[K\n", - "remote: Total 64 (delta 25), reused 49 (delta 15)\u001b[K\n", - "Receiving objects: 100% (64/64), 6.17 KiB | 0 bytes/s, done.\n", - "Resolving deltas: 100% (25/25), done.\n" + "remote: Counting objects: 67, done.\u001b[K\n", + "remote: Compressing objects: 100% (42/42), done.\u001b[K\n", + "remote: Total 67 (delta 27), reused 49 (delta 15)\u001b[K\n", + "Receiving objects: 100% (67/67), 6.40 KiB | 0 bytes/s, done.\n", + "Resolving deltas: 100% (27/27), done.\n" ] } ], "source": [ + "# Clone the repository we want to work on\n", + "\n", "git clone ssh://git@c4science.ch/source/so-workshop-test.git\n", "cd so-workshop-test" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "On branch master\n", "Your branch is up-to-date with 'origin/master'.\n", "Changes not staged for commit:\n", " (use \"git add ...\" to update what will be committed)\n", " (use \"git checkout -- ...\" to discard changes in working directory)\n", "\n", "\t\u001b[31mmodified: README\u001b[m\n", "\n", "no changes added to commit (use \"git add\" and/or \"git commit -a\")\n" ] } ], "source": [ + "# Make a change to the REAME file.\n", + "# The git status command is the most important one there, run this command everytime your\n", + "# want to know what to do. Git will usually tell you the next steps.\n", + "\n", "echo 'My Change' >> README\n", "git status" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[master 4f6ea84] My change summary\n", - " 1 file changed, 3 insertions(+)\n" + "[master 8fcf724] My change summary\n", + " 1 file changed, 1 insertion(+)\n" ] } ], "source": [ + "# A change must be added to the list of changes (the index) before being commited.\n", + "# A commit is a snapshot of the changes you added, it consist of a message, a date and an author.\n", + "# It is represented by a SHA-1 hash which we can see 7 bytes of here.\n", + "\n", "git add README\n", "git commit -m \"My change summary\"" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "On branch master\n", "Your branch is ahead of 'origin/master' by 1 commit.\n", " (use \"git push\" to publish your local commits)\n", "nothing to commit, working tree clean\n" ] } ], "source": [ + "# Again, this command is your friend.\n", + "\n", "git status" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counting objects: 3, done.\n", "Delta compression using up to 4 threads.\n", "Compressing objects: 100% (3/3), done.\n", - "Writing objects: 100% (3/3), 310 bytes | 0 bytes/s, done.\n", + "Writing objects: 100% (3/3), 300 bytes | 0 bytes/s, done.\n", "Total 3 (delta 2), reused 0 (delta 0)\n", "To ssh://c4science.ch/source/so-workshop-test.git\n", - " f46883a..4f6ea84 master -> master\n" + " 4f6ea84..8fcf724 master -> master\n" ] } ], "source": [ + "# When we are happy about the changes we made,\n", + "# we push them on the remote server.\n", + "\n", "git push" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "On branch master\n", "Your branch is up-to-date with 'origin/master'.\n", "nothing to commit, working tree clean\n" ] } ], "source": [ "git status" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## The GitFlow methodology" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Github/Gitlab pages, C4science wiki" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Adding a public version of the code" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "From github.com:epfl-scitas/so-workshop\n", " * [new branch] master -> public/master\n" ] } ], "source": [ "git remote add public git@github.com:epfl-scitas/so-workshop.git\n", "git fetch public" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Branch 'public' set up to track remote branch 'master' from 'public'.\n", "Switched to a new branch 'public'\n" ] } ], "source": [ "git checkout -b public public/master" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Squash commit -- not updating HEAD\n", "Automatic merge went well; stopped before committing as requested\n" ] } ], "source": [ "git merge --squash --allow-unrelated-histories master" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "scrolled": false, "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[public 46ccdaa] First public revision\n", " 5 files changed, 32 insertions(+)\n", " create mode 100644 .gitignore\n", " create mode 100644 README\n", " create mode 100644 code/mylib.py\n", " create mode 100644 new_feature\n", " create mode 100644 new_file\n", "Enumerating objects: 9, done.\n", "Counting objects: 100% (9/9), done.\n", "Delta compression using up to 4 threads\n", "Compressing objects: 100% (5/5), done.\n", "Writing objects: 100% (8/8), 885 bytes | 885.00 KiB/s, done.\n", "Total 8 (delta 0), reused 2 (delta 0)\n", "To github.com:epfl-scitas/so-workshop.git\n", " eaf9eaf..46ccdaa public -> master\n", "Branch 'public' set up to track remote branch 'master' from 'public'.\n" ] } ], "source": [ "git commit -m \"First public revision\"\n", "git push -u public public:master" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "slideshow": { "slide_type": "fragment" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "* \u001b[33mcommit 46ccdaa253a2960c2c8cb77dea0885b232d0044a\u001b[m\u001b[33m (\u001b[m\u001b[1;36mHEAD -> \u001b[m\u001b[1;32mpublic\u001b[m\u001b[33m, \u001b[m\u001b[1;31mpublic/master\u001b[m\u001b[33m\u001b[m\u001b[33m\u001b[m\u001b[33m\u001b[m\u001b[1;36m\u001b[m\u001b[1;32m\u001b[m\u001b[33m\u001b[m\u001b[1;31m\u001b[m\u001b[33m)\u001b[m\u001b[m\n", "\u001b[31m|\u001b[m Author: Nicolas Richart \u001b[m\n", "\u001b[31m|\u001b[m Date: Mon Dec 3 15:38:14 2018 +0100\u001b[m\n", "\u001b[31m|\u001b[m \u001b[m\n", "\u001b[31m|\u001b[m First public revision\u001b[m\n", "\u001b[31m|\u001b[m \u001b[m\n", "* \u001b[33mcommit eaf9eaf6260bfda406becd0adb39cc90474b3691\u001b[m\u001b[m\n", " Author: Nicolas Richart \u001b[m\n", " Date: Mon Dec 3 15:37:16 2018 +0100\u001b[m\n", " \u001b[m\n", " Create README.md\u001b[m\n" ] } ], "source": [ "git log --graph -n3" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# Continuous Integration (CI)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "fragment" } }, "source": [ "## What is CI ?\n", "\n", "- Automation of build and tests\n", "- Run tests at each commit\n", "- You need to write tests !" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "fragment" } }, "source": [ "## Why use CI ?\n", "\n", "- Detect bugs when they are introduced\n", "- Detect when fixed bugs appear again (regression)\n", "- Published code is tested and should be working\n", "- Maintain code quality\n", " - Linters\n", " - Test Coverage" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# Jenkins on c4science" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## \n", "Jenkins jobs are created from the c4science web interface\n", "![c4s jenkinsjob](images/c4s-jenkinsjob.png)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "## \n", "You can have more information about the job by going to the Jenkins standard page\n", "![c4s jenkins](images/c4s-jenkins.png)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# Example\n", "## Jenkins base" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "cd ../so-workshop" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Branch feature/jenkins-base set up to track remote branch feature/jenkins-base from origin.\n", "Switched to a new branch 'feature/jenkins-base'\n", "\u001b[01;34m.\u001b[00m\n", "├── \u001b[01;34mcode\u001b[00m\n", "│   ├── mylib.py\n", "│   └── \u001b[01;32mtest.py\u001b[00m\n", "├── Jenkinsfile\n", "└── README\n", "\n", "1 directory, 4 files\n" ] } ], "source": [ "# List of the Jenkins example files\n", "git checkout feature/jenkins-base\n", "tree" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "#!/usr/bin/env python3\n", "\n", "class MyLib():\n", "\n", " def __init__(self):\n", " pass\n", "\n", " def MyFunc(self):\n", " return 'test'\n", "\n", "if __name__ == '__main__':\n", " pass\n" ] } ], "source": [ "# Here a simple python class with 2 methods\n", "cat code/mylib.py" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "#!/usr/bin/env python3\n", "import unittest, pytest\n", "import mylib\n", "\n", "class Test(unittest.TestCase):\n", "\n", " def test_return(self):\n", " m = mylib.MyLib()\n", " self.assertTrue(m.MyFunc() == 'test')\n", "\n", "if __name__ == '__main__':\n", " pytest.main(['./test.py'])\n" ] } ], "source": [ "# We create a unit test file that check the return value of the MyFunc() function\n", "cat code/test.py" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pipeline {\n", "\n", " agent {\n", " docker {\n", " image 'python:3.7'\n", " }\n", " }\n", "\n", " environment {\n", " HOME = \"$WORKSPACE\"\n", " }\n", "\n", " stages {\n", " stage('install dependencies') {\n", " steps {\n", " sh 'pip3 install --user pytest'\n", " }\n", " }\n", " stage('run tests') {\n", " steps {\n", " sh '$HOME/.local/bin/pytest code/test.py'\n", " }\n", " }\n", " }\n", "\n", "}\n" ] } ], "source": [ "# Here's the Jenkins job defition. We use the python docker image in which\n", "# we will run pytest on our previous unit test file\n", "cat Jenkinsfile" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "And here's the result job on the Jenkins Blue web interface\n", "![jenkins base](images/jenkins-base.png)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# Example\n", "## Jenkins dockerfile" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Branch feature/jenkins-dockerfile set up to track remote branch feature/jenkins-dockerfile from origin.\n", "Switched to a new branch 'feature/jenkins-dockerfile'\n", "\u001b[01;34m.\u001b[00m\n", "├── \u001b[01;34mcode\u001b[00m\n", "│   ├── mylib.py\n", "│   └── \u001b[01;32mtest.py\u001b[00m\n", "├── Dockerfile\n", "├── Jenkinsfile\n", "└── README\n", "\n", "1 directory, 5 files\n" ] } ], "source": [ "git checkout feature/jenkins-dockerfile\n", "tree" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FROM python:3.7\n", "RUN pip3 install --no-cache-dir pytest\n" ] } ], "source": [ "# Instead of using a pre-made python docker image, we create our own image based on the same image.\n", "# We customize the image by directly installing pytest\n", "cat Dockerfile" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pipeline {\n", "\n", " agent {\n", " dockerfile true \n", " }\n", "\n", " stages {\n", " stage('run tests') {\n", " steps {\n", " sh 'pytest code/test.py'\n", " }\n", " }\n", " }\n", "\n", "}\n" ] } ], "source": [ "# We can remove the pytest installation from the previous example as\n", "# it's now integrated into the Docker image\n", "cat Jenkinsfile" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Example\n", "### Jenkins junit" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Branch feature/jenkins-junit set up to track remote branch feature/jenkins-junit from origin.\n", "Switched to a new branch 'feature/jenkins-junit'\n", "\u001b[01;34m.\u001b[00m\n", "├── \u001b[01;34mcode\u001b[00m\n", "│   ├── mylib.py\n", "│   └── \u001b[01;32mtest.py\u001b[00m\n", "├── Dockerfile\n", "├── Jenkinsfile\n", "└── README\n", "\n", "1 directory, 5 files\n" ] } ], "source": [ "git checkout feature/jenkins-junit\n", "tree" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pipeline {\n", "\n", " agent {\n", " dockerfile true \n", " }\n", "\n", " stages {\n", " stage('run tests') {\n", " steps {\n", " sh 'pytest --junitxml results.xml code/test.py'\n", " }\n", " }\n", " }\n", "\n", " post {\n", " always {\n", " junit 'results.xml'\n", " }\n", " }\n", "\n", "}\n" ] } ], "source": [ "# We can export the pytest result in a standard junit format that Jenkins can read\n", "cat Jenkinsfile" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "The result are visible in the web interface\n", "![jenkins junit](images/jenkins-junit.png)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "![jenkins junit 2](images/jenkins-junit2.png)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# C4science\n", "## Arcanist\n", "Command line tool to interact with c4science\n", "- Code review\n", "```\n", "arc diff\n", "arc list\n", "arc cover\n", "arc patch\n", "arc land\n", "```\n", "- Code quality\n", "```\n", "arc lint\n", "arc unit\n", "```\n", "- Other tools\n", "```\n", "arc call-conduit\n", "arc upload\n", "arc download\n", "arc paste\n", "arc anoid\n", "```" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " \u001b[35mNeeds Review\u001b[39m \u001b[1mD73:\u001b[m Add Wikimedia Sprint extension\n", " \u001b[35mNeeds Review\u001b[39m \u001b[1mD86:\u001b[m Add mysql logs to rsyslog\n", " \u001b[35mNeeds Review\u001b[39m \u001b[1mD130:\u001b[m Enable automation on all repositories\n", " \u001b[35mNeeds Review\u001b[39m \u001b[1mD151:\u001b[m Separate name and hashtag for Project allowing to have multiple projects with the same name\n", " \u001b[35mNeeds Review\u001b[39m \u001b[1mD200:\u001b[m Every command run using ExecFuture use a custom cgroup 'future'\n", " \u001b[35mNeeds Review\u001b[39m \u001b[1mD209:\u001b[m Migrate from GlusterFS to native Phabricator clustered repo\n" ] } ], "source": [ "arc list" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " F10024657 hello: https://c4science.ch/F10024657\n", "\n", "Done.\n", "Getting file information...\n", "Downloading file 'hello' (6 bytes)...\n", "Saved file as '/tmp/hello-new'.\n", "P6: https://c4science.ch/P6\n" ] } ], "source": [ "echo hello > /tmp/hello\n", "arc upload --temporary /tmp/hello\n", "arc download --as /tmp/hello-new F10024647\n", "cat /tmp/hello | arc paste" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[1mT1411\u001b[m Huge repositories and push \u001b[47m \u001b[49m Unbreak Now! \u001b[42m \u001b[49m Open\n", "\u001b[1mT2377\u001b[m c4science Winter Update \u001b[47m \u001b[49m Unbreak Now! \u001b[42m \u001b[49m Open\n", "\u001b[1mT2143\u001b[m C4science: One of your repository is too big \u001b[47m \u001b[49m Unbreak Now! \u001b[42m \u001b[49m Open\n", "\u001b[1mT2027\u001b[m Cannot download binary file from web browser interface \u001b[47m \u001b[49m Unbreak Now! \u001b[42m \u001b[49m Open\n", "\u001b[1mT1477\u001b[m Couldn't set refs/heads/master - failed to write \u001b[47m \u001b[49m Unbreak Now! \u001b[42m \u001b[49m Open\n", "\u001b[1mT1882\u001b[m Externals can't view Conpherence room \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT1607\u001b[m Download raw files \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT1531\u001b[m SSH access for Administrator fatal \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT2320\u001b[m fatal: premature end of pack file, XX bytes missing \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT2386\u001b[m Test OOMd from Facebook instea of stock OOMKiller \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT2002\u001b[m Pygmentize stuck at CPU 100% for >60h \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT1485\u001b[m Repository don't appear in the new Home \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT1079\u001b[m Realtime replication for Gluster and MySQL \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT816\u001b[m Leaving university keeps the user in Creators group \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT666\u001b[m MySQL database table schema need update \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT2317\u001b[m Problem with fs load leads to filesystem deconnection \u001b[41m \u001b[49m High \u001b[42m \u001b[49m Open\n", "\u001b[1mT1790\u001b[m Remove users when Switch AAI account is not valid anymore \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1699\u001b[m Add image in README.md on c4science \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1696\u001b[m Automatic badges \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1597\u001b[m Upload file deletes typed issue description \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1525\u001b[m Better project page \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1519\u001b[m Rename All Users (Creators) to Swiss Universities \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1476\u001b[m warning: remote HEAD refers to nonexistent ref, unable to checkout. \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1310\u001b[m Escape closes new issue \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2131\u001b[m C4science: One of your repository is too big \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2212\u001b[m Images in readme \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2384\u001b[m Project members of child project cannot edit \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2334\u001b[m List repositories and access fields via web request \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2322\u001b[m Many tags \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1975\u001b[m .pdf files in SVN repository \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1099\u001b[m Faire une annonce sur les écrans polynex \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT1024\u001b[m Utilisation de Jenkins en remote \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2106\u001b[m C4science: One of your repository is too big \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT948\u001b[m Error message for Shibboleth authentication \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT945\u001b[m Problem with Herald rules \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT919\u001b[m Allow All Users to create Bot user \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT750\u001b[m segmentation fault when arc install-certificate \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT632\u001b[m Plugins in jenkins, `xUnit` `Warnings` \u001b[41m \u001b[49m Normal \u001b[42m \u001b[49m Open\n", "\u001b[1mT2223\u001b[m No markdown rendering in Jupyter Notebooks \u001b[43m \u001b[49m Low \u001b[42m \u001b[49m Open\n", "\u001b[1mT1514\u001b[m Edit a file as admin fatal with Undefined variable: can_manage \u001b[43m \u001b[49m Low \u001b[42m \u001b[49m Open\n", "\u001b[1mT1504\u001b[m Uninstall Owners and Packages \u001b[43m \u001b[49m Low \u001b[42m \u001b[49m Open\n", "\u001b[1mT1437\u001b[m Some items in mgmt panels of a repo are greyed \u001b[43m \u001b[49m Low \u001b[42m \u001b[49m Open\n", "\u001b[1mT1120\u001b[m Remove Spaces \u001b[43m \u001b[49m Low \u001b[42m \u001b[49m Open\n", "\u001b[1mT969\u001b[m Markdown doesn't render nested blockquotes \u001b[43m \u001b[49m Low \u001b[42m \u001b[49m Open\n", "\u001b[1mT1731\u001b[m Cannot tag a repository \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT1571\u001b[m Better visibility for wiki pages \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT1480\u001b[m Is it possible to render md files that are not called README? \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT1459\u001b[m Project global namespace \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT1375\u001b[m Allow downloading code directly from the web interface \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT2486\u001b[m GitHub API or something similar \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT1231\u001b[m Rendering of arbitrary markdown files within repos \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT754\u001b[m subprojects and member inheritance \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT333\u001b[m Repo statistics \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT311\u001b[m Markdown links for local files not interpreted \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n", "\u001b[1mT307\u001b[m Show tags/branches in repo history \u001b[46m \u001b[49m Wishlist \u001b[42m \u001b[49m Open\n" ] } ], "source": [ "arc tasks" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# C4science\n", "## Arcanist\n", "### Linters\n", "\n", "> A linter or lint refers to tools that analyze source code to flag programming errors, bugs, stylistic errors, and suspicious constructs.\n", "\n", "- \n", "- " ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Branch feature/arcanist-linter set up to track remote branch feature/arcanist-linter from origin.\n", "Switched to a new branch 'feature/arcanist-linter'\n", "\u001b[01;34m.\u001b[00m\n", "├── .arclint\n", "├── \u001b[01;34mcode\u001b[00m\n", "│   ├── mylib.py\n", "│   └── \u001b[01;32mtest.py\u001b[00m\n", "├── Dockerfile\n", "├── .gitignore\n", "├── Jenkinsfile\n", "└── README\n", "\n", "1 directory, 7 files\n" ] } ], "source": [ "git checkout feature/arcanist-linter\n", "tree -a -I \".git\"" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[42m\u001b[1m CONFIGURED \u001b[m\u001b[49m \u001b[1mtext\u001b[m (Basic Text Linter)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mcsharp\u001b[m (C#)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mcpplint\u001b[m (C++ Google's Styleguide)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mcppcheck\u001b[m (C++ linter)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mlessc\u001b[m (CSS pre-processor)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mcsslint\u001b[m (CSSLint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mchmod\u001b[m (Chmod)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mgjslint\u001b[m (Closure Linter)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mcoffeelint\u001b[m (CoffeeLint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mcomposer\u001b[m (Composer Dependency Manager)\n", "\u001b[42m\u001b[1m CONFIGURED \u001b[m\u001b[49m \u001b[1mfilename\u001b[m (Filename)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mgenerated\u001b[m (Generated Code)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mgolint\u001b[m (Golint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mhlint\u001b[m (Haskell Linter)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mjsonlint\u001b[m (JSON Lint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mjson\u001b[m (JSON Lint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mjscs\u001b[m (JavaScript Code Style)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mjshint\u001b[m (JavaScript error checking)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mnolint\u001b[m (Lint Disabler)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mmerge-conflict\u001b[m (Merge Conflicts)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mphpcs\u001b[m (PHP_CodeSniffer)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mphutil-library\u001b[m (Phutil Library Linter)\n", "\u001b[42m\u001b[1m CONFIGURED \u001b[m\u001b[49m \u001b[1mpylint\u001b[m (PyLint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mflake8\u001b[m (Python Flake8 multi-linter)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mpep8\u001b[m (Python PEP 8)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mpyflakes\u001b[m (Python PyFlakes)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mruby\u001b[m (Ruby)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mrubocop\u001b[m (Ruby static code analyzer)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mscript-and-regex\u001b[m (Script and Regex)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mxml\u001b[m (SimpleXML Linter)\n", "\u001b[42m\u001b[1m CONFIGURED \u001b[m\u001b[49m \u001b[1mspelling\u001b[m (Spellchecker)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mxhpast\u001b[m (XHPAST Lint)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mphp\u001b[m (php -l)\n", "\u001b[43m\u001b[1m AVAILABLE \u001b[m\u001b[49m \u001b[1mpuppet-lint\u001b[m (puppet-lint)\n", "(Run `arc linters --verbose` for more details.)\n" ] } ], "source": [ "# Integrated linters present in Arcanist. You can also add your own.\n", "arc linters" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"linters\": {\n", " \"spelling-linter\": {\n", " \"type\": \"spelling\"\n", " },\n", " \"filename-linter\": {\n", " \"type\": \"filename\"\n", " },\n", " \"text-linter\": {\n", " \"type\": \"text\"\n", " },\n", " \"python-checks\": {\n", " \"type\": \"pylint\",\n", " \"include\": \"(\\\\.py$)\"\n", " }\n", " }\n", "}\n" ] } ], "source": [ "# Let's configure some linters for our project\n", "cat .arclint" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Jenkinsfile:4:Auto-Fix: Trailing Whitespace\n", "README:3:Auto-Fix: Trailing Whitespace at EOF\n", "code/mylib.py:1:Advice: Missing Docstring\n", "code/mylib.py:3:Advice: Missing Docstring\n", "code/mylib.py:3:Advice: Old Style Class\n", "code/mylib.py:3:Advice: Too Few Public Methods\n", "code/mylib.py:8:Advice: Invalid Name\n", "code/mylib.py:8:Advice: Missing Docstring\n", "code/mylib.py:8:Advice: No Self Use\n", "code/mylib.py:11:Warning: Mixed Indentation\n", "code/mylib.py:11:Error: Tab Literal\n", "code/mylib.py:11:Warning: Unreachable\n", "code/mylib.py:11:Advice: Invalid Name\n", "code/mylib.py:11:Advice: Missing Docstring\n", "code/mylib.py:11:Warning: Unused Variable\n", "code/mylib.py:11:Warning: Unused Argument\n", "code/mylib.py:12:Warning: Mixed Indentation\n", "code/test.py:1:Advice: Missing Docstring\n", "code/test.py:5:Advice: Missing Docstring\n", "code/test.py:9:Advice: Missing Docstring\n", "code/test.py:12:Advice: Missing Docstring\n", "code/test.py:13:Error: No Member\n" ] }, { "ename": "", "evalue": "2", "output_type": "error", "traceback": [] } ], "source": [ "# Run the linters on all the files (--everything)\n", "arc lint --never-apply-patches --output summary --everything" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# C4science\n", "## Arcanist\n", "### Unit-Tests\n", "Using the TAP (Test Anything Protocol) extension of Arcanist, we can report test units result to c4science" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Switched to branch 'feature/arcanist-unittests'\n", "Your branch is up to date with 'origin/feature/arcanist-unittests'.\n", "Submodule path '.arcanist-extensions': checked out 'b89c749a90154676967a1601b62563fed0cbd572'\n", "\u001b[01;34m.\u001b[00m\n", "├── .arcconfig\n", "├── .arclint\n", "├── \u001b[01;34mcode\u001b[00m\n", "│   ├── mylib.py\n", "│   └── \u001b[01;32mtest.py\u001b[00m\n", "├── Dockerfile\n", "├── .gitignore\n", "├── .gitmodules\n", "├── Jenkinsfile\n", "└── README\n", "\n", "1 directory, 9 files\n" ] } ], "source": [ "git checkout feature/arcanist-unittests\n", "git submodule update --init\n", "tree -a -I \".git|.arcanist-extensions\"" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"load\": [\n", " \".arcanist-extensions/tap_test_engine\"\n", " ],\n", "\n", " \"unit.engine\": \"TAPTestEngine\",\n", " \"unit.engine.tap.command\": \"pytest --tap-stream code/test.py\"\n", "}\n" ] } ], "source": [ "# Add a custom arcanist unit test engine\n", "cat .arcconfig" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ok 1 - Test.test_capitalize\n", "not ok 2 - Test.test_capitalize_fail\n", "# \n", "# self = \n", "# \n", "# def test_capitalize_fail(self):\n", "# > self.assertTrue(self.m.Capitalize('TeSt') == 'test')\n", "# E AssertionError: False is not true\n", "# \n", "# code/test.py:16: AssertionError\n", "ok 3 - Test.test_return\n", "1..3\n" ] }, { "ename": "", "evalue": "1", "output_type": "error", "traceback": [] } ], "source": [ "# Run pytest and export the result as TAP\n", "pytest --tap-stream ./code/test.py" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ok 1 - Test.test_capitalize\n", "not ok 2 - Test.test_capitalize_fail\n", "# \n", "# self = \n", "# \n", "# def test_capitalize_fail(self):\n", "# > self.assertTrue(self.m.Capitalize('TeSt') == 'test')\n", "# E AssertionError: False is not true\n", "# \n", "# code/test.py:16: AssertionError\n", "ok 3 - Test.test_return\n", "1..3\n", " \u001b[42m\u001b[1m PASS \u001b[m\u001b[49m \u001b[32m <1ms\u001b[39m\u001b[33m★\u001b[39m Test.test_capitalize\n", " \u001b[41m\u001b[1m FAIL \u001b[m\u001b[49m Test.test_capitalize_fail\n", "#\n", " \u001b[42m\u001b[1m PASS \u001b[m\u001b[49m \u001b[32m <1ms\u001b[39m\u001b[33m★\u001b[39m Test.test_return\n" ] }, { "ename": "", "evalue": "2", "output_type": "error", "traceback": [] } ], "source": [ "# Run the tests using Arcanist\n", "arc unit --rev HEAD^" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FROM dkdde/arcanist\n", "RUN apk add --update bash \\\n", " git \\\n", " php7 \\\n", " php7-curl \\\n", " php7-json \\\n", " py2-pip \\\n", " && pip install pytest pytest-tap tap.py \\\n", " && rm -rf /var/cache/apk/*\n", "RUN ln -s /arc/arcanist/bin/arc /bin/arc\n", "\n", "ENTRYPOINT []\n" ] } ], "source": [ "# Install arcanist in a docker image, as well as pytest and the TAP engine\n", "cat Dockerfile" ] }, { "cell_type": "code", "execution_count": 54, "metadata": { "slideshow": { "slide_type": "subslide" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "pipeline {\n", "\n", " agent {\n", " dockerfile true\n", " }\n", "\n", " stages {\n", " stage('run tests') {\n", " steps {\n", " sh 'git submodule update --init'\n", " sh 'arc unit --rev HEAD^'\n", " }\n", " }\n", " }\n", "\n", " post {\n", " always {\n", " archiveArtifacts artifacts: '*.tap'\n", " step([$class: \"TapPublisher\", testResults: \"*.tap\"])\n", " }\n", " }\n", "\n", "}\n" ] } ], "source": [ "# Configure the Jenkins job to use arcanist and to save the TAP result\n", "cat Jenkinsfile" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "subslide" } }, "source": [ "The results are visible in the Jenkins web interface\n", "![jenkins tap](images/jenkins-tap.png)" ] } ], "metadata": { "celltoolbar": "Slideshow", "kernelspec": { "display_name": "Bash", "language": "bash", "name": "bash" }, "language_info": { "codemirror_mode": "shell", "file_extension": ".sh", "mimetype": "text/x-sh", "name": "bash" }, "livereveal": { "height": 768, "theme": "serif", "transition": "none", "width": 1024 } }, "nbformat": 4, "nbformat_minor": 2 }