diff --git a/roles/nagios/files/mattermost.py b/roles/nagios/files/mattermost.py new file mode 100644 index 0000000..9a96302 --- /dev/null +++ b/roles/nagios/files/mattermost.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 NDrive SA +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import argparse +import json +import urllib2 + +VERSION = "0.3.1" + + +def parse(): + parser = argparse.ArgumentParser(description='Sends alerts to Mattermost') + parser.add_argument('--url', help='Incoming Webhook URL', required=True) + parser.add_argument('--channel', help='Channel to notify') + parser.add_argument('--username', help='Username to notify as', + default='Nagios') + parser.add_argument('--iconurl', help='URL of icon to use for username', + default='https://slack.global.ssl.fastly.net/7bf4/img/services/nagios_128.png') # noqa + parser.add_argument('--notificationtype', help='Notification Type', + required=True) + parser.add_argument('--hostalias', help='Host Alias', required=True) + parser.add_argument('--hostaddress', help='Host Address', required=True) + parser.add_argument('--hoststate', help='Host State') + parser.add_argument('--hostoutput', help='Host Output') + parser.add_argument('--servicedesc', help='Service Description') + parser.add_argument('--servicestate', help='Service State') + parser.add_argument('--serviceoutput', help='Service Output') + parser.add_argument('--cgiurl', help='Link to extinfo.cgi on your Nagios instance') + parser.add_argument('--version', action='version', + version='% (prog)s {version}'.format(version=VERSION)) + args = parser.parse_args() + return args + + +def encode_special_characters(text): + text = text.replace("%", "%25") + text = text.replace("&", "%26") + return text + + +def emoji(notificationtype): + return { + "RECOVERY": ":white_check_mark: ", + "PROBLEM": ":fire: ", + "DOWNTIMESTART": ":clock10: ", + "DOWNTIMEEND": ":sunny: " + }.get(notificationtype, "") + + +def text(args): + template_host = "__{notificationtype}__ {hostalias} is {hoststate}\n{hostoutput}" # noqa + template_service = "__{notificationtype}__ {hostalias} at {hostaddress}/{servicedesc} is {servicestate}\n{serviceoutput}" # noqa + if args.hoststate is not None: + template_cgiurl = " [View :link:]({cgiurl}?type=1&host={hostalias})" + elif args.servicestate is not None: + template_cgiurl = " [View :link:]({cgiurl}?type=2&host={hostalias}&service={servicedesc})" + template = template_service if args.servicestate else template_host + + text = emoji(args.notificationtype) + template.format(**vars(args)) + if args.cgiurl is not None: + # If we know the CGI url provide a clickable link to the nagios CGI + text = text + template_cgiurl.format(**vars(args)) + + return encode_special_characters(text) + + +def payload(args): + payload = { + "username": args.username, + "icon_url": args.iconurl, + "text": text(args) + } + + if args.channel: + payload["channel"] = args.channel + + data = "payload=" + json.dumps(payload) + return data + + +def request(url, data): + req = urllib2.Request(url, data) + response = urllib2.urlopen(req) + return response.read() + + +if __name__ == "__main__": + args = parse() + response = request(args.url, payload(args)) + print response diff --git a/roles/nagios/tasks/main.yml b/roles/nagios/tasks/main.yml index f9f5c2d..a0c8c57 100644 --- a/roles/nagios/tasks/main.yml +++ b/roles/nagios/tasks/main.yml @@ -1,39 +1,49 @@ --- - yum: name="{{ item }}" state=present with_items: - nagios - nagios-plugins-nrpe - name: Create nagios directory (Bug 1291734) file: state=directory path=/var/log/nagios/rw/ owner=nagios group=nagios + +- name: Create nagios scripts directory + file: state=directory path=/usr/local/nagios/libexec owner=nagios group=nagios + +- name: Mattermost script + copy: + src: mattermost.py + dest: /usr/local/nagios/libexec/mattermost.py + mode: '0755' + - name: Nagios configurations template: src: "{{ item }}.cfg" dest: "/etc/nagios/conf.d/{{ item }}.cfg" with_items: - commands - hosts - services - contacts - groups notify: reload nagios - name: Nagios cgi configurations template: src: "cgi.cfg" dest: "/etc/nagios/cgi.cfg" notify: reload nagios - name: Start nagios service: name=nagios state=started enabled=true - name: Nagios config for httpd template: src=nagios.conf dest=/opt/rh/httpd24/root/etc/httpd/conf.d/nagios.conf notify: reload apache - name: Apache landing page template: src=index.html dest=/opt/rh/httpd24/root/var/www/html/index.html notify: reload apache diff --git a/roles/nagios/templates/commands.cfg b/roles/nagios/templates/commands.cfg index 8af5d24..defb4cf 100644 --- a/roles/nagios/templates/commands.cfg +++ b/roles/nagios/templates/commands.cfg @@ -1,4 +1,26 @@ define command{ command_name check_nrpe command_line $USER1$/check_nrpe -H $HOSTADDRESS$ -c $ARG1$ } + +define command { + command_name notify-service-by-mattermost + command_line /usr/local/nagios/libexec/mattermost.py --url {{ mattermost_hook }} \ + --notificationtype "$NOTIFICATIONTYPE$" \ + --hostalias "$HOSTNAME$" \ + --hostaddress "$HOSTADDRESS$" \ + --servicedesc "$SERVICEDESC$" \ + --servicestate "$SERVICESTATE$" \ + --serviceoutput "$SERVICEOUTPUT$" +} + +define command { + command_name notify-host-by-mattermost + command_line /usr/local/nagios/libexec/mattermost.py --url {{ mattermost_hook }} \ + --notificationtype "$NOTIFICATIONTYPE$" \ + --hostalias "$HOSTNAME$" \ + --hostaddress "$HOSTADDRESS$" \ + --hoststate "$HOSTSTATE$" \ + --hostoutput "$HOSTOUTPUT$" +} + diff --git a/roles/nagios/templates/contacts.cfg b/roles/nagios/templates/contacts.cfg index f918761..0ec0cd8 100644 --- a/roles/nagios/templates/contacts.cfg +++ b/roles/nagios/templates/contacts.cfg @@ -1,12 +1,24 @@ define contactgroup{ contactgroup_name c4science_admins alias Nagios Administrators - members admin + members admin, mattermost } define contact { contact_name admin use generic-contact alias Nagios Admin email {{ email_monitoring }} } + +define contact { + contact_name mattermost + alias Mattermost + service_notification_period 24x7 + host_notification_period 24x7 + service_notification_options w,u,c,r + host_notification_options d,r + host_notification_commands notify-host-by-mattermost + service_notification_commands notify-service-by-mattermost +} + diff --git a/vars/main.yml.example b/vars/main.yml.example index 3188579..68c0120 100644 --- a/vars/main.yml.example +++ b/vars/main.yml.example @@ -1,126 +1,127 @@ --- env: prod project_name: "c4science" ansible_ssh_user: "centos" proxy: no http_proxy: "" https_proxy: "" no_proxy: "localhost" proxy_url: "{{ http_proxy }}" __no_proxy: "{{ no_proxy }}" user_pwd_root: "" user_pwd_centos: "" # OpenStack keypair_name: "" image_id: "" image_id_coreos: "" public_net: "" private_net: "" flavor_id_small: "" flavor_id_medium: "" flavor_id_large: "" flavor_id_small_backup: "" private_net_backup: "" image_id_backup: "" region_main: "" region_back: "" # Lbs internal_ip: 0.0.0.0 external_ip: 0.0.0.0 backup_ip: 0.0.0.0 ip_range: "10.0.0.0/16" notif_port: 22280 jenkins_port: 8080 stats_port: 8082 monit_port: 8081 monit_user: admin monit_pass: +mattermost_hook: https://mattermost-instance/token rolling_reboot_proxy_host: "{{ external_ip }}" rolling_reboot_pause: 30 rolling_reboot_wait_delay: 20 # App developer_mode: false http_scheme: 'http://' domain: example.com file_domain: "example-cdn.com" phabricator_branch: production phabricator_path: /srv/ repositories_path: /var/repo/ files_path: /var/files/ phd_user: phabricator www_user: apache vcs_user: git vcs_port_front: 22 vcs_port_back: 2222 jenkins_prefix: "/" jenkins_url: "{{ http_scheme }}jenkins.{{ domain }}{{ jenkins_prefix }}" jenkins_user: "admin" jenkins_token: "" jenkins_cred: "xxx-yyy-zzz" shib_metadata_file: "metadata.xml" shib_metadata_provider: "http://example.com/{{ shib_metadata_file }}" google_webmaster_file: "googleXYZ.html" phab_admin_apikey: 'api-xyz' phd_pull_number: 6 # Dbs mysql_repl_user: "repl" mysql_repl_pass: "" mysql_app_user: "" mysql_app_pass: "" mysql_shib_user: "" mysql_shib_pass: "" mysql_shib_db: "shibboleth" mysql_lbs_user: "haproxy" mysql_cluster_name: "" mysql_host: "10.0.0.0/255.255.0.0" mysql_binlog_expire: 7 # SSH ssh_keys: - "ssh-rsa mysuperkey1" - "ssh-rsa mysuperkey2" # Swap swap_path: "/swapfile" dd_bs_size_mb: 32 swap_count: 128 swappiness: 10 vfs_cache_pressure: 50 # Email configure_postfix: true enable_postfix_domain_rewrite: true enable_postfix_relayhost: true postfix_relayhost: "[{{ internal_ip }}]" postfix_rewrite_domain: "{{ domain }}" email_alias_phabricator: "" email_alias_postmaster: "" email_alias_admin: "" email_monitoring: "" # Config files php_ini: '../roles/phabricator/templates/php.ini' php_enable: '../roles/phabricator/templates/php.sh' sudoers: '../roles/phabricator/templates/sudoers' ssh_hook: '../roles/phabricator/templates/ssh_hook.sh' sshd_config: '../roles/phabricator/templates/sshd_config' sshd_init: '../roles/phabricator/templates/sshd_init' phd_init: '../roles/phabricator/templates/phd_init' phd_pull: '../roles/phabricator/templates/pull-daemons.sh' aphlict_init: '../roles/phabricator/templates/aphlict_init' aphlict_conf: '../roles/phabricator/templates/aphlict.custom.json' myconfig: '../roles/phabricator/templates/myconfig.conf.php' cust_phab_update_repo: '../roles/phabricator/templates/phab_update_repo.sh' cust_repo_gc: '../roles/phabricator/templates/repo-gc.sh' cust_compact_storage: '../roles/phabricator/templates/compact-storage.sh' repo_clusterize: '../roles/phabricator/templates/clusterize.py' arcanist_config: '../roles/phabricator/templates/arcrc'