From f6d484ffe91e3a64c651481c8e7481d32d801152 Mon Sep 17 00:00:00 2001
From: Paul Maruhn <paulmaruhn@posteo.de>
Date: Tue, 16 Nov 2021 02:37:46 +0100
Subject: [PATCH] wip: add monitoring configs
---
group_vars/{all.yml => gateways.yml} | 0
hosts.yml | 7 ++
playbook.yml | 5 ++
roles/services/files/authorized_keys | 11 +++
roles/services/files/etc/apt/sources.list | 8 ++
.../services/files/etc/cron.d/update-apt-list | 4 +
roles/services/files/etc/motd | 26 ++++++
roles/services/files/etc/resolv.conf | 4 +
roles/services/files/grafana.list | 1 +
.../services/files/prometheus/first_rules.yml | 34 +++++++
.../services/files/prometheus/prometheus.yml | 88 +++++++++++++++++++
roles/services/tasks/base.yml | 42 +++++++++
roles/services/tasks/main.yml | 16 ++++
roles/services/tasks/monitoring.yml | 31 +++++++
roles/services/tasks/software.yml | 46 ++++++++++
15 files changed, 323 insertions(+)
rename group_vars/{all.yml => gateways.yml} (100%)
create mode 100644 roles/services/files/authorized_keys
create mode 100644 roles/services/files/etc/apt/sources.list
create mode 100644 roles/services/files/etc/cron.d/update-apt-list
create mode 100644 roles/services/files/etc/motd
create mode 100644 roles/services/files/etc/resolv.conf
create mode 100644 roles/services/files/grafana.list
create mode 100644 roles/services/files/prometheus/first_rules.yml
create mode 100644 roles/services/files/prometheus/prometheus.yml
create mode 100644 roles/services/tasks/base.yml
create mode 100644 roles/services/tasks/main.yml
create mode 100644 roles/services/tasks/monitoring.yml
create mode 100644 roles/services/tasks/software.yml
diff --git a/group_vars/all.yml b/group_vars/gateways.yml
similarity index 100%
rename from group_vars/all.yml
rename to group_vars/gateways.yml
diff --git a/hosts.yml b/hosts.yml
index d36ef30..ec06d1d 100644
--- a/hosts.yml
+++ b/hosts.yml
@@ -1,4 +1,7 @@
gateways:
+ vars:
+ ansible_python_interpreter: /usr/bin/env python3
+ ansible_ssh_user: root
hosts:
kaisertor:
ansible_ssh_host: kaisertor.mesh.ffhl.chaotikum.org
@@ -10,6 +13,10 @@ gateways:
ansible_ssh_host: muehlentor.mesh.ffhl.chaotikum.org
test:
ansible_ssh_host: test.mesh.ffhl.chaotikum.org
+service_hosts:
vars:
ansible_python_interpreter: /usr/bin/env python3
ansible_ssh_user: root
+ hosts:
+ srv02:
+ ansible_ssh_host: srv02.luebeck.freifunk.net
diff --git a/playbook.yml b/playbook.yml
index e9a3039..e4a8184 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -8,3 +8,8 @@
become: yes
roles:
- icvpn
+
+- hosts: service_hosts
+ become: yes
+ roles:
+ - services
diff --git a/roles/services/files/authorized_keys b/roles/services/files/authorized_keys
new file mode 100644
index 0000000..0acdd27
--- /dev/null
+++ b/roles/services/files/authorized_keys
@@ -0,0 +1,11 @@
+# paul
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFEE6VP2jNtotQHEdc+qyw9jHA8Z2Bj2BAwKyhH/SjRG paul@tapas
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMNci5346re/3QqOhjC9PW1Zo0MA47hMm2r1GcEvdgff paul@taco
+
+# yksflip
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGFEc3u8Zffw9l7kIJRBB5p1RXHtA7LSDl6li/Zr6C1e yksflip@laptop
+
+# linus
+ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEApOTMJ+6z+x7jKeMRKj0imqdNdc65T+n8GKtNtf9KZjHf17K547JfDc0i4/SdCJO6Kud8gaoSm4YxhzOsJ/9c3GmyedvtyMJ1KFyTqd0PDOmvsgjviD4EfdENen55rYRwovKAHFgBN7vksgdcZcgVBCkApTrJelHVaHRukji7lNDNdA09qZBYkBPluPWA5MxFkntfarKzkSxdPnspkVmjSYYJ+PRiVJ5kcFv+FwjwmWjBSg8Ua5CSQ4kpUfCzXVXdnWKaaOcOqDkoIyr7/FHPNt9t35MpTjFrZ9s9o0DvPjaMW6fG6m0oq7IqBIZQR/iI8zTG+3LqkNwE0UH1gfidGrWLLNfZsP6HWKuJ6aUSaEBLBQV03wrj+9TDbCj4BiJHEUoG8rtPomsoP/kVuxvQhazJtGBvi2dY7lvirNrDdL+Cx3Po6DhTsJ4ScadWBjIiLJD4aN/k7WEp5i71du0PesyMiWlFyFlUXJh04OR7S7lD09s6m1bpNnyYXsyekJDbROckCZJCBd6Nq9JS0OekbFhGD2a9Yn9SZwl6qxsxvvadr0DoGVKBAo5VBOEsCFOZdQ9GPhBg8alQLBpjfG7M1rIP4Fq+5GYgsApq0SXP9HUXqpCTU1G3GKuaPO6dU1onJn9CxaJ/IrPzkbah7pLEtCypu2NjD0LXEFVfmzi/vm0= linus@Linus-Debian
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVrG1RXV37uiljc2NuBnrJ3j4t29sdmGTk7LsWohevSM7hhFxv2d3gEaaHqW0GXtDjtGJBYruDo+/rDdsyx10Sc94WkIoWYaSEqE5h9VSCgzeLPZZdV79HtEdsS989wvOyQ6ontNkkPv3WE4Xn9WgVd2ZKvTmglWt8kr6bzUfrIw2FduZqcogs/QmkCmVbsPjCVN9GChqPfGYy/rvfibA4KgJ81mbvFnQEfaCVgIoqBjdxsrLPFVYULsYOozNRvxNMNbZwVhqLO52ixY1sDEH7fsPHGmAX/KLJykJeD+m5Hq6y0y2CVO6/xs7rbENItq8bzwI2jom3Afvfdtjh3IxKSVmda+oXYVPrLdpd8nK8757gimWjHhSofJyybKpyxj6GhuJy+v+7SbBTY4jxPzyQxI1TDtCxrYD8hoM8oGHtjitNuP/GeMjPZTpkapvELiuDKZtSUrMKqRtQhDOQDFF2TwOIfIqUWp2YoMZCEvov0uR3CyHQVLR3+0GFoN1kYcAAALzFGWDGjH/KgBdjIBz7USAKRCjeMzy48DG/Qy4Xoz6fRBvyIZRAK3JJIAcCJWwRYi35LhT11kAqtM34qshvTXD6OulpbCOiBxXJCt383ILUcbPQYYbHaEptTs0QDtMJziE3PyVcSNKmvVB/IIJc4cC4W2adKnk9fwL4auoYmw== linus@work-lptp-sw
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYFR2UBd/Kv2eBCq3uYJWHhnxmC7903lQizTx2s6iTaFKR69qDjwdRP+bLHGwO0/uvlgDOh+SH+im3wA4mqGQCSaKB2eTJszkiuFCczLfx0ivhoGVu8myGLmSwbwP6COyQoFDvpayCL3lV4nVGFxgukOb89LiGlzScgKQEaQxPMNT+xijt1uPACxtybNtKLXmXlz1tDPgJzmswZM0tSuPVCjIqNAgvC+T70mow1KfSh3sE5e12PLQ8J43sD8UwuC1j0o6taZ2PleUSiCYOy+mzQjdN2+Ibq88pfRByWE18RfQEkvOygSv0rvMynjN7Cd/72jAActuhQZeAKDXQsxtB l@flausch
diff --git a/roles/services/files/etc/apt/sources.list b/roles/services/files/etc/apt/sources.list
new file mode 100644
index 0000000..4209473
--- /dev/null
+++ b/roles/services/files/etc/apt/sources.list
@@ -0,0 +1,8 @@
+deb http://deb.debian.org/debian bullseye main
+deb-src http://deb.debian.org/debian bullseye main
+
+deb http://deb.debian.org/debian-security/ bullseye-security main
+deb-src http://deb.debian.org/debian-security/ bullseye-security main
+
+deb http://deb.debian.org/debian bullseye-updates main
+deb-src http://deb.debian.org/debian bullseye-updates main
diff --git a/roles/services/files/etc/cron.d/update-apt-list b/roles/services/files/etc/cron.d/update-apt-list
new file mode 100644
index 0000000..eb550fd
--- /dev/null
+++ b/roles/services/files/etc/cron.d/update-apt-list
@@ -0,0 +1,4 @@
+# update the package lists so prometheus can alert us
+# if there are many updates available
+
+7 22 * * * apt-get update
diff --git a/roles/services/files/etc/motd b/roles/services/files/etc/motd
new file mode 100644
index 0000000..bed787c
--- /dev/null
+++ b/roles/services/files/etc/motd
@@ -0,0 +1,26 @@
+The programs included with the Debian GNU/Linux system are free software;
+the exact distribution terms for each program are described in the
+individual files in /usr/share/doc/*/copyright.
+
+Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
+permitted by applicable law.
+
+MMMMMMMMMMMMMMMMMMMMMMMMMWMMMMMMWWMMMMMMMMWNKkdlc:;,,,,;;:lox0XWMMMMMMMMMMMMMMMM
+MMMMMMMMMMMMMMMMMMMMMMMMNKXWWWWWKKMMMMMWNKxl:,,;cloddddooc:,,;cd0NWMMMMMMMMMMMMM
+MMMMMMMMMMMMWWMMMMMMMMWN0okXXXXKddKXNNKdl:,,:okOKXXNNNNXXK0kdc;,;lONMMMMMMMMMMMM
+MMMMMWWNXKK0000KKXNWNXXXkckWMMMNooXN0xl;,;cx0XXNNWWWNXWWWNNNXKOo;,;dKWMMMMMMMMMM
+MMMWXKOO0000OO0000OOO0NMk,dWMMMK:cKXd::cldKXXNWMMMMM0ONMMMMWWNXXkc,,l0WMMMMMMMMM
+MWX0O000KKXKxkXXK0OOOO0Ko'lXMMMO,;kd;,:kK0KNWMMMMMMWxlKMMMMMMWNXX0l,,lKWMMMMMMMM
+WKOOKKKXNWMXokWMWKO0KKOx:.:KMMWx''::,;xXXK0XWMMMMMMNl;OMMMMMMMWXKKOl::dKXXNWMMMM
+KO0K0KWMMMM0:oNMNKXNKK0o'.'dNM0:...',l0XKXXKWNNMMMMK:'xWMMMMWXXXKKKd,,c0WNXXXWMM
+OOK0KWMMMMWx':KMNKNMWKkc...:0Wx'...',oXXKNXKNOkWMMMO,.oNMMWNKXWMXk0x;':OWMMWNKXW
+OOK0XMMMMMNo.,OMNKNMMXx:...:0Wx'...',oKXKXXKXdlXMW0c..;xNWXKNMMMKokx;':0MMMMMWKX
+OOK0KMMMMWk;..lXWXKWWKk:...:0Wx'...',:OXKKKXKc:0MNo....;0NKNMMMWk:ll,.cKMMMMMMNK
+0O0K0NMMMNl...,kMWXXX0Oc...:0Xo.....',l0KKKNk,'dNNo....;0XKWMMWXo'','.:0MMMMMMWK
+X0OKK0XWMXl...,kMMN0k0k:...,:;'.......;oxOXO:..:0MO;...oNXKWMWN0l..''.,kMMMMMMWK
+WN0O0KKKX0c...,xXXKK0Od;...............,;ckk:..;0M0:..'xWNKXNXXx,......cKMMMMMNK
+MMWX0O000k:...'o0000O0O:................'';;'..;kNO;..'dXNX00kl,.......,kMMMMNKX
+MMMMWXKOdc'....;ok0OkXXc.................,;,...'okl'...:xkdlcc;'.......,kMMNXXNM
+MMMMMW0o'........;l:':o;.................','.............'''',,'.......,xXXXNWMM
+MMMMNx,................................................................,xNWMMMMM
+MMMMXc.................................................................,kMMMMMMM
diff --git a/roles/services/files/etc/resolv.conf b/roles/services/files/etc/resolv.conf
new file mode 100644
index 0000000..86434c7
--- /dev/null
+++ b/roles/services/files/etc/resolv.conf
@@ -0,0 +1,4 @@
+search luebeck.freifunk.net ffhl.de
+nameserver 1.1.1.1
+nameserver 2001:4860:4860::8888
+nameserver 8.8.4.4
diff --git a/roles/services/files/grafana.list b/roles/services/files/grafana.list
new file mode 100644
index 0000000..adbad20
--- /dev/null
+++ b/roles/services/files/grafana.list
@@ -0,0 +1 @@
+deb https://packages.grafana.com/oss/deb stable main
diff --git a/roles/services/files/prometheus/first_rules.yml b/roles/services/files/prometheus/first_rules.yml
new file mode 100644
index 0000000..fdf8d95
--- /dev/null
+++ b/roles/services/files/prometheus/first_rules.yml
@@ -0,0 +1,34 @@
+groups:
+ - name: ffhl
+ rules:
+ - record: "fastd_peer_traffic_sum"
+ expr: 'sum by (key, name) (rate(fastd_peer_traffic{iface=~"ffhl_mesh_vpn.*", kind="bytes", type=~"rx|tx"}[1m]))'
+ - record: "ffhl_mesh_links"
+ expr: 'count by (link_type) (link_tq{link_type!="undefined"})'
+
+ - record: "ffhl_nodes_online_percentage"
+ expr: 'meshnodes_online_total{job="hopglass"} / meshnodes_total{job="hopglass"}'
+
+ - record: "ffhl_mesh_avg_link_quality"
+ expr: 'avg by (link_type)(link_tq{link_type!="undefined"})'
+
+ - record: "ffhl_mesh_connected_clients_24"
+ expr: 'avg_over_time(total_clients{job="hopglass",instance_!="hopglass"}[24h])'
+
+ - record: 'ffhl_firmware_distribution'
+ expr: 'count by (firmware)(online{firmware!="", instance_!="hopglass"})'
+
+ - record: 'ffhl_device_distribution'
+ expr: 'count by (model)(online{model!="", instance_!="hopglass"})'
+
+ - record: 'ffhl_gateway_distribution'
+ expr: 'count by (gateway) (online{gateway!="", job="hopglass"})'
+
+ - record: 'ffhl_mesh_sum_traffic_type'
+ expr: 'sum by (type) (rate(statistics_traffic[300s]))'
+
+ - record: "ffhl_mesh_sum_traffic_mtype"
+ expr: 'sum by (mtype) (rate(statistics_traffic[300s]))'
+
+ - record: 'ffhl_node_statistics_traffic'
+ expr: 'sum by (nodeid, mtype) (rate(statistics_traffic{site="ffhl", instance_!="hopglass"}[5m]) * 8)'
diff --git a/roles/services/files/prometheus/prometheus.yml b/roles/services/files/prometheus/prometheus.yml
new file mode 100644
index 0000000..5e2a665
--- /dev/null
+++ b/roles/services/files/prometheus/prometheus.yml
@@ -0,0 +1,88 @@
+# Sample config for Prometheus.
+
+global:
+ scrape_interval: 90s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+ evaluation_interval: 60s # Evaluate rules every 15 seconds. The default is every 1 minute.
+ # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+ alertmanagers:
+ - static_configs:
+ - targets: ['localhost:9093']
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+ - "first_rules.yml"
+ #- "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+ # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+ - job_name: 'hopglass'
+ static_configs:
+ - targets: ['localhost:4000']
+ - job_name: 'gateways'
+ static_configs:
+ - targets:
+ - "muehlentor.mesh.ffhl.chaotikum.org:9100"
+ - "holstentor.mesh.ffhl.chaotikum.org:9100"
+ - "kaisertor.mesh.ffhl.chaotikum.org:9100"
+ - "huextertor.mesh.ffhl.chaotikum.org:9100"
+ - "builder.luebeck.freifunk.net:9100"
+ - "srv02.luebeck.freifunk.net:9100"
+ - "srv03.luebeck.freifunk.net:9100"
+ - "blueberry.luebeck.freifunk.net:9100"
+ - "strawberry.luebeck.freifunk.net:9100"
+ - job_name: powerdns
+ static_configs:
+ - targets:
+ - 'blueberry.luebeck.freifunk.net:8082'
+ - 'srv02.luebeck.freifunk.net:8082'
+ - 'kaisertor.luebeck.freifunk.net:8082'
+ - 'huextertor.luebeck.freifunk.net:8082'
+ - 'holstentor.luebeck.freifunk.net:8082'
+ - 'muehlentor.luebeck.freifunk.net:8082'
+ - job_name: gitea
+ static_configs:
+ - targets: ['git.luebeck.freifunk.net']
+ - job_name: requestd
+ scrape_interval: 60s
+ metrics_path: "/hooks/metrics"
+ static_configs:
+ - targets: ['localhost:21001']
+ - job_name: fastd
+ scrape_interval: 15s
+ static_configs:
+ - targets:
+ - 'kaisertor.luebeck.freifunk.net:9281'
+ - 'muehlentor.luebeck.freifunk.net:9281'
+ - 'holstentor.luebeck.freifunk.net:9281'
+ - 'huextertor.luebeck.freifunk.net:9281'
+ - 'testgw.luebeck.freifunk.net:9281'
+ - job_name: bird
+ scrape_interval: 15s
+ static_configs:
+ - targets:
+ - 'kaisertor.luebeck.freifunk.net:9324'
+ - 'huextertor.luebeck.freifunk.net:9324'
+ - 'holstentor.luebeck.freifunk.net:9324'
+ - 'muehlentor.luebeck.freifunk.net:8082'
+ - job_name: 'blackbox'
+ metrics_path: /probe
+ scrape_interval: 15s
+ params:
+ module: [icmp]
+ static_configs:
+ - targets:
+ - 'google.com'
+ - 'ipv6.google.com'
+ - '1.1.1.1'
+ relabel_configs:
+ - source_labels: [__address__]
+ target_label: __param_target
+ - source_labels: [__param_target]
+ target_label: target
+ - target_label: __address__
+ replacement: '192.168.1.22:9115'
diff --git a/roles/services/tasks/base.yml b/roles/services/tasks/base.yml
new file mode 100644
index 0000000..2a5e024
--- /dev/null
+++ b/roles/services/tasks/base.yml
@@ -0,0 +1,42 @@
+---
+
+- name: Disable root login with password
+ tags: [base]
+ lineinfile:
+ dest: /etc/ssh/sshd_config
+ regexp: "^#?PermitRootLogin"
+ line: "PermitRootLogin prohibit-password"
+
+
+# configurations and stuff
+- name: set local timezone
+ file:
+ state: link
+ src: /usr/share/zoneinfo/Europe/Berlin
+ dest: /etc/localtime
+
+
+- name: set locales
+ tags: [base]
+ block:
+ - lineinfile: dest=/etc/locale.gen line="en_US.UTF-8 UTF-8"
+ - lineinfile: dest=/etc/locale.gen line="de_DE.UTF-8 UTF-8"
+ - command: locale-gen
+
+- name: Copy authorized keys file
+ ansible.builtin.copy:
+ src: authorized_keys
+ dest: /root/.ssh/authorized_keys
+ owner: root
+ group: root
+
+- name: copy base configs
+ tags: [base, etc, apt, powerdns]
+ copy:
+ src: etc/
+ dest: /etc
+
+
+- name: install base tools
+ include: software.yml
+ tags: [base, apt, software]
diff --git a/roles/services/tasks/main.yml b/roles/services/tasks/main.yml
new file mode 100644
index 0000000..399967c
--- /dev/null
+++ b/roles/services/tasks/main.yml
@@ -0,0 +1,16 @@
+---
+- name: base config
+ tags: [base]
+ include_tasks:
+ file: base.yml
+ apply:
+ tags: [base]
+
+
+- name: install packages
+ include: software.yml
+ tags: [base, apt, software]
+
+- name: install packages
+ include: monitoring.yml
+ tags: [base, apt, software]
diff --git a/roles/services/tasks/monitoring.yml b/roles/services/tasks/monitoring.yml
new file mode 100644
index 0000000..fc92473
--- /dev/null
+++ b/roles/services/tasks/monitoring.yml
@@ -0,0 +1,31 @@
+---
+- name: copy prometheus config files
+ copy:
+ src: prometheus
+ dest: /etc/
+
+- name: restart prometheus
+ systemd:
+ state: restarted
+ name: prometheus
+
+
+#
+# Install Grafana
+#
+
+- name: add grafana repo pubkey
+ shell:
+ cmd: wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
+ warn: false
+
+- name: setup grafana repo
+ copy:
+ src: grafana.list
+ dest: /etc/apt/sources.list.d/
+
+- name: install grafana
+ apt:
+ update_cache: yes
+ state: present
+ name: grafana
diff --git a/roles/services/tasks/software.yml b/roles/services/tasks/software.yml
new file mode 100644
index 0000000..3ab69da
--- /dev/null
+++ b/roles/services/tasks/software.yml
@@ -0,0 +1,46 @@
+---
+
+- name: install python3-apt
+ command:
+ cmd: apt-get install -y python3-apt
+ warn: false
+
+- name: remove packages that are not needed
+ apt:
+ update_cache: yes
+ state: absent
+ name:
+ - cron-apt
+ - mutt
+
+- name: install tools
+ apt:
+ autoremove: yes
+ update_cache: yes
+ state: present
+ name:
+ # essential packages
+ - git
+ - openssh-server
+ - prometheus-node-exporter
+ - python3-apt
+ - apt-transport-https
+ - curl
+ # other useful tools
+ - apt-file
+ - bridge-utils
+ - dnsutils
+ - htop
+ - iftop
+ - iperf3
+ - iputils-ping
+ - jq
+ - molly-guard
+ - openssh-client
+ - python3-yaml
+ - socat
+ - tcpdump
+ - vim
+ - wget
+ - rsync
+ - nmap
--
GitLab