Merge branch 'ansiblilify-srv02' into master

hosts.yml

 gateways:
+  vars:
+    ansible_python_interpreter: /usr/bin/env python3
+    ansible_ssh_user: root
   hosts:
     kaisertor:
       ansible_ssh_host: kaisertor.mesh.ffhl.chaotikum.org
@@ -10,6 +13,10 @@ gateways:
       ansible_ssh_host: muehlentor.mesh.ffhl.chaotikum.org
     test:
       ansible_ssh_host: test.mesh.ffhl.chaotikum.org
+service_hosts:
   vars:
     ansible_python_interpreter: /usr/bin/env python3
     ansible_ssh_user: root
+  hosts:
+    srv02:
+      ansible_ssh_host: srv02.luebeck.freifunk.net

playbook.yml

@@ -8,3 +8,8 @@
   become: yes
   roles:
     - icvpn
+
+- hosts: service_hosts
+  become: yes
+  roles:
+    - services

roles/services/files/authorized_keys

+# paul
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFEE6VP2jNtotQHEdc+qyw9jHA8Z2Bj2BAwKyhH/SjRG paul@tapas
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMNci5346re/3QqOhjC9PW1Zo0MA47hMm2r1GcEvdgff paul@taco
+
+# yksflip
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGFEc3u8Zffw9l7kIJRBB5p1RXHtA7LSDl6li/Zr6C1e yksflip@laptop
+
+# linus
+ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEApOTMJ+6z+x7jKeMRKj0imqdNdc65T+n8GKtNtf9KZjHf17K547JfDc0i4/SdCJO6Kud8gaoSm4YxhzOsJ/9c3GmyedvtyMJ1KFyTqd0PDOmvsgjviD4EfdENen55rYRwovKAHFgBN7vksgdcZcgVBCkApTrJelHVaHRukji7lNDNdA09qZBYkBPluPWA5MxFkntfarKzkSxdPnspkVmjSYYJ+PRiVJ5kcFv+FwjwmWjBSg8Ua5CSQ4kpUfCzXVXdnWKaaOcOqDkoIyr7/FHPNt9t35MpTjFrZ9s9o0DvPjaMW6fG6m0oq7IqBIZQR/iI8zTG+3LqkNwE0UH1gfidGrWLLNfZsP6HWKuJ6aUSaEBLBQV03wrj+9TDbCj4BiJHEUoG8rtPomsoP/kVuxvQhazJtGBvi2dY7lvirNrDdL+Cx3Po6DhTsJ4ScadWBjIiLJD4aN/k7WEp5i71du0PesyMiWlFyFlUXJh04OR7S7lD09s6m1bpNnyYXsyekJDbROckCZJCBd6Nq9JS0OekbFhGD2a9Yn9SZwl6qxsxvvadr0DoGVKBAo5VBOEsCFOZdQ9GPhBg8alQLBpjfG7M1rIP4Fq+5GYgsApq0SXP9HUXqpCTU1G3GKuaPO6dU1onJn9CxaJ/IrPzkbah7pLEtCypu2NjD0LXEFVfmzi/vm0= linus@Linus-Debian
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVrG1RXV37uiljc2NuBnrJ3j4t29sdmGTk7LsWohevSM7hhFxv2d3gEaaHqW0GXtDjtGJBYruDo+/rDdsyx10Sc94WkIoWYaSEqE5h9VSCgzeLPZZdV79HtEdsS989wvOyQ6ontNkkPv3WE4Xn9WgVd2ZKvTmglWt8kr6bzUfrIw2FduZqcogs/QmkCmVbsPjCVN9GChqPfGYy/rvfibA4KgJ81mbvFnQEfaCVgIoqBjdxsrLPFVYULsYOozNRvxNMNbZwVhqLO52ixY1sDEH7fsPHGmAX/KLJykJeD+m5Hq6y0y2CVO6/xs7rbENItq8bzwI2jom3Afvfdtjh3IxKSVmda+oXYVPrLdpd8nK8757gimWjHhSofJyybKpyxj6GhuJy+v+7SbBTY4jxPzyQxI1TDtCxrYD8hoM8oGHtjitNuP/GeMjPZTpkapvELiuDKZtSUrMKqRtQhDOQDFF2TwOIfIqUWp2YoMZCEvov0uR3CyHQVLR3+0GFoN1kYcAAALzFGWDGjH/KgBdjIBz7USAKRCjeMzy48DG/Qy4Xoz6fRBvyIZRAK3JJIAcCJWwRYi35LhT11kAqtM34qshvTXD6OulpbCOiBxXJCt383ILUcbPQYYbHaEptTs0QDtMJziE3PyVcSNKmvVB/IIJc4cC4W2adKnk9fwL4auoYmw== linus@work-lptp-sw
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYFR2UBd/Kv2eBCq3uYJWHhnxmC7903lQizTx2s6iTaFKR69qDjwdRP+bLHGwO0/uvlgDOh+SH+im3wA4mqGQCSaKB2eTJszkiuFCczLfx0ivhoGVu8myGLmSwbwP6COyQoFDvpayCL3lV4nVGFxgukOb89LiGlzScgKQEaQxPMNT+xijt1uPACxtybNtKLXmXlz1tDPgJzmswZM0tSuPVCjIqNAgvC+T70mow1KfSh3sE5e12PLQ8J43sD8UwuC1j0o6taZ2PleUSiCYOy+mzQjdN2+Ibq88pfRByWE18RfQEkvOygSv0rvMynjN7Cd/72jAActuhQZeAKDXQsxtB l@flausch

roles/services/files/etc/apt/sources.list

+deb http://deb.debian.org/debian bullseye main
+deb-src http://deb.debian.org/debian bullseye main
+
+deb http://deb.debian.org/debian-security/ bullseye-security main
+deb-src http://deb.debian.org/debian-security/ bullseye-security main
+
+deb http://deb.debian.org/debian bullseye-updates main
+deb-src http://deb.debian.org/debian bullseye-updates main

roles/services/files/etc/cron.d/update-apt-list

+# update the package lists so prometheus can alert us
+# if there are many updates available
+
+7 22 * * * apt-get update

roles/services/files/etc/motd

+The programs included with the Debian GNU/Linux system are free software;
+the exact distribution terms for each program are described in the
+individual files in /usr/share/doc/*/copyright.
+
+Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
+permitted by applicable law.
+
+MMMMMMMMMMMMMMMMMMMMMMMMMWMMMMMMWWMMMMMMMMWNKkdlc:;,,,,;;:lox0XWMMMMMMMMMMMMMMMM
+MMMMMMMMMMMMMMMMMMMMMMMMNKXWWWWWKKMMMMMWNKxl:,,;cloddddooc:,,;cd0NWMMMMMMMMMMMMM
+MMMMMMMMMMMMWWMMMMMMMMWN0okXXXXKddKXNNKdl:,,:okOKXXNNNNXXK0kdc;,;lONMMMMMMMMMMMM
+MMMMMWWNXKK0000KKXNWNXXXkckWMMMNooXN0xl;,;cx0XXNNWWWNXWWWNNNXKOo;,;dKWMMMMMMMMMM
+MMMWXKOO0000OO0000OOO0NMk,dWMMMK:cKXd::cldKXXNWMMMMM0ONMMMMWWNXXkc,,l0WMMMMMMMMM
+MWX0O000KKXKxkXXK0OOOO0Ko'lXMMMO,;kd;,:kK0KNWMMMMMMWxlKMMMMMMWNXX0l,,lKWMMMMMMMM
+WKOOKKKXNWMXokWMWKO0KKOx:.:KMMWx''::,;xXXK0XWMMMMMMNl;OMMMMMMMWXKKOl::dKXXNWMMMM
+KO0K0KWMMMM0:oNMNKXNKK0o'.'dNM0:...',l0XKXXKWNNMMMMK:'xWMMMMWXXXKKKd,,c0WNXXXWMM
+OOK0KWMMMMWx':KMNKNMWKkc...:0Wx'...',oXXKNXKNOkWMMMO,.oNMMWNKXWMXk0x;':OWMMWNKXW
+OOK0XMMMMMNo.,OMNKNMMXx:...:0Wx'...',oKXKXXKXdlXMW0c..;xNWXKNMMMKokx;':0MMMMMWKX
+OOK0KMMMMWk;..lXWXKWWKk:...:0Wx'...',:OXKKKXKc:0MNo....;0NKNMMMWk:ll,.cKMMMMMMNK
+0O0K0NMMMNl...,kMWXXX0Oc...:0Xo.....',l0KKKNk,'dNNo....;0XKWMMWXo'','.:0MMMMMMWK
+X0OKK0XWMXl...,kMMN0k0k:...,:;'.......;oxOXO:..:0MO;...oNXKWMWN0l..''.,kMMMMMMWK
+WN0O0KKKX0c...,xXXKK0Od;...............,;ckk:..;0M0:..'xWNKXNXXx,......cKMMMMMNK
+MMWX0O000k:...'o0000O0O:................'';;'..;kNO;..'dXNX00kl,.......,kMMMMNKX
+MMMMWXKOdc'....;ok0OkXXc.................,;,...'okl'...:xkdlcc;'.......,kMMNXXNM
+MMMMMW0o'........;l:':o;.................','.............'''',,'.......,xXXXNWMM
+MMMMNx,................................................................,xNWMMMMM
+MMMMXc.................................................................,kMMMMMMM

roles/services/files/etc/resolv.conf

+search luebeck.freifunk.net ffhl.de
+nameserver 1.1.1.1
+nameserver 2001:4860:4860::8888
+nameserver 8.8.4.4

roles/services/files/grafana.list

+deb https://packages.grafana.com/oss/deb stable main

roles/services/files/prometheus/first_rules.yml

+groups:
+  - name: ffhl
+    rules:
+    - record: "fastd_peer_traffic_sum"
+      expr: 'sum by (key, name) (rate(fastd_peer_traffic{iface=~"ffhl_mesh_vpn.*", kind="bytes", type=~"rx|tx"}[1m]))'
+    - record: "ffhl_mesh_links"
+      expr: 'count by (link_type) (link_tq{link_type!="undefined"})'
+
+    - record: "ffhl_nodes_online_percentage"
+      expr: 'meshnodes_online_total{job="hopglass"} / meshnodes_total{job="hopglass"}'
+
+    - record: "ffhl_mesh_avg_link_quality"
+      expr: 'avg by (link_type)(link_tq{link_type!="undefined"})'
+
+    - record: "ffhl_mesh_connected_clients_24"
+      expr: 'avg_over_time(total_clients{job="hopglass",instance_!="hopglass"}[24h])'
+
+    - record: 'ffhl_firmware_distribution'
+      expr: 'count by (firmware)(online{firmware!="", instance_!="hopglass"})'
+
+    - record: 'ffhl_device_distribution'
+      expr: 'count by (model)(online{model!="", instance_!="hopglass"})'
+
+    - record: 'ffhl_gateway_distribution'
+      expr: 'count by (gateway) (online{gateway!="", job="hopglass"})'
+
+    - record: 'ffhl_mesh_sum_traffic_type'
+      expr: 'sum by (type) (rate(statistics_traffic[300s]))'
+
+    - record: "ffhl_mesh_sum_traffic_mtype"
+      expr: 'sum by (mtype) (rate(statistics_traffic[300s]))'
+
+    - record: 'ffhl_node_statistics_traffic'
+      expr: 'sum by (nodeid, mtype) (rate(statistics_traffic{site="ffhl", instance_!="hopglass"}[5m]) * 8)'

roles/services/files/prometheus/prometheus.yml

+# Sample config for Prometheus.
+
+global:
+  scrape_interval:     90s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 60s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: ['localhost:9093']
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  - "first_rules.yml"
+  #- "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'hopglass'
+    static_configs:
+      - targets: ['localhost:4000']
+  - job_name: 'gateways'
+    static_configs:
+    - targets:
+      - "muehlentor.mesh.ffhl.chaotikum.org:9100"
+      - "holstentor.mesh.ffhl.chaotikum.org:9100"
+      - "kaisertor.mesh.ffhl.chaotikum.org:9100"
+      - "huextertor.mesh.ffhl.chaotikum.org:9100"
+      - "builder.luebeck.freifunk.net:9100"
+      - "srv02.luebeck.freifunk.net:9100"
+      - "srv03.luebeck.freifunk.net:9100"
+      - "blueberry.luebeck.freifunk.net:9100"
+      - "strawberry.luebeck.freifunk.net:9100"
+  - job_name: powerdns
+    static_configs:
+    - targets:
+      - 'blueberry.luebeck.freifunk.net:8082'
+      - 'srv02.luebeck.freifunk.net:8082'
+      - 'kaisertor.luebeck.freifunk.net:8082'
+      - 'huextertor.luebeck.freifunk.net:8082'
+      - 'holstentor.luebeck.freifunk.net:8082'
+      - 'muehlentor.luebeck.freifunk.net:8082'
+  - job_name: gitea
+    static_configs:
+      - targets: ['git.luebeck.freifunk.net']
+  - job_name: requestd
+    scrape_interval: 60s
+    metrics_path: "/hooks/metrics"
+    static_configs:
+    - targets: ['localhost:21001']
+  - job_name: fastd
+    scrape_interval: 15s
+    static_configs:
+    - targets:
+      - 'kaisertor.luebeck.freifunk.net:9281'
+      - 'muehlentor.luebeck.freifunk.net:9281'
+      - 'holstentor.luebeck.freifunk.net:9281'
+      - 'huextertor.luebeck.freifunk.net:9281'
+      - 'testgw.luebeck.freifunk.net:9281'
+  - job_name: bird
+    scrape_interval: 15s
+    static_configs:
+      - targets:
+        - 'kaisertor.luebeck.freifunk.net:9324'
+        - 'huextertor.luebeck.freifunk.net:9324'
+        - 'holstentor.luebeck.freifunk.net:9324'
+        - 'muehlentor.luebeck.freifunk.net:8082'
+  - job_name: 'blackbox'
+    metrics_path: /probe
+    scrape_interval: 15s
+    params:
+      module: [icmp]
+    static_configs:
+    - targets:
+      - 'google.com'
+      - 'ipv6.google.com'
+      - '1.1.1.1'
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: __param_target
+    - source_labels: [__param_target]
+      target_label: target
+    - target_label: __address__
+      replacement: '192.168.1.22:9115'

roles/services/tasks/base.yml

+---
+
+- name: Disable root login with password
+  tags: [base]
+  lineinfile:
+    dest: /etc/ssh/sshd_config
+    regexp: "^#?PermitRootLogin"
+    line: "PermitRootLogin prohibit-password"
+
+
+# configurations and stuff
+- name: set local timezone
+  file:
+    state: link
+    src: /usr/share/zoneinfo/Europe/Berlin
+    dest: /etc/localtime
+
+
+- name: set locales
+  tags: [base]
+  block:
+    - lineinfile: dest=/etc/locale.gen line="en_US.UTF-8 UTF-8"
+    - lineinfile: dest=/etc/locale.gen line="de_DE.UTF-8 UTF-8"
+    - command: locale-gen
+
+- name: Copy authorized keys file
+  ansible.builtin.copy:
+    src: authorized_keys
+    dest: /root/.ssh/authorized_keys
+    owner: root
+    group: root
+
+- name: copy base configs
+  tags: [base, etc, apt, powerdns]
+  copy:
+    src: etc/
+    dest: /etc
+
+
+- name: install base tools
+  include: software.yml
+  tags: [base, apt, software]

roles/services/tasks/main.yml

+---
+- name: base config
+  tags: [base]
+  include_tasks:
+    file: base.yml
+    apply:
+      tags: [base]
+
+
+- name: install packages
+  include: software.yml
+  tags: [base, apt, software]
+
+- name: install packages
+  include: monitoring.yml
+  tags: [base, apt, software]

roles/services/tasks/monitoring.yml

+---
+- name: copy prometheus config files
+  copy:
+    src: prometheus
+    dest: /etc/
+
+- name: restart prometheus
+  systemd:
+    state: restarted
+    name: prometheus
+
+
+#
+# Install Grafana
+#
+
+- name: add grafana repo pubkey
+  shell:
+    cmd: wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
+    warn: false
+
+- name: setup grafana repo
+  copy:
+    src: grafana.list
+    dest: /etc/apt/sources.list.d/
+
+- name: install grafana
+  apt:
+    update_cache: yes
+    state: present
+    name: grafana

roles/services/tasks/software.yml

+---
+
+- name: install python3-apt
+  command:
+    cmd: apt-get install -y python3-apt
+    warn: false
+
+- name: remove packages that are not needed
+  apt:
+    update_cache: yes
+    state: absent
+    name:
+      - cron-apt
+      - mutt
+
+- name: install tools
+  apt:
+    autoremove: yes
+    update_cache: yes
+    state: present
+    name:
+      # essential packages
+      - git
+      - openssh-server
+      - prometheus-node-exporter
+      - python3-apt
+      - apt-transport-https
+      - curl
+      # other useful tools
+      - apt-file
+      - bridge-utils
+      - dnsutils
+      - htop
+      - iftop
+      - iperf3
+      - iputils-ping
+      - jq
+      - molly-guard
+      - openssh-client
+      - python3-yaml
+      - socat
+      - tcpdump
+      - vim
+      - wget
+      - rsync
+      - nmap