From 95b73daa36b23565a8566f71f9b202d3459b685f Mon Sep 17 00:00:00 2001 From: Sam Chudnick Date: Sun, 25 Jun 2023 09:52:36 -0400 Subject: Initial Commit --- .../services/monitoring/grafana/defaults/main.yml | 5 + .../services/monitoring/grafana/handlers/main.yml | 13 ++ roles/services/monitoring/grafana/tasks/main.yml | 125 +++++++++++++++++ .../services/monitoring/influxdb/defaults/main.yml | 6 + .../services/monitoring/influxdb/handlers/main.yml | 4 + roles/services/monitoring/influxdb/tasks/main.yml | 19 +++ roles/services/monitoring/loki/handlers/main.yml | 8 ++ roles/services/monitoring/loki/tasks/main.yml | 80 +++++++++++ .../prometheus/blackbox-exporter/tasks/main.yml | 0 .../prometheus/nginx_exporter/defaults/main.yml | 4 + .../prometheus/nginx_exporter/handlers/main.yml | 9 ++ .../prometheus/nginx_exporter/tasks/main.yml | 44 ++++++ .../prometheus/node_exporter/defaults/main.yml | 4 + .../prometheus/node_exporter/tasks/main.yml | 28 ++++ .../monitoring/prometheus/server/defaults/main.yml | 6 + .../monitoring/prometheus/server/tasks/main.yml | 79 +++++++++++ .../services/monitoring/promtail/handlers/main.yml | 39 ++++++ roles/services/monitoring/promtail/tasks/main.yml | 151 +++++++++++++++++++++ 18 files changed, 624 insertions(+) create mode 100644 roles/services/monitoring/grafana/defaults/main.yml create mode 100644 roles/services/monitoring/grafana/handlers/main.yml create mode 100644 roles/services/monitoring/grafana/tasks/main.yml create mode 100644 roles/services/monitoring/influxdb/defaults/main.yml create mode 100644 roles/services/monitoring/influxdb/handlers/main.yml create mode 100644 roles/services/monitoring/influxdb/tasks/main.yml create mode 100644 roles/services/monitoring/loki/handlers/main.yml create mode 100644 roles/services/monitoring/loki/tasks/main.yml create mode 100644 roles/services/monitoring/prometheus/blackbox-exporter/tasks/main.yml create mode 100644 roles/services/monitoring/prometheus/nginx_exporter/defaults/main.yml create mode 100644 roles/services/monitoring/prometheus/nginx_exporter/handlers/main.yml create mode 100644 roles/services/monitoring/prometheus/nginx_exporter/tasks/main.yml create mode 100644 roles/services/monitoring/prometheus/node_exporter/defaults/main.yml create mode 100644 roles/services/monitoring/prometheus/node_exporter/tasks/main.yml create mode 100644 roles/services/monitoring/prometheus/server/defaults/main.yml create mode 100644 roles/services/monitoring/prometheus/server/tasks/main.yml create mode 100644 roles/services/monitoring/promtail/handlers/main.yml create mode 100644 roles/services/monitoring/promtail/tasks/main.yml (limited to 'roles/services/monitoring') diff --git a/roles/services/monitoring/grafana/defaults/main.yml b/roles/services/monitoring/grafana/defaults/main.yml new file mode 100644 index 0000000..c346e54 --- /dev/null +++ b/roles/services/monitoring/grafana/defaults/main.yml @@ -0,0 +1,5 @@ +grafana_package: + - grafana + - nginx +grafana_config: files/grafana_config/ +grafana_data: files/grafana.db diff --git a/roles/services/monitoring/grafana/handlers/main.yml b/roles/services/monitoring/grafana/handlers/main.yml new file mode 100644 index 0000000..8026c6d --- /dev/null +++ b/roles/services/monitoring/grafana/handlers/main.yml @@ -0,0 +1,13 @@ +- name: update repos + apt: + update_cache: yes + +- name: restart grafana + service: + name: grafana-server + state: restarted + +- name: restart nginx + service: + name: nginx + state: restarted diff --git a/roles/services/monitoring/grafana/tasks/main.yml b/roles/services/monitoring/grafana/tasks/main.yml new file mode 100644 index 0000000..e9f824e --- /dev/null +++ b/roles/services/monitoring/grafana/tasks/main.yml @@ -0,0 +1,125 @@ +- name: install extrepo + package: + name: extrepo + state: latest + +- name: add Grafana repo + register: result + changed_when: result.stdout | regex_search("skipped") | bool + notify: update repos + command: + cmd: extrepo enable grafana + creates: /etc/apt/sources.list.d/extrepo_grafana.sources + +- meta: flush_handlers + +- name: update Grafana repo + changed_when: false + command: + cmd: extrepo update grafana + +- name: install grafana + package: + name: "{{ grafana_package }}" + +- name: deploy grafana config + notify: restart grafana + template: + src: "{{ grafana_config }}" + dest: /etc/grafana/grafana.ini + owner: root + group: grafana + mode: '0640' + +- name: deploy nginx configuration + notify: restart nginx + copy: + src: "{{ grafana_nginx_config }}" + dest: /etc/nginx/sites-available/grafana.conf + owner: root + group: root + mode: '0644' + +- name: symlink site + notify: restart nginx + file: + src: /etc/nginx/sites-available/grafana.conf + dest: /etc/nginx/sites-enabled/grafana.conf + owner: root + group: root + state: link + +- name: allow http (80/tcp) traffic + ufw: + rule: allow + port: '80' + proto: tcp + +- name: allow https (443/tcp) traffic + ufw: + rule: allow + port: '443' + proto: tcp + +- name: enable grafana + systemd: + daemon_reload: yes + enabled: yes + masked: no + name: grafana-server + +- meta: flush_handlers + +- name: add grafana user + ignore_errors: yes + community.grafana.grafana_user: + name: "{{ grafana_admin }}" + email: "{{ grafana_email }}" + url: "{{ grafana_url }}" + login: "{{ grafana_admin }}" + password: "{{ grafana_password }}" + is_admin: true + state: present + +- name: add prometheus datasource + community.grafana.grafana_datasource: + grafana_url: "{{ grafana_url }}" + grafana_user: "{{ grafana_admin }}" + grafana_password: "{{ grafana_password }}" + name: "Prometheus" + ds_type: prometheus + ds_url: "{{ prometheus_url }}" + access: proxy + +- name: add influxdb datasource + community.grafana.grafana_datasource: + grafana_url: "{{ grafana_url }}" + grafana_user: "{{ grafana_admin }}" + grafana_password: "{{ grafana_password }}" + name: "Proxmox InfluxDB" + ds_type: influxdb + ds_url: "{{ influxdb_url }}" + database: "{{ influx_database }}" + user: "{{ influx_user }}" + password: "{{ influx_password }}" + access: proxy + +- name: add loki datasource + community.grafana.grafana_datasource: + grafana_url: "{{ grafana_url }}" + grafana_user: "{{ grafana_admin }}" + grafana_password: "{{ grafana_password }}" + name: "Loki" + ds_type: loki + ds_url: "{{ loki_url }}" + access: proxy + +- name: import main custom dashboard + delegate_to: localhost + become: no + community.grafana.grafana_dashboard: + grafana_url: "{{ grafana_url }}" + grafana_user: "{{ grafana_admin }}" + grafana_password: "{{ grafana_password }}" + path: "{{ grafana_dashboard_main }}" + overwrite: yes diff --git a/roles/services/monitoring/influxdb/defaults/main.yml b/roles/services/monitoring/influxdb/defaults/main.yml new file mode 100644 index 0000000..180ad8e --- /dev/null +++ b/roles/services/monitoring/influxdb/defaults/main.yml @@ -0,0 +1,6 @@ +influxdb_packages: + - influxdb + - influxdb-client + +influx_config: files/influxdb.conf +influx_data: files/influx_data/ diff --git a/roles/services/monitoring/influxdb/handlers/main.yml b/roles/services/monitoring/influxdb/handlers/main.yml new file mode 100644 index 0000000..765a040 --- /dev/null +++ b/roles/services/monitoring/influxdb/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart influxdb + service: + name: influxdb + state: restarted diff --git a/roles/services/monitoring/influxdb/tasks/main.yml b/roles/services/monitoring/influxdb/tasks/main.yml new file mode 100644 index 0000000..06d6e86 --- /dev/null +++ b/roles/services/monitoring/influxdb/tasks/main.yml @@ -0,0 +1,19 @@ +- name: install packages + package: + name: "{{ influxdb_packages }}" + state: latest + +- name: copy config + notify: restart influxdb + copy: + src: "{{ influx_config }}" + dest: /etc/influxdb/influxdb.conf + owner: root + group: root + mode: '0644' + +- name: enable influxdb + systemd: + name: influxdb + enabled: yes + masked: no diff --git a/roles/services/monitoring/loki/handlers/main.yml b/roles/services/monitoring/loki/handlers/main.yml new file mode 100644 index 0000000..e70412f --- /dev/null +++ b/roles/services/monitoring/loki/handlers/main.yml @@ -0,0 +1,8 @@ +- name: update repos + apt: + update_cache: yes + +- name: restart nginx + service: + name: nginx + state: restarted diff --git a/roles/services/monitoring/loki/tasks/main.yml b/roles/services/monitoring/loki/tasks/main.yml new file mode 100644 index 0000000..31a7375 --- /dev/null +++ b/roles/services/monitoring/loki/tasks/main.yml @@ -0,0 +1,80 @@ +- name: install extrepo + package: + name: extrepo + state: latest + +- name: add Grafana repo + register: result + changed_when: result.stdout | regex_search("skipped") | bool + notify: update repos + command: + cmd: extrepo enable grafana + creates: /etc/apt/sources.list.d/extrepo_grafana.sources + +- meta: flush_handlers + +- name: add Grafana repo + changed_when: false + command: + cmd: extrepo update grafana + +- name: install loki + package: + name: loki + state: latest + +- name: deploy loki configuration + copy: + src: "{{ loki_config }}" + dest: /etc/loki/config.yml + owner: root + group: root + mode: '0644' + +- name: deploy nginx configuration + copy: + src: "{{ loki_nginx_config }}" + dest: /etc/nginx/sites-available/loki.conf + owner: root + group: root + mode: '0644' + register: nginxconfig + notify: restart nginx + +- name: symlink site + file: + src: /etc/nginx/sites-available/loki.conf + dest: /etc/nginx/sites-enabled/loki.conf + owner: root + group: root + state: link + +- name: allow http (80/tcp) traffic + ufw: + rule: allow + port: '80' + proto: tcp + +- name: allow https (443/tcp) traffic + ufw: + rule: allow + port: '443' + proto: tcp + +- name: allow loki log (3100/tcp) traffic + ufw: + rule: allow + port: '3100' + proto: tcp + +- name: enable loki + systemd: + daemon_reload: yes + enabled: yes + masked: no + name: loki + +- name: restart loki + systemd: + name: loki + state: restarted diff --git a/roles/services/monitoring/prometheus/blackbox-exporter/tasks/main.yml b/roles/services/monitoring/prometheus/blackbox-exporter/tasks/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/roles/services/monitoring/prometheus/nginx_exporter/defaults/main.yml b/roles/services/monitoring/prometheus/nginx_exporter/defaults/main.yml new file mode 100644 index 0000000..9d2b8a5 --- /dev/null +++ b/roles/services/monitoring/prometheus/nginx_exporter/defaults/main.yml @@ -0,0 +1,4 @@ +nginx_exporter_debian_package: prometheus-nginx-exporter +nginx_exporter_fedora_package: golang-github-prometheus-node-exporter +prometheus_server_ip: 192.168.88.32 +nginx_exporter_port: '9113' diff --git a/roles/services/monitoring/prometheus/nginx_exporter/handlers/main.yml b/roles/services/monitoring/prometheus/nginx_exporter/handlers/main.yml new file mode 100644 index 0000000..fe9a90d --- /dev/null +++ b/roles/services/monitoring/prometheus/nginx_exporter/handlers/main.yml @@ -0,0 +1,9 @@ +- name: restart nginx + service: + name: nginx + state: restarted + +- name: restart nginx-exporter + service: + name: prometheus-nginx-exporter + state: started diff --git a/roles/services/monitoring/prometheus/nginx_exporter/tasks/main.yml b/roles/services/monitoring/prometheus/nginx_exporter/tasks/main.yml new file mode 100644 index 0000000..819f71e --- /dev/null +++ b/roles/services/monitoring/prometheus/nginx_exporter/tasks/main.yml @@ -0,0 +1,44 @@ +- name: install package (Debian) + when: ansible_facts['distribution'] == "Debian" + package: + name: "{{ nginx_exporter_debian_package }}" + +- name: allow port + ufw: + rule: allow + direction: in + proto: tcp + src: "{{ prometheus_server_ip }}" + to_port: "{{ nginx_exporter_port }}" + +- name: copy defaults file + notify: restart nginx-exporter + copy: + src: "{{ nginx_exporter_defaults }}" + dest: /etc/default/prometheus-nginx-exporter + owner: root + group: root + mode: '0644' + +- name: deploy nginx configuration + notify: restart nginx + copy: + src: "{{ nginx_exporter_config }}" + dest: /etc/nginx/sites-available/metrics.conf + owner: root + group: root + mode: '0644' + +- name: symlink site + file: + src: /etc/nginx/sites-available/metrics.conf + dest: /etc/nginx/sites-enabled/metrics.conf + owner: root + group: root + state: link + +- name: enable service + systemd: + name: prometheus-nginx-exporter + enabled: yes + masked: no diff --git a/roles/services/monitoring/prometheus/node_exporter/defaults/main.yml b/roles/services/monitoring/prometheus/node_exporter/defaults/main.yml new file mode 100644 index 0000000..e4ff351 --- /dev/null +++ b/roles/services/monitoring/prometheus/node_exporter/defaults/main.yml @@ -0,0 +1,4 @@ +node_exporter_debian_package: prometheus-node-exporter +node_exporter_fedora_package: golang-github-prometheus-node-exporter +prometheus_server_ip: 192.168.88.32 +node_exporter_port: '9100' diff --git a/roles/services/monitoring/prometheus/node_exporter/tasks/main.yml b/roles/services/monitoring/prometheus/node_exporter/tasks/main.yml new file mode 100644 index 0000000..6bbcc08 --- /dev/null +++ b/roles/services/monitoring/prometheus/node_exporter/tasks/main.yml @@ -0,0 +1,28 @@ +- name: install package (Debian) + when: ansible_facts['distribution'] == "Debian" + package: + name: "{{ node_exporter_debian_package }}" + +- name: install package (Fedora) + when: ansible_facts['distribution'] == "Fedora" + package: + name: "{{ node_exporter_fedora_package }}" + +- name: allow port + ufw: + rule: allow + direction: in + proto: tcp + src: "{{ prometheus_server_ip }}" + to_port: "{{ node_exporter_port }}" + +- name: enable service + systemd: + name: prometheus-node-exporter + enabled: yes + masked: no + +- name: restart service + service: + name: prometheus-node-exporter + state: restarted diff --git a/roles/services/monitoring/prometheus/server/defaults/main.yml b/roles/services/monitoring/prometheus/server/defaults/main.yml new file mode 100644 index 0000000..696e7cc --- /dev/null +++ b/roles/services/monitoring/prometheus/server/defaults/main.yml @@ -0,0 +1,6 @@ +prometheus_package: prometheus +management_ip: 192.168.88.254 +grafana_server_ip: 192.168.88.21 +prometheus_port: '9090' +prometheus_config: files/prometheus.yml +prometheus_defaults: files/prometheus diff --git a/roles/services/monitoring/prometheus/server/tasks/main.yml b/roles/services/monitoring/prometheus/server/tasks/main.yml new file mode 100644 index 0000000..06ecc10 --- /dev/null +++ b/roles/services/monitoring/prometheus/server/tasks/main.yml @@ -0,0 +1,79 @@ +- name: install package + package: + name: "{{ prometheus_package }}" + +- name: allow access to metrics from grafana + ufw: + rule: allow + direction: in + proto: tcp + src: "{{ grafana_server_ip }}" + to_port: "{{ prometheus_port }}" + +- name: allow access to metrics from management + ufw: + rule: allow + direction: in + proto: tcp + src: "{{ management_ip }}" + to_port: "{{ prometheus_port }}" + +- name: copy config file + copy: + src: "{{ prometheus_config }}" + dest: /etc/prometheus/prometheus.yml + owner: root + group: root + mode: '0644' + +- name: copy defaults file + copy: + src: "{{ prometheus_defaults }}" + dest: /etc/default/prometheus + owner: root + group: root + mode: '0644' + +- name: enable service + systemd: + name: prometheus + enabled: yes + masked: no + +- name: restart service + service: + name: prometheus + state: restarted + +- name: deploy nginx configuration + copy: + src: "{{ prometheus_nginx_config }}" + dest: /etc/nginx/sites-available/grafana.conf + owner: root + group: root + mode: '0644' + +- name: symlink site + file: + src: /etc/nginx/sites-available/grafana.conf + dest: /etc/nginx/sites-enabled/grafana.conf + owner: root + group: root + state: link + +- name: allow http (80/tcp) traffic + ufw: + rule: allow + port: '80' + proto: tcp + +- name: allow https (443/tcp) traffic + ufw: + rule: allow + port: '443' + proto: tcp + +- name: restart nginx + service: + name: nginx + state: restarted diff --git a/roles/services/monitoring/promtail/handlers/main.yml b/roles/services/monitoring/promtail/handlers/main.yml new file mode 100644 index 0000000..97ea7d3 --- /dev/null +++ b/roles/services/monitoring/promtail/handlers/main.yml @@ -0,0 +1,39 @@ +- name: update repos - debian + apt: + update_cache: yes + +- name: update repos - fedora + dnf: + name: "*" + state: latest + +- name: build loki-docker-driver plugin for private repo + become: yes + become_user: "{{ docker_username }}" + environment: + LOKI_DOCKER_DRIVER: "{{ docker_registry_url }}/{{ docker_registry_username }}/loki-docker-driver" + community.general.make: + chdir: "{{ docker_home }}/plugins/loki" + target: docker-driver-push + +- name: restart rootless docker + become: yes + become_user: "{{ docker_username }}" + systemd: + name: docker + enabled: yes + state: restarted + scope: user + environment: + XDG_RUNTIME_DIR: "/run/user/{{ docker_uid }}" + +- name: restart docker + service: + name: docker + state: restarted + +- name: restart promtail + when: promtail_config.changed + service: + name: promtail + state: restarted diff --git a/roles/services/monitoring/promtail/tasks/main.yml b/roles/services/monitoring/promtail/tasks/main.yml new file mode 100644 index 0000000..f8b28cc --- /dev/null +++ b/roles/services/monitoring/promtail/tasks/main.yml @@ -0,0 +1,151 @@ +- name: install extrepo + when: ansible_facts['distribution'] == 'Debian' + package: + name: extrepo + state: latest + +- name: add grafana repo | debian + when: ansible_facts['distribution'] == 'Debian' + register: result + changed_when: result.stdout | regex_search("skipped") | bool + notify: update repos - debian + command: + cmd: extrepo enable grafana + creates: /etc/apt/sources.list.d/extrepo_grafana.sources + +- meta: flush_handlers + +- name: update grafana extrepo data | debian + when: ansible_facts['distribution'] == 'Debian' + changed_when: false + command: + cmd: extrepo update grafana + +- name: add Grafana repo | fedora + when: ansible_facts['distribution'] == 'Fedora' + notify: update repos - fedora + yum_repository: + name: grafana + file: grafna + description: "Grafana OSS Repo" + baseurl: "https://rpm.grafana.com" + repo_gpgcheck: yes + enabled: yes + gpgcheck: yes + gpgkey: https://rpm.grafana.com/gpg.key + sslverify: yes + sslcacert: /etc/pki/tls/certs/ca-bundle.crt + exclude: "*beta*" + +- name: install promtail + package: + name: promtail + state: latest + +- name: add promtail to adm group for log access (debian) + when: ansible_facts['distribution'] == 'Debian' + user: + name: promtail + groups: adm + append: yes + +- name: add promtail to systemd-journal group for journal access + user: + name: promtail + groups: systemd-journal + append: yes + +- name: create docker plugin directory + when: "'docker_hosts' in group_names" + become: yes + become_user: "{{ docker_username }}" + file: + path: "{{ docker_home }}/plugins" + state: directory + owner: "{{ docker_username }}" + group: "{{ docker_username }}" + mode: "0755" + +- name: clone loki repo + when: "'docker_hosts' in group_names" + become: yes + become_user: "{{ docker_username }}" + git: + repo: "{{ loki_repo }}" + dest: "{{ docker_home }}/plugins/loki" + version: "{{ loki_version }}" + register: repo + notify: build loki-docker-driver plugin for private repo + +- meta: flush_handlers + +- name: login to docker registry + when: "'docker_hosts' in group_names" + become: yes + become_user: "{{ docker_username }}" + environment: + XDG_RUNTIME_DIR: "/run/user/{{ docker_uid }}" + docker_login: + docker_host: "unix://run/user/{{ docker_uid }}/docker.sock" + registry_url: "{{ docker_registry_url }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + +# docker driver rootless + +- name: enable loki-docker-driver plugin + when: "'docker_hosts' in group_names" + become: yes + become_user: "{{ docker_username }}" + notify: restart rootless docker + community.docker.docker_plugin: + plugin_name: "{{ docker_registry_url }}/{{ docker_registry_username }}/loki-docker-driver:main" + state: enable + docker_host: "unix://run/user/{{ docker_uid }}/docker.sock" + alias: loki + +- name: deploy docker config + when: "'docker_hosts' in group_names" + notify: restart rootless docker + copy: + src: "{{ docker_config }}" + dest: "{{ docker_home }}/.config/docker/daemon.json" + owner: "{{ docker_username }}" + group: "{{ docker_username }}" + mode: '0644' + +# docker driver root + +- name: enable loki-docker-driver plugin + when: "'docker_hosts' in group_names" + notify: restart docker + community.docker.docker_plugin: + plugin_name: "{{ docker_registry_url }}/{{ docker_registry_username }}/loki-docker-driver:main" + state: enable + alias: loki + +- name: deploy docker config + when: "'docker_hosts' in group_names" + notify: restart docker + copy: + src: "{{ docker_config }}" + dest: /etc/docker/daemon.json + owner: root + group: root + mode: '0644' + +- name: deploy promtail configuration + notify: restart promtail + copy: + src: "{{ promtail_config }}" + dest: /etc/promtail/config.yml + owner: root + group: root + mode: '0644' + +- name: enable promtail + systemd: + daemon_reload: yes + enabled: yes + masked: no + name: promtail -- cgit v1.2.3