diff --git a/Makefile b/Makefile index 1bbc546..55e96e4 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,38 @@ cdir = $(shell pwd) -up: - docker-compose -f ./cicd/docker-compose.yml up -d --build +app-up: + docker-compose -f ./cicd/docker-compose.yml up -d --build ; \ ./cicd/init.sh -down: shutdown clean -shutdown: - docker-compose -f ./cicd/docker-compose.yml down - sudo docker rmi $$(sudo docker images -a | grep '' | awk '{print $$3}') +app-down: + docker-compose -f ./cicd/docker-compose.yml down ; \ + docker rmi $$(sudo docker images -a | grep '' | awk '{print $$3}') ; \ + rm -rf /opt/mysql_slave1/* ; \ + rm -rf /opt/mysql_slave2/* -clean: - rm -rf /opt/mysql_master/* ; \ +app-reload: + docker-compose -f ./cicd/docker-compose.yml down ; \ + docker-compose -f ./cicd/docker-compose.yml up -d ; \ + ./cicd/init.sh + +prom-up: + docker-compose -f ./test/monitor/docker-compose.yml up -d --build + +prom-down: + docker-compose -f ./test/monitor/docker-compose.yml down ; \ + docker rmi $$(sudo docker images -a | grep '' | awk '{print $$3}') + +up: rm -rf /opt/mysql_slave1/* ; \ rm -rf /opt/mysql_slave2/* ; \ - sudo docker rmi $$(sudo docker images -a | grep '' | awk '{print $$3}') + docker-compose -f ./cicd/docker-compose.yml up -d --build ; \ + ./cicd/init.sh + docker-compose -f ./test/monitor/docker-compose.yml up -d --build ; \ -.PHONY: up down \ No newline at end of file +down: + docker-compose -f ./test/monitor/docker-compose.yml down ; \ + docker-compose -f ./cicd/docker-compose.yml down ; \ + docker rmi $$(sudo docker images -a | grep '' | awk '{print $$3}') + + +.PHONY: app-up app-down app-reload prom-up prom-down up down \ No newline at end of file diff --git a/cicd/docker-compose.yml b/cicd/docker-compose.yml index 39b57d7..a7dbfc3 100644 --- a/cicd/docker-compose.yml +++ b/cicd/docker-compose.yml @@ -58,138 +58,4 @@ services: APP_DSN_PASS: app APP_DSN_BASE: app ports: - - "8080:8080" - - prometheus: - image: prom/prometheus:v2.24.1 - container_name: prometheus - volumes: - - ./prometheus:/etc/prometheus - - prometheus_data:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--storage.tsdb.retention.time=200h' - - '--web.enable-lifecycle' - restart: unless-stopped - expose: - - 9090 - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - - alertmanager: - image: prom/alertmanager:v0.21.0 - container_name: alertmanager - volumes: - - ./alertmanager:/etc/alertmanager - command: - - '--config.file=/etc/alertmanager/config.yml' - - '--storage.path=/alertmanager' - restart: unless-stopped - expose: - - 9093 - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - - nodeexporter: - image: prom/node-exporter:v1.1.0 - container_name: nodeexporter - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - '--path.procfs=/host/proc' - - '--path.rootfs=/rootfs' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' - restart: unless-stopped - expose: - - 9100 - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - - cadvisor: - image: gcr.io/cadvisor/cadvisor:v0.38.7 - container_name: cadvisor - volumes: - - /:/rootfs:ro - - /var/run:/var/run:rw - - /sys:/sys:ro - - /var/lib/docker:/var/lib/docker:ro - #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux - restart: unless-stopped - expose: - - 8080 - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - - grafana: - image: grafana/grafana:7.4.0 - container_name: grafana - volumes: - - grafana_data:/var/lib/grafana - - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards - - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources - environment: - - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin} - - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} - - GF_USERS_ALLOW_SIGN_UP=false - restart: unless-stopped - expose: - - 3000 - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - ports: - - "3001:3000" - - pushgateway: - image: prom/pushgateway:v1.4.0 - container_name: pushgateway - restart: unless-stopped - expose: - - 9091 - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - - caddy: - image: stefanprodan/caddy - container_name: caddy - ports: - - "3000:3000" - - "9090:9090" - - "9093:9093" - - "9091:9091" - volumes: - - ./caddy:/etc/caddy - environment: - - ADMIN_USER=${ADMIN_USER:-admin} - - ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} - restart: unless-stopped - networks: - - monitor-net - labels: - org.label-schema.group: "monitoring" - - -networks: - monitor-net: - driver: bridge - -volumes: - prometheus_data: { } - grafana_data: { } + - "8080:8080" \ No newline at end of file diff --git a/test/dz002.sh b/test/dz002/dz002.sh similarity index 100% rename from test/dz002.sh rename to test/dz002/dz002.sh diff --git a/test/dz003/README.md b/test/dz003/README.md index aa8ce3f..fc3a057 100644 --- a/test/dz003/README.md +++ b/test/dz003/README.md @@ -1,2 +1,34 @@ -Встроен запуск prometeus, grafana и т.д. -Графана доступна на http://localhost:3001/ \ No newline at end of file +#Полусинхронная репликация + +В результате выполнения ДЗ вы настроите полусинхронную репликацию, протестируете ее влияние на производительность системы и убедитесь, что теперь вы не теряете транзакции в случае аварии. + +### В данном задании тренируются навыки: +- обеспечение отказоустойчивости проекта; +- администрирование MySQL; +- настройка репликации; +- проведение нагрузочных тестов. + +### План выполнения ДЗ: +1) Настраиваем асинхронную репликацию. +2) Выбираем 2 любых запроса на чтения (в идеале самых частых и тяжелых по логике работы сайта) и переносим их на чтение со слейва. +3) Делаем нагрузочный тест по странице, которую перевели на слейв до и после репликации. Замеряем нагрузку мастера (CPU, la, disc usage, memory usage). +4) ОПЦИОНАЛЬНО: в качестве конфига, который хранит IP реплики сделать массив для легкого добавления реплики. Это не самый правильный способ балансирования нагрузки. Поэтому опционально. +5) Настроить 2 слейва и 1 мастер. +6) Включить row-based репликацию. +7) Включить GTID. +8) Настроить полусинхронную репликацию. +9) Создать нагрузку на запись в любую тестовую таблицу. На стороне, которой нагружаем считать, сколько строк мы успешно записали. +10) С помощью kill -9 убиваем мастер MySQL. +11) Заканчиваем нагрузку на запись. +12) Выбираем самый свежий слейв. Промоутим его до мастера. Переключаем на него второй слейв. +13) Проверяем, есть ли потери транзакций. + +Результатом сдачи ДЗ будет в виде исходного кода на github и отчета в текстовом виде, где вы отразите как вы выполняли каждый из пунктов. +Критерии оценки: Оценка происходит по принципу зачет/незачет. + +### Требования: +- В отчете корректно описано, как настроена репликация. +- 2 запроса переведено на чтение со слейва. +- Нагрузочное тестирование показало, что нагрузка перешла на слейв. +- В отчете описано как включить row-based репликацию и GTID +- Проведен эксперимент по потере и непотере транзакций при аварийной остановке master. \ No newline at end of file diff --git a/test/dz003.sh b/test/dz003/dz003.sh similarity index 100% rename from test/dz003.sh rename to test/dz003/dz003.sh diff --git a/test/dz003/img/dz003_part1.jpg b/test/dz003/img/dz003_part1.jpg new file mode 100644 index 0000000..56c44e5 Binary files /dev/null and b/test/dz003/img/dz003_part1.jpg differ diff --git a/test/dz003/report.md b/test/dz003/report.md new file mode 100644 index 0000000..634915b --- /dev/null +++ b/test/dz003/report.md @@ -0,0 +1,27 @@ +Встроен запуск prometeus, grafana и т.д. +Графана доступна на http://localhost:3001/ + +$ sudo docker run --rm -v /root/scripts:/scripts williamyeh/wrk -t1 -c10 -d5m --timeout 30s http://192.168.1.66:8080/search -s /scripts/post.lua -- debug true +Running 5m test @ http://192.168.1.66:8080/search +1 threads and 10 connections +Thread Stats Avg Stdev Max +/- Stdev +Latency 5.83s 1.74s 13.29s 77.58% +Req/Sec 3.57 4.74 30.00 82.50% +511 requests in 5.00m, 18.69MB read +Requests/sec: 1.70 +Transfer/sec: 63.79KB + +$ sudo make app-reload +... +... +... +$ sudo docker run --rm -v /root/scripts:/scripts williamyeh/wrk -t1 -c10 -d5m --timeout 30s http://192.168.1.66:8080/search -s /scripts/post.lua -- debug true +Running 5m test @ http://192.168.1.66:8080/search +1 threads and 10 connections +Thread Stats Avg Stdev Max +/- Stdev +Latency 32.12ms 31.59ms 561.62ms 90.65% +Req/Sec 373.34 147.37 610.00 68.45% +110834 requests in 5.00m, 433.90MB read +Non-2xx or 3xx responses: 110834 +Requests/sec: 369.39 +Transfer/sec: 1.45MB \ No newline at end of file diff --git a/test/dz003/post.lua b/test/dz003/scripts/post.lua similarity index 100% rename from test/dz003/post.lua rename to test/dz003/scripts/post.lua diff --git a/cicd/alertmanager/config.yml b/test/monitor/alertmanager/config.yml similarity index 100% rename from cicd/alertmanager/config.yml rename to test/monitor/alertmanager/config.yml diff --git a/cicd/caddy/Caddyfile b/test/monitor/caddy/Caddyfile similarity index 100% rename from cicd/caddy/Caddyfile rename to test/monitor/caddy/Caddyfile diff --git a/cicd/config b/test/monitor/config similarity index 100% rename from cicd/config rename to test/monitor/config diff --git a/test/monitor/docker-compose.yml b/test/monitor/docker-compose.yml new file mode 100644 index 0000000..fad070c --- /dev/null +++ b/test/monitor/docker-compose.yml @@ -0,0 +1,117 @@ +version: '3' +services: + + prometheus: + image: prom/prometheus:v2.24.1 + container_name: prometheus + volumes: + - ./prometheus:/etc/prometheus + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + restart: unless-stopped + expose: + - 9090 + labels: + org.label-schema.group: "monitoring" + + alertmanager: + image: prom/alertmanager:v0.21.0 + container_name: alertmanager + volumes: + - ./alertmanager:/etc/alertmanager + command: + - '--config.file=/etc/alertmanager/config.yml' + - '--storage.path=/alertmanager' + restart: unless-stopped + expose: + - 9093 + labels: + org.label-schema.group: "monitoring" + + nodeexporter: + image: prom/node-exporter:v1.1.0 + container_name: nodeexporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' + restart: unless-stopped + expose: + - 9100 + labels: + org.label-schema.group: "monitoring" + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.38.7 + container_name: cadvisor + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux + restart: unless-stopped + expose: + - 8080 + labels: + org.label-schema.group: "monitoring" + + grafana: + image: grafana/grafana:7.4.0 + container_name: grafana + volumes: + - grafana_data:/var/lib/grafana + - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources + environment: + - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + restart: unless-stopped + expose: + - 3000 + labels: + org.label-schema.group: "monitoring" + ports: + - "3001:3000" + + pushgateway: + image: prom/pushgateway:v1.4.0 + container_name: pushgateway + restart: unless-stopped + expose: + - 9091 + labels: + org.label-schema.group: "monitoring" + + caddy: + image: stefanprodan/caddy + container_name: caddy + ports: + - "3000:3000" + - "9090:9090" + - "9093:9093" + - "9091:9091" + volumes: + - ./caddy:/etc/caddy + environment: + - ADMIN_USER=${ADMIN_USER:-admin} + - ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin} + restart: unless-stopped + labels: + org.label-schema.group: "monitoring" + +volumes: + prometheus_data: { } + grafana_data: { } diff --git a/test/monitor/grafana/MyDashboard.json b/test/monitor/grafana/MyDashboard.json new file mode 100644 index 0000000..f89c668 --- /dev/null +++ b/test/monitor/grafana/MyDashboard.json @@ -0,0 +1,1133 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Containers metrics", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 10, + "links": [], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "7.4.0", + "targets": [ + { + "expr": "node_load1", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load 1m", + "metric": "node_load1", + "refId": "A", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System Load", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m])) / scalar(count(node_cpu_seconds_total{mode=\"user\"})) * 100", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_cpu_user_seconds_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 0 + }, + "hiddenSeries": false, + "id": 11, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name)(container_memory_usage_bytes{image!=\"\",container_label_org_label_schema_group=\"\"})", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "metric": "container_memory_usage", + "refId": "A", + "step": 1 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 4 + }, + "id": 9, + "links": [], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "7.4.0", + "targets": [ + { + "expr": "scalar(count(container_memory_usage_bytes{image!=\"\"}) > 0)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "containers", + "refId": "A", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Running Containers", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 10, + "x": 3, + "y": 7 + }, + "hiddenSeries": false, + "id": 15, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "written", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_fs_reads_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))", + "instant": false, + "interval": "", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "", + "refId": "C", + "step": 8 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container FS Reads/min", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 7 + }, + "hiddenSeries": false, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "written", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_fs_writes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))", + "instant": false, + "interval": "", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "", + "refId": "C", + "step": 8 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container FS Writes/min", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 8 + }, + "hideTimeOverride": true, + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_user_seconds_total{image!=\"\"}[1m])) / count(node_cpu_seconds_total{mode=\"user\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "timeFrom": "10s", + "title": "CPU Load", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 12 + }, + "hideTimeOverride": true, + "id": 5, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "65, 90", + "timeFrom": "10s", + "title": "Memory Load", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 11, + "x": 2, + "y": 14 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_network_receive_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_network_receive_bytes_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container Network Input", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 14 + }, + "hiddenSeries": false, + "id": 14, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\",container_label_org_label_schema_group=\"\"}[1m]))", + "intervalFactor": 10, + "legendFormat": "{{ name }}", + "metric": "container_network_transmit_bytes_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container Network Output", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 16 + }, + "hideTimeOverride": true, + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\"})", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "timeFrom": "10s", + "title": "Used Memory", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "refresh": "30s", + "schemaVersion": 27, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Docker Containers", + "uid": "kcaeWAPGk", + "version": 10 +} \ No newline at end of file diff --git a/cicd/grafana/provisioning/dashboards/dashboard.yml b/test/monitor/grafana/provisioning/dashboards/dashboard.yml similarity index 100% rename from cicd/grafana/provisioning/dashboards/dashboard.yml rename to test/monitor/grafana/provisioning/dashboards/dashboard.yml diff --git a/cicd/grafana/provisioning/dashboards/docker_containers.json b/test/monitor/grafana/provisioning/dashboards/docker_containers.json similarity index 100% rename from cicd/grafana/provisioning/dashboards/docker_containers.json rename to test/monitor/grafana/provisioning/dashboards/docker_containers.json diff --git a/cicd/grafana/provisioning/dashboards/docker_host.json b/test/monitor/grafana/provisioning/dashboards/docker_host.json similarity index 100% rename from cicd/grafana/provisioning/dashboards/docker_host.json rename to test/monitor/grafana/provisioning/dashboards/docker_host.json diff --git a/cicd/grafana/provisioning/dashboards/monitor_services.json b/test/monitor/grafana/provisioning/dashboards/monitor_services.json similarity index 100% rename from cicd/grafana/provisioning/dashboards/monitor_services.json rename to test/monitor/grafana/provisioning/dashboards/monitor_services.json diff --git a/cicd/grafana/provisioning/dashboards/nginx_container.json b/test/monitor/grafana/provisioning/dashboards/nginx_container.json similarity index 100% rename from cicd/grafana/provisioning/dashboards/nginx_container.json rename to test/monitor/grafana/provisioning/dashboards/nginx_container.json diff --git a/cicd/grafana/provisioning/datasources/datasource.yml b/test/monitor/grafana/provisioning/datasources/datasource.yml similarity index 100% rename from cicd/grafana/provisioning/datasources/datasource.yml rename to test/monitor/grafana/provisioning/datasources/datasource.yml diff --git a/cicd/helpers/aws/README.md b/test/monitor/helpers/aws/README.md similarity index 100% rename from cicd/helpers/aws/README.md rename to test/monitor/helpers/aws/README.md diff --git a/cicd/helpers/aws/cadvisor_ecs_task_definition.json b/test/monitor/helpers/aws/cadvisor_ecs_task_definition.json similarity index 100% rename from cicd/helpers/aws/cadvisor_ecs_task_definition.json rename to test/monitor/helpers/aws/cadvisor_ecs_task_definition.json diff --git a/cicd/helpers/aws/node_exporter_task_definition.json b/test/monitor/helpers/aws/node_exporter_task_definition.json similarity index 100% rename from cicd/helpers/aws/node_exporter_task_definition.json rename to test/monitor/helpers/aws/node_exporter_task_definition.json diff --git a/cicd/helpers/aws/prometheus.yml b/test/monitor/helpers/aws/prometheus.yml similarity index 100% rename from cicd/helpers/aws/prometheus.yml rename to test/monitor/helpers/aws/prometheus.yml diff --git a/cicd/prometheus/alert.rules b/test/monitor/prometheus/alert.rules similarity index 100% rename from cicd/prometheus/alert.rules rename to test/monitor/prometheus/alert.rules diff --git a/cicd/prometheus/prometheus.yml b/test/monitor/prometheus/prometheus.yml similarity index 100% rename from cicd/prometheus/prometheus.yml rename to test/monitor/prometheus/prometheus.yml