OTUS
/
HighLoad_HomeWork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add prom&grafana

main
Andrey Ivanov 2021-02-13 09:23:19 -05:00 committed by Andrey Ivanov
parent 6f0d85600d
commit f7cf301800
25 changed files with 1342 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
Makefile
View File

@ -1,18 +1,38 @@
cdir = $(shell pwd)
up:
docker-compose -f ./cicd/docker-compose.yml up -d --build
app-up:
docker-compose -f ./cicd/docker-compose.yml up -d --build ; \
./cicd/init.sh
down: shutdown clean
shutdown:
docker-compose -f ./cicd/docker-compose.yml down
sudo docker rmi $$(sudo docker images -a | grep '<none>' | awk '{print $$3}')
app-down:
docker-compose -f ./cicd/docker-compose.yml down ; \
docker rmi $$(sudo docker images -a | grep '<none>' | awk '{print $$3}') ; \
rm -rf /opt/mysql_slave1/* ; \
rm -rf /opt/mysql_slave2/*
clean:
rm -rf /opt/mysql_master/* ; \
app-reload:
docker-compose -f ./cicd/docker-compose.yml down ; \
docker-compose -f ./cicd/docker-compose.yml up -d ; \
./cicd/init.sh
prom-up:
docker-compose -f ./test/monitor/docker-compose.yml up -d --build
prom-down:
docker-compose -f ./test/monitor/docker-compose.yml down ; \
docker rmi $$(sudo docker images -a | grep '<none>' | awk '{print $$3}')
up:
rm -rf /opt/mysql_slave1/* ; \
rm -rf /opt/mysql_slave2/* ; \
sudo docker rmi $$(sudo docker images -a | grep '<none>' | awk '{print $$3}')
docker-compose -f ./cicd/docker-compose.yml up -d --build ; \
./cicd/init.sh
docker-compose -f ./test/monitor/docker-compose.yml up -d --build ; \
.PHONY: up down
down:
docker-compose -f ./test/monitor/docker-compose.yml down ; \
docker-compose -f ./cicd/docker-compose.yml down ; \
docker rmi $$(sudo docker images -a | grep '<none>' | awk '{print $$3}')
.PHONY: app-up app-down app-reload prom-up prom-down up down

136
cicd/docker-compose.yml
View File

@ -58,138 +58,4 @@ services:
APP_DSN_PASS: app
APP_DSN_BASE: app
ports:
- "8080:8080"
prometheus:
image: prom/prometheus:v2.24.1
container_name: prometheus
volumes:
- ./prometheus:/etc/prometheus
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--storage.tsdb.retention.time=200h'
- '--web.enable-lifecycle'
restart: unless-stopped
expose:
- 9090
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
alertmanager:
image: prom/alertmanager:v0.21.0
container_name: alertmanager
volumes:
- ./alertmanager:/etc/alertmanager
command:
- '--config.file=/etc/alertmanager/config.yml'
- '--storage.path=/alertmanager'
restart: unless-stopped
expose:
- 9093
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
nodeexporter:
image: prom/node-exporter:v1.1.0
container_name: nodeexporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
restart: unless-stopped
expose:
- 9100
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.38.7
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker:/var/lib/docker:ro
#- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
restart: unless-stopped
expose:
- 8080
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
grafana:
image: grafana/grafana:7.4.0
container_name: grafana
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
- ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
environment:
- GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
- GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
restart: unless-stopped
expose:
- 3000
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
ports:
- "3001:3000"
pushgateway:
image: prom/pushgateway:v1.4.0
container_name: pushgateway
restart: unless-stopped
expose:
- 9091
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
caddy:
image: stefanprodan/caddy
container_name: caddy
ports:
- "3000:3000"
- "9090:9090"
- "9093:9093"
- "9091:9091"
volumes:
- ./caddy:/etc/caddy
environment:
- ADMIN_USER=${ADMIN_USER:-admin}
- ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
restart: unless-stopped
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
networks:
monitor-net:
driver: bridge
volumes:
prometheus_data: { }
grafana_data: { }
- "8080:8080"

0
test/dz002.sh → test/dz002/dz002.sh
View File

36
test/dz003/README.md
View File

@ -1,2 +1,34 @@
Встроен запуск prometeus, grafana и т.д.
Графана доступна на http://localhost:3001/
#Полусинхронная репликация
В результате выполнения ДЗ вы настроите полусинхронную репликацию, протестируете ее влияние на производительность системы и убедитесь, что теперь вы не теряете транзакции в случае аварии.
### В данном задании тренируются навыки:
- обеспечение отказоустойчивости проекта;
- администрирование MySQL;
- настройка репликации;
- проведение нагрузочных тестов.
### План выполнения ДЗ:
1) Настраиваем асинхронную репликацию.
2) Выбираем 2 любых запроса на чтения (в идеале самых частых и тяжелых по логике работы сайта) и переносим их на чтение со слейва.
3) Делаем нагрузочный тест по странице, которую перевели на слейв до и после репликации. Замеряем нагрузку мастера (CPU, la, disc usage, memory usage).
4) ОПЦИОНАЛЬНО: в качестве конфига, который хранит IP реплики сделать массив для легкого добавления реплики. Это не самый правильный способ балансирования нагрузки. Поэтому опционально.
5) Настроить 2 слейва и 1 мастер.
6) Включить row-based репликацию.
7) Включить GTID.
8) Настроить полусинхронную репликацию.
9) Создать нагрузку на запись в любую тестовую таблицу. На стороне, которой нагружаем считать, сколько строк мы успешно записали.
10) С помощью kill -9 убиваем мастер MySQL.
11) Заканчиваем нагрузку на запись.
12) Выбираем самый свежий слейв. Промоутим его до мастера. Переключаем на него второй слейв.
13) Проверяем, есть ли потери транзакций.
Результатом сдачи ДЗ будет в виде исходного кода на github и отчета в текстовом виде, где вы отразите как вы выполняли каждый из пунктов.
Критерии оценки: Оценка происходит по принципу зачет/незачет.
### Требования:
- В отчете корректно описано, как настроена репликация.
- 2 запроса переведено на чтение со слейва.
- Нагрузочное тестирование показало, что нагрузка перешла на слейв.
- В отчете описано как включить row-based репликацию и GTID
- Проведен эксперимент по потере и непотере транзакций при аварийной остановке master.

0
test/dz003.sh → test/dz003/dz003.sh
View File

BIN
test/dz003/img/dz003_part1.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 183 KiB

27
test/dz003/report.md Normal file
View File

@ -0,0 +1,27 @@
Встроен запуск prometeus, grafana и т.д.
Графана доступна на http://localhost:3001/
$ sudo docker run --rm -v /root/scripts:/scripts williamyeh/wrk -t1 -c10 -d5m --timeout 30s http://192.168.1.66:8080/search -s /scripts/post.lua -- debug true
Running 5m test @ http://192.168.1.66:8080/search
1 threads and 10 connections
Thread Stats Avg Stdev Max +/- Stdev
Latency 5.83s 1.74s 13.29s 77.58%
Req/Sec 3.57 4.74 30.00 82.50%
511 requests in 5.00m, 18.69MB read
Requests/sec: 1.70
Transfer/sec: 63.79KB
$ sudo make app-reload
...
...
...
$ sudo docker run --rm -v /root/scripts:/scripts williamyeh/wrk -t1 -c10 -d5m --timeout 30s http://192.168.1.66:8080/search -s /scripts/post.lua -- debug true
Running 5m test @ http://192.168.1.66:8080/search
1 threads and 10 connections
Thread Stats Avg Stdev Max +/- Stdev
Latency 32.12ms 31.59ms 561.62ms 90.65%
Req/Sec 373.34 147.37 610.00 68.45%
110834 requests in 5.00m, 433.90MB read
Non-2xx or 3xx responses: 110834
Requests/sec: 369.39
Transfer/sec: 1.45MB

0
test/dz003/post.lua → test/dz003/scripts/post.lua
View File

0
cicd/alertmanager/config.yml → test/monitor/alertmanager/config.yml
View File

0
cicd/caddy/Caddyfile → test/monitor/caddy/Caddyfile
View File

0
cicd/config → test/monitor/config
View File

117
test/monitor/docker-compose.yml Normal file
View File

@ -0,0 +1,117 @@
version: '3'
services:
prometheus:
image: prom/prometheus:v2.24.1
container_name: prometheus
volumes:
- ./prometheus:/etc/prometheus
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--storage.tsdb.retention.time=200h'
- '--web.enable-lifecycle'
restart: unless-stopped
expose:
- 9090
labels:
org.label-schema.group: "monitoring"
alertmanager:
image: prom/alertmanager:v0.21.0
container_name: alertmanager
volumes:
- ./alertmanager:/etc/alertmanager
command:
- '--config.file=/etc/alertmanager/config.yml'
- '--storage.path=/alertmanager'
restart: unless-stopped
expose:
- 9093
labels:
org.label-schema.group: "monitoring"
nodeexporter:
image: prom/node-exporter:v1.1.0
container_name: nodeexporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
restart: unless-stopped
expose:
- 9100
labels:
org.label-schema.group: "monitoring"
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.38.7
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker:/var/lib/docker:ro
#- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
restart: unless-stopped
expose:
- 8080
labels:
org.label-schema.group: "monitoring"
grafana:
image: grafana/grafana:7.4.0
container_name: grafana
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
- ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
environment:
- GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
- GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
restart: unless-stopped
expose:
- 3000
labels:
org.label-schema.group: "monitoring"
ports:
- "3001:3000"
pushgateway:
image: prom/pushgateway:v1.4.0
container_name: pushgateway
restart: unless-stopped
expose:
- 9091
labels:
org.label-schema.group: "monitoring"
caddy:
image: stefanprodan/caddy
container_name: caddy
ports:
- "3000:3000"
- "9090:9090"
- "9093:9093"
- "9091:9091"
volumes:
- ./caddy:/etc/caddy
environment:
- ADMIN_USER=${ADMIN_USER:-admin}
- ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
restart: unless-stopped
labels:
org.label-schema.group: "monitoring"
volumes:
prometheus_data: { }
grafana_data: { }

1133
test/monitor/grafana/MyDashboard.json Normal file
View File

File diff suppressed because it is too large Load Diff

0
cicd/grafana/provisioning/dashboards/dashboard.yml → test/monitor/grafana/provisioning/dashboards/dashboard.yml
View File

0
cicd/grafana/provisioning/dashboards/docker_containers.json → test/monitor/grafana/provisioning/dashboards/docker_containers.json
View File

0
cicd/grafana/provisioning/dashboards/docker_host.json → test/monitor/grafana/provisioning/dashboards/docker_host.json
View File

0
cicd/grafana/provisioning/dashboards/monitor_services.json → test/monitor/grafana/provisioning/dashboards/monitor_services.json
View File

0
cicd/grafana/provisioning/dashboards/nginx_container.json → test/monitor/grafana/provisioning/dashboards/nginx_container.json
View File

0
cicd/grafana/provisioning/datasources/datasource.yml → test/monitor/grafana/provisioning/datasources/datasource.yml
View File

0
cicd/helpers/aws/README.md → test/monitor/helpers/aws/README.md
View File

0
cicd/helpers/aws/cadvisor_ecs_task_definition.json → test/monitor/helpers/aws/cadvisor_ecs_task_definition.json
View File

0
cicd/helpers/aws/node_exporter_task_definition.json → test/monitor/helpers/aws/node_exporter_task_definition.json
View File

0
cicd/helpers/aws/prometheus.yml → test/monitor/helpers/aws/prometheus.yml
View File

0
cicd/prometheus/alert.rules → test/monitor/prometheus/alert.rules
View File

0
cicd/prometheus/prometheus.yml → test/monitor/prometheus/prometheus.yml
View File