LOKI to grab all logs from dockers

.gitignore

@@ -1,10 +1,6 @@
-
-docker/management/.env.dev
-
-docker/home-assistant/.env.dev
-*.dev
 dist/
+dest/
 *.dev
 *.pem
 *.secret
-.idea/workspace.xml
+.idea/workspace.xml

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+ "files.associations": {
+ "*.yaml": "home-assistant"
+ }
+}

README.md

@@ -5,9 +5,11 @@ A tutorial how to build smart home from scratch. Still PoC.
 
 | Decision | Rationale | Status |
 |--------------------|------------------------------------------------------------------------------------|----------------------------------------------------|
-| big-picture | Want to track vision of the system with big picture. | TBD |
-| hosting/deployment | Want to host applications with docker - defined with docker compose. | (PR3)[https://github.com/mjendza/SmartHome/pull/3] |
-| development | Verify development first. Need to use github actions to check it can use solution. | (PR4)[https://github.com/mjendza/SmartHome/pull/4] |
+| big-picture | Want to track vision of the system with big picture. | TBD |
+| hosting/deployment | Want to host applications with docker - defined with docker compose. | (PR3)[https://github.com/mjendza/SmartHome/pull/3] |
+| development | Verify development first. Need to use github actions to check it can use solution. | (PR4)[https://github.com/mjendza/SmartHome/pull/4] |
+| SingleSingOn - homeassistant | enable SingleSingOn for HomeAssistant | (PR7)[https://github.com/mjendza/SmartHome/pull/7] |
+| observability - logs | Push all logs to the one place to improve observability for the solution. | (PR8)[https://github.com/mjendza/SmartHome/pull/8] |
 
 # requirements
 - SSH development with VS Code

doc/architecture/decisions-log/18-12-2022-centralized-logging/docker.md

@@ -0,0 +1,24 @@
+## requirements
+- push all logs to the one place to improve observability for the solution 
+
+### docker
+Push logs via loki driver inside docker-compose
+```
+x-logging: &loki-logging
+ driver: loki
+ options: # note: the ip is the ip we will assign later to loki in this file
+ loki-url: "https://localhost:3100/loki/api/v1/push"
+```
+sample serivce 
+```
+services:
+ homeassistant:
+ container_name: home-assistant
+ restart: always
+ image: homeassistant/home-assistant:2022.10.5
+ ports:
+ - 8123:8123
+ environment:
+ - TZ=${TZ}
+ logging: *loki-logging
+```

docker/home-assistant/docker-compose.yaml

@@ -1,4 +1,8 @@
 version: '3.4'
+x-logging: &loki-logging
+ driver: loki
+ options: # note: the ip is the ip we will assign later to loki in this file
+ loki-url: "https://localhost:3100/loki/api/v1/push"
 
 services:
  homeassistant:
@@ -15,7 +19,7 @@ services:
  - 8123:8123
  environment:
  - TZ=${TZ}
-
+ logging: *loki-logging
  auth-proxy:
  # can base on https://github.com/grafana/grafana/issues/52681
  # https://developer.okta.com/blog/2022/07/14/add-auth-to-any-app-with-oauth2-proxy
@@ -56,7 +60,7 @@ services:
  - OAUTH2_PROXY_AUTH_LOGGING=true
  - OAUTH2_PROXY_REQUEST_LOGGING=true
  - OAUTH2_PROXY_ERRORS_TO_INFO_LOG=true
-  
+ logging: *loki-logging
  proxy-server:
  image: nginx:1.19
  container_name: proxy-server
@@ -69,13 +73,15 @@ services:
  - ./nginx-config/key.pem.secret:/etc/nginx/ssl/key.pem:ro
  - ./nginx-config/passw.secret:/etc/nginx/ssl/passw:ro
  - ./nginx-config/nginx.conf:/etc/nginx/conf.d/default.conf
+ logging: *loki-logging
  # certbot:
  # image: certbot/certbot:latest
  # volumes:
  # - ${USERDIR}/certbot/www/:/var/www/certbot/:rw
  # - ${USERDIR}/certbot/conf/:/etc/letsencrypt/:rw
 
 
+
 networks:
  default:
  external:

docker/home-assistant/home-assistant-run.sh

@@ -1,9 +1,9 @@
-docker-compose stop
+#docker-compose stop
 
-mkdir -p ../../dist
-mkdir -p ../../dist/ha
-mkdir -p ../../dist/ha/config
+# mkdir -p ../../dist
+# mkdir -p ../../dist/ha
+# mkdir -p ../../dist/ha/config
 
-cp -a ../../home-assistant-configuration/ ../../dist/ha/config/
+# cp -a ../../home-assistant-configuration/. ../../dist/ha/config/
 
 docker-compose -f docker-compose.yaml -f docker-compose.dev.override.yaml --env-file .env.dev up --detach --force-recreate

docker/home-assistant/readme.md

@@ -0,0 +1,9 @@
+# dev environment 
+## VS Code
+- https://marketplace.visualstudio.com/items?itemName=keesschollaart.vscode-home-assistant
+- 
+
+
+# setup
+## publish metrics to prometheus 
+- https://www.home-assistant.io/docs/authentication/ generate Long-Lived Access Token - via user profile

docker/management/docker-compose.yaml

@@ -1,4 +1,8 @@
 version: '3.7'
+x-logging: &loki-logging
+ driver: loki
+ options: # note: the ip is the ip we will assign later to loki in this file
+ loki-url: "https://localhost:3100/loki/api/v1/push"
 
 services:
  portainer:
@@ -13,6 +17,7 @@ services:
  - "9000:9000"
  environment:
  - TZ=${TZ}
+ logging: *loki-logging 
 networks:
  default:
  external:

docker/observability/.env

@@ -0,0 +1,5 @@
+HA_LONGLIVED_TOKEN=JWT TOKEN
+USERDIR=../../dist
+HOST_PORT_PREFIX=
+ENV_NAME_PREFIX=dev
+ENV_NETWORK_NAME_PREFIX=iot

docker/observability/docker-compose.yaml

@@ -1,19 +1,25 @@
-version: '3.7'
+version: '3.4'
+x-logging: &loki-logging
+ driver: loki
+ options: # note: the ip is the ip we will assign later to loki in this file
+ loki-url: "https://localhost:3100/loki/api/v1/push"
+
 
 networks:
  default:
  external:
- name: iot-smart-network
-
+ name: ${ENV_NETWORK_NAME_PREFIX}-smart-network
+ 
 services:
  prometheus:
  image: prom/prometheus:v2.36.2
- container_name: prometheus
+ container_name: ${ENV_NAME_PREFIX}-prometheus
  volumes:
  - ./prometheus/:/etc/prometheus/
  - ${USERDIR}/prometheus_data:/prometheus
  command:
  - '--config.file=/etc/prometheus/prometheus.yml'
+ - '--log.level=debug'
  - '--storage.tsdb.path=/prometheus'
  - '--web.console.libraries=/usr/share/prometheus/console_libraries'
  - '--web.console.templates=/usr/share/prometheus/consoles'
@@ -25,11 +31,12 @@ services:
  - alertmanager:alertmanager
  depends_on:
  - cadvisor
+ - loki
  restart: always
-
+ logging: *loki-logging
  node-exporter:
  image: quay.io/prometheus/node-exporter:latest
- container_name: node-exporter
+ container_name: ${ENV_NAME_PREFIX}-node-exporter
  volumes:
  - /proc:/host/proc:ro
  - /sys:/host/sys:ro
@@ -45,10 +52,9 @@ services:
  ports:
  - ${HOST_PORT_PREFIX}9100:9100
  restart: always
-
  alertmanager:
  image: prom/alertmanager
- container_name: alertmanager
+ container_name: ${ENV_NAME_PREFIX}-alertmanager
  ports:
  - ${HOST_PORT_PREFIX}9093:9093
  volumes:
@@ -61,9 +67,10 @@ services:
 # placement:
 # constraints:
 # - node.hostname == ${HOSTNAME}
+
  cadvisor:
  image: gcr.io/cadvisor/cadvisor
- container_name: cadvisor
+ container_name: ${ENV_NAME_PREFIX}-cadvisor
  volumes:
  - /:/rootfs:ro
  - /var/run:/var/run:rw
@@ -72,10 +79,9 @@ services:
  ports:
  - ${HOST_PORT_PREFIX}8080:8080
  restart: always
-
  grafana:
  image: grafana/grafana
- container_name: grafana
+ container_name: ${ENV_NAME_PREFIX}-grafana
  user: "0:0"
  depends_on:
  - prometheus
@@ -87,3 +93,16 @@ services:
  env_file:
  - ./grafana/config.monitoring
  restart: always
+ logging: *loki-logging
+ loki:
+ container_name: ${ENV_NAME_PREFIX}-loki
+ image: grafana/loki:2.3.0
+ restart: unless-stopped
+ ports:
+ - ${HOST_PORT_PREFIX}3100:3100
+ volumes:
+ - ./loki/config/loki-config.yaml:/etc/loki/loki-config.yaml
+ - ${USERDIR}/loki/data:/data/loki
+ - ${USERDIR}/loki/data:/loki
+ command: -config.file=/etc/loki/loki-config.yaml
+

docker/observability/grafana/provisioning/datasources/datasource.yml

@@ -18,7 +18,7 @@ datasources:
  # <int> org id. will default to orgId 1 if not specified
  orgId: 1
  # <string> url
- url: https://prometheus:9090
+ url: https://dev-prometheus:9090
  # <string> database password, if used
  password:
  # <string> database user, if used
@@ -48,3 +48,8 @@ datasources:
  version: 1
  # <bool> allow users to edit datasources from the UI.
  editable: true
+
+- name: Loki
+ type: loki
+ url: https://dev-loki:3100
+ isDefault: false

docker/observability/loki/config/loki-config.yaml

@@ -0,0 +1,37 @@
+auth_enabled: false
+
+server:
+ http_listen_port: 3100
+
+ingester:
+ lifecycler:
+ address: 127.0.0.1
+ ring:
+ kvstore:
+ store: inmemory
+ replication_factor: 1
+ final_sleep: 0s
+ chunk_idle_period: 5m
+ chunk_retain_period: 30s
+
+schema_config:
+ configs:
+ - from: 2020-05-15
+ store: boltdb
+ object_store: filesystem
+ schema: v11
+ index:
+ prefix: index_
+ period: 168h
+
+storage_config:
+ boltdb:
+ directory: /tmp/loki/index
+
+ filesystem:
+ directory: /tmp/loki/chunks
+
+limits_config:
+ enforce_metric_name: false
+ reject_old_samples: true
+ reject_old_samples_max_age: 168h

docker/observability/prometheus/prometheus.yml

@@ -21,7 +21,7 @@ alerting:
  - scheme: http
  static_configs:
  - targets:
- - "alertmanager:9093"
+ - "dev-alertmanager:9093"
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
@@ -43,15 +43,22 @@ scrape_configs:
  scrape_interval: 15s
 
  static_configs:
- - targets: ['cadvisor:8080']
+ - targets: ['dev-cadvisor:8080']
 
  - job_name: 'node-exporter'
-
  # Override the global default and scrape targets from this job every 5 seconds.
  scrape_interval: 15s
-
  static_configs:
- - targets: ['node-exporter:9100']
+ - targets: ['dev-node-exporter:9100']
+ # home assistant 
+ - job_name: "hass"
+ scrape_interval: 15s
+ metrics_path: /api/prometheus
+ # Long-Lived Access Token
+ bearer_token: 'REPLACE_ME'
+ scheme: http
+ static_configs:
+ - targets: ['dev-home-assistant:8123']
 
 # - job_name: 'pushgateway'
 # scrape_interval: 10s

docker/observability/readme.md

@@ -0,0 +1,12 @@
+# troubleshooting
+## docker
+### loki loggs
+#### don't see logs in locki
+##### check if plugin to push docker instance logs to loki
+```
+/var/log$ tail -n 100 syslog
+```
+
+```
+grep plugin= syslog
+```

docker/observability/run-dev.sh

@@ -0,0 +1 @@
+docker-compose -p "dev-observability" -f docker-compose.yaml --env-file .env.dev up --detach --force-recreate

docker/observability/run-test.sh

@@ -0,0 +1 @@
+docker-compose -p "test-observability" -f docker-compose.yaml --env-file .env.test.dev up --detach --force-recreate

docker/readme.md

@@ -0,0 +1,23 @@
+# requirements 
+## docker
+### loki plugin 
+```
+docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+```
+### shared network 
+```
+docker network create \
+ --driver=bridge \
+ --subnet=172.23.0.0/16 \
+ --ip-range=172.23.0.0/16 \
+ --gateway=172.23.0.1 \
+ iot-smart-network
+ ```
+ ```
+docker network create \
+ --driver=bridge \
+ --subnet=172.55.0.0/16 \
+ --ip-range=172.55.0.0/16 \
+ --gateway=172.55.0.1 \
+ test-smart-network
+ ```