HardMax71 · Oct 22, 2026 · Oct 22, 2026
@@ -206,6 +206,7 @@ jobs:
             # These read their mounted config only at start, and compose recreates a container only when its service definition changes
             RESTART=""
             git diff --quiet "$PREVIOUS" HEAD -- backend/otel-collector-config.yaml || RESTART="$RESTART otel-collector"
+            git diff --quiet "$PREVIOUS" HEAD -- backend/otel-host-metrics.yaml || RESTART="$RESTART host-metrics"
             git diff --quiet "$PREVIOUS" HEAD -- backend/victoria-metrics || RESTART="$RESTART victoria-metrics"
             git diff --quiet "$PREVIOUS" HEAD -- backend/loki || RESTART="$RESTART loki"
             git diff --quiet "$PREVIOUS" HEAD -- backend/grafana ':(exclude)backend/grafana/provisioning/dashboards' || RESTART="$RESTART grafana"

@@ -10,7 +10,7 @@ Platform for running scripts online in isolated Kubernetes pods. Users write Pyt
 | Backend | FastAPI, Dishka DI, Beanie ODM (MongoDB), Python 3.14 (backend/.python-version), async throughout, httpx2 for HTTP |
 | Events | Kafka (KRaft, no ZooKeeper) + FastStream (Pydantic JSON) |
 | Worker | one process (`app/worker.py`): execution scheduler, k8s worker, pod monitor, result processor, DLQ retries |
-| Infra | MongoDB 9, Redis 8; optional Grafana, Jaeger, Victoria Metrics, Loki, OTel Collector (`--observability`) and Kafdrop (`--debug`) |
+| Infra | MongoDB 9, Redis 8; optional Grafana, Jaeger, Victoria Metrics, Loki, OTel Collector plus its host-metrics collector (`--observability`) and Kafdrop (`--debug`) |
 | Runtimes | Python 3.11 to 3.14, Node 22/24/26, Ruby 3.3/3.4/4.0, Bash 5.3, Go 1.26/1.27 (app/runtime_registry.py) |
 
 ## Key Directory Map
@@ -81,7 +81,7 @@ npm run test:e2e          # Playwright E2E, needs the running stack
 ./deploy.sh dev                  # start
 ./deploy.sh dev --build          # rebuild and start
 ./deploy.sh dev --wait           # start and wait for healthy
-./deploy.sh dev --observability  # plus Grafana, Jaeger, Victoria Metrics, Loki, OTel Collector
+./deploy.sh dev --observability  # plus Grafana, Jaeger, Victoria Metrics, Loki, OTel Collector, host-metrics
 ./deploy.sh dev --debug          # observability plus Kafdrop
 ./deploy.sh down                 # stop
 ./deploy.sh openapi              # regenerate docs/reference/openapi.json

@@ -399,6 +399,60 @@
       ],
       "title": "Disk I/O",
       "type": "timeseries"
+    },
+    {
+      "datasource": "Victoria Metrics",
+      "description": "Bytes in and out per host interface, from the host-metrics collector; loopback and per-container veth interfaces are left out",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 48
+      },
+      "id": 107,
+      "targets": [
+        {
+          "expr": "sum(rate(system_network_io_bytes_total[5m])) by (device, direction)",
+          "legendFormat": "{{device}} {{direction}}",
+          "refId": "A",
+          "datasource": "Victoria Metrics"
+        }
+      ],
+      "title": "Network I/O",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Victoria Metrics",
+      "description": "Used space per filesystem, as df counts it: used over used plus free, without the root-reserved blocks",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "id": 108,
+      "targets": [
+        {
+          "expr": "sum(system_filesystem_usage_bytes{state=\"used\"}) by (mountpoint) / sum(system_filesystem_usage_bytes{state=~\"used|free\"}) by (mountpoint)",
+          "legendFormat": "{{mountpoint}}",
+          "refId": "A",
+          "datasource": "Victoria Metrics"
+        }
+      ],
+      "title": "Disk Space Used",
+      "type": "timeseries"
     }
   ],
   "refresh": "10s",

@@ -9,29 +9,6 @@ receivers:
   # Every container's stdout and stderr, from Docker's fluentd log driver; the tag is the container name
   fluent_forward:
     endpoint: 0.0.0.0:24224
-
-  # Collect host metrics
-  host_metrics:
-    collection_interval: 10s
-    root_path: /hostfs
-    scrapers:
-      cpu:
-        metrics:
-          system.cpu.utilization:
-            enabled: true
-      disk:
-      filesystem:
-        exclude_mount_points:
-          mount_points: ["^(/hostfs)?/boot", "^(/hostfs)?/var/lib/kubelet"]
-          match_type: regexp
-      load:
-      memory:
-        metrics:
-          system.memory.utilization:
-            enabled: true
-      network:
-      paging:
-      processes:
 
   kafka_metrics:
     brokers: ["kafka:29092"]
@@ -922,7 +899,7 @@ service:
       exporters: [otlp_grpc/jaeger, debug]
 
     metrics:
-      receivers: [otlp, host_metrics, kafka_metrics, mongodb]
+      receivers: [otlp, kafka_metrics, mongodb]
       processors: [memory_limiter, batch, resource]
       exporters: [prometheus_remote_write, debug]
 

@@ -0,0 +1,52 @@
+receivers:
+  host_metrics:
+    collection_interval: 10s
+    root_path: /hostfs
+    scrapers:
+      cpu:
+        metrics:
+          system.cpu.utilization:
+            enabled: true
+      disk:
+      filesystem:
+        exclude_mount_points:
+          mount_points: ["^(/hostfs)?/boot", "^(/hostfs)?/var/lib/kubelet"]
+          match_type: regexp
+      load:
+      memory:
+        metrics:
+          system.memory.utilization:
+            enabled: true
+      network:
+        # A veth pair comes and goes with every container, and its traffic also crosses a bridge and the NIC
+        exclude:
+          interfaces: ["^(lo|veth.*)$"]
+          match_type: regexp
+      paging:
+      processes:
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 100
+    spike_limit_mib: 25
+
+  batch:
+
+exporters:
+  # The otel-collector service publishes OTLP gRPC on the host's loopback
+  otlp_grpc:
+    endpoint: 127.0.0.1:4317
+    tls:
+      insecure: true
+
+service:
+  # Its own metrics would listen on a port of the host
+  telemetry:
+    metrics:
+      level: none
+  pipelines:
+    metrics:
+      receivers: [host_metrics]
+      processors: [memory_limiter, batch]
+      exporters: [otlp_grpc]
@@ -391,8 +391,6 @@ services:
     command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
       - ./backend/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
-      - /proc:/hostfs/proc:ro
-      - /sys:/hostfs/sys:ro
     environment:
       # The backend creates this user on start; until then the mongodb receiver's scrapes fail and retry
       - MONGO_MONITOR_USER=${MONGO_MONITOR_USER:-otel-monitor}
@@ -415,6 +413,24 @@ services:
         condition: service_healthy
     restart: unless-stopped
 
+  host-metrics:
+    logging: *logging
+    image: otel/opentelemetry-collector-contrib:0.163.1
+    container_name: host-metrics
+    profiles: ["observability"]
+    # In the host's network namespace the network scraper reads the host's interfaces, and 127.0.0.1:4317 reaches otel-collector
+    network_mode: host
+    mem_limit: 128m
+    command: ["--config=/etc/otel-host-metrics.yaml"]
+    volumes:
+      - ./backend/otel-host-metrics.yaml:/etc/otel-host-metrics.yaml:ro
+      - /proc:/hostfs/proc:ro
+      - /sys:/hostfs/sys:ro
+    depends_on:
+      otel-collector:
+        condition: service_started
+    restart: unless-stopped
+
 # --8<-- [start:dev_volumes]
 volumes:
   mongo_data:

@@ -18,7 +18,7 @@ Everything runs from one docker-compose.yaml: the backend, the frontend, the wor
 
 ```bash
 ./deploy.sh dev                  # core stack
-./deploy.sh dev --observability  # plus Grafana, Jaeger, Loki, Victoria Metrics, OTel Collector
+./deploy.sh dev --observability  # plus Grafana, Jaeger, Loki, Victoria Metrics, OTel Collector, host-metrics
 ./deploy.sh dev --debug          # all of the above plus Kafdrop
 ```
 
@@ -63,7 +63,7 @@ Data lives in named volumes:
 | Grafana          | `curl localhost:3000/api/health`                                            |
 | Victoria Metrics | `wget http://127.0.0.1:8428/-/healthy`                                      |
 
-Loki has none because its image is distroless and has no shell, so Grafana and the collector wait for `service_started`. Jaeger, the OTel Collector, Kafdrop and the worker have no healthcheck either.
+Loki has none because its image is distroless and has no shell, so Grafana and the collector wait for `service_started`. Jaeger, the OTel Collector, host-metrics, Kafdrop and the worker have no healthcheck either.
 
 ## Memory limits
 
@@ -82,13 +82,16 @@ Every long-running service except Kafdrop has a `mem_limit`.
 | Loki                 | 256m        |                                                             |
 | Victoria Metrics     | 256m        | `--retentionPeriod=30d`                                     |
 | OTel Collector       | 192m        | `memory_limiter` at 150 MiB                                 |
+| host-metrics         | 128m        | `memory_limiter` at 100 MiB; idles near 57 MiB              |
 
 Kafdrop runs with `-Xmx512M` and no limit. All services above except Jaeger have `restart: unless-stopped`.
 
 ## Observability profile
 
 The backend's start creates (or updates) an `otelMonitor` role with `serverStatus`, `top`, `dbStats`, `indexStats` and `listCollections`, plus a user with only that role. The collector's mongodb receiver logs in as that user, since root fails `$indexStats` on `admin.system.*`. The credentials are `MONGO_MONITOR_USER` (default `otel-monitor`) and `MONGO_MONITOR_PASSWORD` in `.env`. Until the backend has run, the receiver's scrapes fail and retry.
 
+host-metrics runs with `network_mode: host`, so its network scraper reads the host's interfaces, and it reaches the collector through the port compose publishes on `127.0.0.1:4317`. It has no ports of its own: `otel-host-metrics.yaml` turns its internal metrics off and runs no extensions.
+
 Grafana starts through a small entrypoint. It resets the admin password from `GF_SECURITY_ADMIN_PASSWORD`, starts Grafana without the alert rules, waits until the dashboard provider has created the `integr8scode` folder, then copies the rules back and calls the alerting reload API. Loaded first, the rules would create their own folder with a random UID. See [Grafana integration](grafana-integration.md).
 
 The frontend gets `GRAFANA_PUBLIC_URL` from `GRAFANA_ROOT_URL` (default `http://localhost:3000/`), which is where `/grafana` on the app host redirects.
@@ -169,7 +172,7 @@ docker compose up -d --remove-orphans --no-build --wait --wait-timeout 180
 docker image prune -af
 ```
 
-Between `up` and the prune it restarts the collector, VictoriaMetrics, Loki or Grafana when the release changed a config file that service mounts, comparing the new tag with the commit the server ran before. They read their config only when they start, and compose recreates a container only when its service definition changes, never for a new file behind a mount. Without the restart, a release that changes only `otel-collector-config.yaml` keeps running the old one; release 2026.10.28 ran 2.5 hours like that, without its new span metrics. Grafana re-reads its dashboards every 10 seconds by itself, so a dashboard change doesn't restart it.
+Between `up` and the prune it restarts the collector, host-metrics, VictoriaMetrics, Loki or Grafana when the release changed a config file that service mounts, comparing the new tag with the commit the server ran before. They read their config only when they start, and compose recreates a container only when its service definition changes, never for a new file behind a mount. Without the restart, a release that changes only `otel-collector-config.yaml` keeps running the old one; release 2026.10.28 ran 2.5 hours like that, without its new span metrics. Grafana re-reads its dashboards every 10 seconds by itself, so a dashboard change doesn't restart it.
 
 Pruning before the pull and after the deploy keeps only the images the running stack uses. To roll back by hand, check out one of the two CalVer tags before the current one (older releases are deleted, and so are their images), set `IMAGE_TAG` in `.env` to it and run the same `pull` and `up`, plus `docker compose restart` for any of those four whose config the rollback changes; the old images come back from GHCR.
 

@@ -39,7 +39,7 @@ All dashboards land in one folder, `Integr8sCode`, with the fixed UID `integr8sc
 --8<-- "backend/grafana/provisioning/alerting/alerting.yml"
 ```
 
-Both rules are instant queries on `avg_over_time(system_memory_utilization_ratio{state="used"}[5m]) * 100`, evaluated every minute. One warns above 85% for 5 minutes, the other goes critical above 95% for 2 minutes. The metric comes from the collector's `host_metrics` memory scraper, which emits `system.memory.utilization` only because the collector config turns it on; it's off by default.
+Both rules are instant queries on `avg_over_time(system_memory_utilization_ratio{state="used"}[5m]) * 100`, evaluated every minute. One warns above 85% for 5 minutes, the other goes critical above 95% for 2 minutes. The metric comes from the host-metrics collector's memory scraper, which emits `system.memory.utilization` only because `otel-host-metrics.yaml` turns it on; it's off by default.
 
 Alerts go to the `email-alerts` contact point, which sends one email addressed to everyone in `GRAFANA_ALERT_RECIPIENTS` (`singleEmail: true`). SMTP is configured in grafana.ini from env vars that compose fills in: `GF_SMTP_HOST`, `MAILJET_API_KEY` as the user, `MAILJET_SECRET_KEY` as the password and `MAILJET_FROM_ADDRESS` as the sender. Locally `GF_SMTP_HOST` is empty, so alerts fire but no mail goes out. Production sets it from the `MAILJET_HOST` secret.
 

@@ -8,15 +8,16 @@ The backend (API) and the worker both push their metrics over OTLP to the OpenTe
 flowchart LR
     API["backend"] -->|"OTLP gRPC, every 60 s"| Collector["otel-collector"]
     Worker["worker"] -->|"OTLP gRPC, every 60 s"| Collector
-    Receivers["host_metrics, mongodb, kafka_metrics receivers"] --> Collector
+    Host["host-metrics (host network)"] -->|"OTLP gRPC to 127.0.0.1:4317"| Collector
+    Receivers["mongodb, kafka_metrics receivers"] --> Collector
     Collector -->|"prometheus remote write"| VM["VictoriaMetrics"]
     VM -->|"scrapes span counters every 15 s"| Collector
     VM --> Grafana
 ```
 
 `create_app` and the worker's `main()` each call `setup_metrics` (`app/core/metrics/base.py`) once. It installs a global `MeterProvider` whose `PeriodicExportingMetricReader` exports every 60 000 ms through an OTLP gRPC exporter. The exporter targets `OTEL_EXPORTER_OTLP_ENDPOINT`, which `config.toml` sets to `http://otel-collector:4317`. It also starts the system metrics instrumentation for the process's own CPU and memory. Without an endpoint it returns early and the no-op provider stays. The OpenTelemetry instrumentations take their meters from that provider. The app's own metrics have no meter at all: they come from spans, described next.
 
-The collector's metrics pipeline (`backend/otel-collector-config.yaml`) reads the `otlp`, `host_metrics`, `kafka_metrics` and `mongodb` receivers, runs `memory_limiter`, `batch` and `resource`, and exports with `prometheus_remote_write` to `http://victoria-metrics:8428/api/v1/write`. VictoriaMetrics keeps 30 days. The collector, VictoriaMetrics and Grafana sit in the `observability` compose profile, so none of this runs unless you start the stack with `./deploy.sh dev --observability`. Counters and histograms made from spans take another way, described next.
+The collector's metrics pipeline (`backend/otel-collector-config.yaml`) reads the `otlp`, `kafka_metrics` and `mongodb` receivers, runs `memory_limiter`, `batch` and `resource`, and exports with `prometheus_remote_write` to `http://victoria-metrics:8428/api/v1/write`. VictoriaMetrics keeps 30 days. Host metrics come from a second collector, the `host-metrics` service (`backend/otel-host-metrics.yaml`). It runs only the `host_metrics` receiver in the host's network namespace, since in a container the network scraper sees only that container's `eth0` and `lo`, and sends to the main collector's OTLP port on `127.0.0.1:4317`. Both collectors, VictoriaMetrics and Grafana sit in the `observability` compose profile, so none of this runs unless you start the stack with `./deploy.sh dev --observability`. Counters and histograms made from spans take another way, described next.
 
 ## Metrics made from spans
 
@@ -203,7 +204,7 @@ A `handle` span contains the discard or immediate retry it starts, so its durati
 
 Collector receivers write these, not Python, so the contract test accepts them by prefix (`COLLECTOR_PREFIXES`).
 
-`system_memory_utilization_ratio{state}` comes from the `host_metrics` memory scraper with `system.memory.utilization` turned on. The overview's "Memory Saturation" gauge and both host memory alert rules read it. The HTTP dashboard's "CPU" panel reads `system_cpu_utilization_ratio{state}`, from the cpu scraper with `system.cpu.utilization` turned on, and "Host Memory Used" reads `system_memory_usage_bytes{state="used"}`. Below them, "Load Average" reads `system_cpu_load_average_{1m,5m,15m}` from the load scraper next to `system_cpu_logical_count`, and "Disk I/O" reads `system_disk_io_bytes_total{device,direction}` from the disk scraper for whole disks only (`sd*`, `vd*`, `xvd*`, `nvme*n*`), since partitions would count the same bytes twice.
+`system_memory_utilization_ratio{state}` comes from the host-metrics collector's memory scraper with `system.memory.utilization` turned on. The overview's "Memory Saturation" gauge and both host memory alert rules read it. The HTTP dashboard's "CPU" panel reads `system_cpu_utilization_ratio{state}`, from the cpu scraper with `system.cpu.utilization` turned on, and "Host Memory Used" reads `system_memory_usage_bytes{state="used"}`. Below them, "Load Average" reads `system_cpu_load_average_{1m,5m,15m}` from the load scraper next to `system_cpu_logical_count`, and "Disk I/O" reads `system_disk_io_bytes_total{device,direction}` from the disk scraper for whole disks only (`sd*`, `vd*`, `xvd*`, `nvme*n*`), since partitions would count the same bytes twice. "Network I/O" reads `system_network_io_bytes_total{device,direction}` for the host's interfaces; host-metrics leaves out `lo` and the per-container `veth*` pairs, whose traffic also crosses a bridge and the NIC. "Disk Space Used" divides `system_filesystem_usage_bytes{state="used"}` by used plus free per mountpoint, the way `df` counts it.
 
 The `mongodb` receiver polls `mongo:27017` every 15 seconds as `MONGO_MONITOR_USER`. Several of these are optional receiver metrics that `otel-collector-config.yaml` turns on. The MongoDB dashboard reads:
 
@@ -221,7 +222,7 @@ The `kafka_metrics` receiver's consumers scraper gives `kafka_consumer_group_lag
 
 The MongoDB dashboard's "Connections" panel reads `mongodb_connection_count{type}` for `current` and `active`; `available` is left out, since it sits in the hundreds of thousands.
 
-Everything else the receivers collect lands in VictoriaMetrics with no panel, and Grafana's Explore view can query it: the other `kafka_*` broker, topic, partition and consumer group families, `mongodb_operation_count_total`, `mongodb_operation_time_milliseconds_total` and `mongodb_document_operation_count`, the rest of the cpu and disk scrapers, and the `host_metrics` filesystem, network, paging and processes scrapers. The collector container mounts only the host's `/proc` and `/sys`, so the network scraper reports the container's own interfaces, not the host's, and the filesystem scraper reports only `/`.
+Everything else the receivers collect lands in VictoriaMetrics with no panel, and Grafana's Explore view can query it: the other `kafka_*` broker, topic, partition and consumer group families, `mongodb_operation_count_total`, `mongodb_operation_time_milliseconds_total` and `mongodb_document_operation_count`, the rest of the cpu, disk, filesystem and network scrapers, and the paging and processes scrapers. host-metrics mounts only the host's `/proc` and `/sys`, so the filesystem scraper reports only `/`, the disk its container's overlay sits on.
 
 ## PromQL examples