# HELP process_cpu_user_seconds_total Total user CPU time spent in seconds. # TYPE process_cpu_user_seconds_total counter process_cpu_user_seconds_total{app="kool-engine"} 63391.53668099998 # HELP process_cpu_system_seconds_total Total system CPU time spent in seconds. # TYPE process_cpu_system_seconds_total counter process_cpu_system_seconds_total{app="kool-engine"} 14171.124953999999 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total{app="kool-engine"} 77562.66163500001 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds{app="kool-engine"} 1786637995 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes{app="kool-engine"} 346132480 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes{app="kool-engine"} 76318490624 # HELP process_heap_bytes Process heap size in bytes. # TYPE process_heap_bytes gauge process_heap_bytes{app="kool-engine"} 69889765376 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge process_open_fds{app="kool-engine"} 26 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds{app="kool-engine"} 1048576 # HELP nodejs_eventloop_lag_seconds Lag of event loop in seconds. # TYPE nodejs_eventloop_lag_seconds gauge nodejs_eventloop_lag_seconds{app="kool-engine"} 0.004354531 # HELP nodejs_eventloop_lag_min_seconds The minimum recorded event loop delay. # TYPE nodejs_eventloop_lag_min_seconds gauge nodejs_eventloop_lag_min_seconds{app="kool-engine"} 2.67e-7 # HELP nodejs_eventloop_lag_max_seconds The maximum recorded event loop delay. # TYPE nodejs_eventloop_lag_max_seconds gauge nodejs_eventloop_lag_max_seconds{app="kool-engine"} 0.236051161 # HELP nodejs_eventloop_lag_mean_seconds The mean of the recorded event loop delays. # TYPE nodejs_eventloop_lag_mean_seconds gauge nodejs_eventloop_lag_mean_seconds{app="kool-engine"} 0.000810844203591838 # HELP nodejs_eventloop_lag_stddev_seconds The standard deviation of the recorded event loop delays. # TYPE nodejs_eventloop_lag_stddev_seconds gauge nodejs_eventloop_lag_stddev_seconds{app="kool-engine"} 0.004823372197890968 # HELP nodejs_eventloop_lag_p50_seconds The 50th percentile of the recorded event loop delays. # TYPE nodejs_eventloop_lag_p50_seconds gauge nodejs_eventloop_lag_p50_seconds{app="kool-engine"} 0.000142975 # HELP nodejs_eventloop_lag_p90_seconds The 90th percentile of the recorded event loop delays. # TYPE nodejs_eventloop_lag_p90_seconds gauge nodejs_eventloop_lag_p90_seconds{app="kool-engine"} 0.000567295 # HELP nodejs_eventloop_lag_p99_seconds The 99th percentile of the recorded event loop delays. # TYPE nodejs_eventloop_lag_p99_seconds gauge nodejs_eventloop_lag_p99_seconds{app="kool-engine"} 0.007708671 # HELP nodejs_active_resources Number of active resources that are currently keeping the event loop alive, grouped by async resource type. # TYPE nodejs_active_resources gauge # HELP nodejs_active_resources_total Total number of active resources. # TYPE nodejs_active_resources_total gauge nodejs_active_resources_total{app="kool-engine"} 0 # HELP nodejs_active_handles Number of active libuv handles grouped by handle type. Every handle type is C++ class name. # TYPE nodejs_active_handles gauge # HELP nodejs_active_handles_total Total number of active handles. # TYPE nodejs_active_handles_total gauge nodejs_active_handles_total{app="kool-engine"} 0 # HELP nodejs_active_requests Number of active libuv requests grouped by request type. Every request type is C++ class name. # TYPE nodejs_active_requests gauge # HELP nodejs_active_requests_total Total number of active requests. # TYPE nodejs_active_requests_total gauge nodejs_active_requests_total{app="kool-engine"} 0 # HELP nodejs_heap_size_total_bytes Process heap size from Node.js in bytes. # TYPE nodejs_heap_size_total_bytes gauge nodejs_heap_size_total_bytes{app="kool-engine"} 64093184 # HELP nodejs_heap_size_used_bytes Process heap size used from Node.js in bytes. # TYPE nodejs_heap_size_used_bytes gauge nodejs_heap_size_used_bytes{app="kool-engine"} 100673385 # HELP nodejs_external_memory_bytes Node.js external memory size in bytes. # TYPE nodejs_external_memory_bytes gauge nodejs_external_memory_bytes{app="kool-engine"} 45707209 # HELP nodejs_version_info Node.js version info. # TYPE nodejs_version_info gauge nodejs_version_info{version="v24.3.0",major="24",minor="3",patch="0",app="kool-engine"} 1 # HELP nodejs_gc_duration_seconds Garbage collection duration by kind, one of major, minor, incremental or weakcb. # TYPE nodejs_gc_duration_seconds histogram # HELP kool_http_requests_total HTTP requests handled by the engine API # TYPE kool_http_requests_total counter kool_http_requests_total{method="GET",route="/api/health",status="200",app="kool-engine"} 36335 kool_http_requests_total{method="GET",route="/api/health/ready",status="200",app="kool-engine"} 109001 kool_http_requests_total{method="POST",route="/api/webhooks/github-app/webhook",status="200",app="kool-engine"} 334 kool_http_requests_total{method="GET",route="/*",status="404",app="kool-engine"} 1928 kool_http_requests_total{method="POST",route="/*",status="404",app="kool-engine"} 5 kool_http_requests_total{method="POST",route="/api/webhooks/github-app/webhook",status="500",app="kool-engine"} 1 kool_http_requests_total{method="POST",route="/api/*",status="401",app="kool-engine"} 1 kool_http_requests_total{method="GET",route="/api/*",status="401",app="kool-engine"} 1 # HELP kool_http_request_duration_ms HTTP request duration in ms # TYPE kool_http_request_duration_ms histogram kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="GET",route="/api/health",status="200"} 36334 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_sum{app="kool-engine",method="GET",route="/api/health",status="200"} 4040 kool_http_request_duration_ms_count{app="kool-engine",method="GET",route="/api/health",status="200"} 36335 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 46022 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 49726 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 97047 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 107765 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 108986 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109000 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109000 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109000 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109001 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109001 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109001 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109001 kool_http_request_duration_ms_sum{app="kool-engine",method="GET",route="/api/health/ready",status="200"} 2787989 kool_http_request_duration_ms_count{app="kool-engine",method="GET",route="/api/health/ready",status="200"} 109001 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 330 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_sum{app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 819 kool_http_request_duration_ms_count{app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="200"} 334 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_sum{app="kool-engine",method="GET",route="/*",status="404"} 120 kool_http_request_duration_ms_count{app="kool-engine",method="GET",route="/*",status="404"} 1928 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_sum{app="kool-engine",method="POST",route="/*",status="404"} 0 kool_http_request_duration_ms_count{app="kool-engine",method="POST",route="/*",status="404"} 5 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 0 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_sum{app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 16 kool_http_request_duration_ms_count{app="kool-engine",method="POST",route="/api/webhooks/github-app/webhook",status="500"} 1 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_sum{app="kool-engine",method="POST",route="/api/*",status="401"} 0 kool_http_request_duration_ms_count{app="kool-engine",method="POST",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="10",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="25",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="50",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="100",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="250",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="500",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="1000",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="2500",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="5000",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="10000",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="30000",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_bucket{le="+Inf",app="kool-engine",method="GET",route="/api/*",status="401"} 1 kool_http_request_duration_ms_sum{app="kool-engine",method="GET",route="/api/*",status="401"} 0 kool_http_request_duration_ms_count{app="kool-engine",method="GET",route="/api/*",status="401"} 1 # HELP kool_conversations_created_total Conversations created via POST /conversations # TYPE kool_conversations_created_total counter # HELP kool_conversation_turns_created_total Conversation turns persisted (user + agent + system) # TYPE kool_conversation_turns_created_total counter # HELP kool_hermes_runs_started_total Hermes runs started via /v1/runs # TYPE kool_hermes_runs_started_total counter kool_hermes_runs_started_total{app="kool-engine"} 0 # HELP kool_hermes_runs_completed_total Hermes runs that produced a run.completed event # TYPE kool_hermes_runs_completed_total counter # HELP kool_hermes_run_duration_ms Hermes run duration from start to completion in ms # TYPE kool_hermes_run_duration_ms histogram # HELP kool_errors_total Counted unexpected engine errors (5xx + caught fire-and-forget rejects) # TYPE kool_errors_total counter kool_errors_total{source="http",app="kool-engine"} 1 # HELP kool_hermes_provisions_total Hermes lifecycle operations (engine-driven k8s API calls) # TYPE kool_hermes_provisions_total counter kool_hermes_provisions_total{op="provision_pool",outcome="ok",app="kool-engine"} 4 # HELP kool_primer_body_miss_total Per-conversation primer found no matching artifact body despite the entity having artifacts # TYPE kool_primer_body_miss_total counter # HELP kool_ws_notify_dropped_total WS events skipped from cross-pod NOTIFY broadcast # TYPE kool_ws_notify_dropped_total counter # HELP kool_aql_task_queue_depth AQL interpreter task-queue backlog (schedule-to-start pressure) per task queue # TYPE kool_aql_task_queue_depth gauge # HELP kool_aql_workflow_failures_total AQL flow workflow executions that ended in a failed/terminal-error state # TYPE kool_aql_workflow_failures_total counter # HELP kool_aql_replay_latency_ms AQL worker replay latency (history-to-running catch-up) in ms # TYPE kool_aql_replay_latency_ms histogram # HELP kool_aql_prep_cost_usd_total Cumulative prep-generation USD spend attributed per engine path (legacy vs AQL) # TYPE kool_aql_prep_cost_usd_total counter