mirror of https://github.com/synctv-org/synctv
refactor(metrics): centralize registry and lifecycle tracking
zijiren233
4 weeks ago
parent
7fb7f6c64f
commit
537b946e36
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,21 @@
|
||||
use super::*;
|
||||
|
||||
pub static ROOMS_ACTIVE: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| int_gauge("rooms_active", "Number of currently active rooms"));
|
||||
|
||||
pub static USERS_ONLINE: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| int_gauge("users_online", "Number of currently online users"));
|
||||
|
||||
pub static STREAMS_ACTIVE: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| int_gauge("streams_active", "Number of active live streams"));
|
||||
|
||||
pub static WEBRTC_PEERS_ACTIVE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"webrtc_peers_active",
|
||||
"Number of active WebRTC peer connections",
|
||||
)
|
||||
});
|
||||
|
||||
pub static CHAT_MESSAGES_TOTAL: std::sync::LazyLock<IntCounter> = std::sync::LazyLock::new(|| {
|
||||
int_counter("chat_messages_total", "Total number of chat messages sent")
|
||||
});
|
||||
@ -0,0 +1,114 @@
|
||||
use super::*;
|
||||
|
||||
pub static CACHE_HITS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_hits_total",
|
||||
"Total number of cache hits",
|
||||
&["cache_type", "level"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_MISSES: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_misses_total",
|
||||
"Total number of cache misses",
|
||||
&["cache_type", "level"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_EVICTIONS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_evictions_total",
|
||||
"Total number of cache evictions",
|
||||
&["cache_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_ERRORS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_errors_total",
|
||||
"Total number of cache operation errors",
|
||||
&["cache_type", "operation"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_INVALIDATIONS: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_invalidations_total",
|
||||
"Total number of cache invalidations",
|
||||
&["cache_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_OPERATION_DURATION: std::sync::LazyLock<HistogramVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
histogram_vec(
|
||||
HistogramOpts::new(
|
||||
"cache_operation_duration_seconds",
|
||||
"Duration of cache operations in seconds",
|
||||
),
|
||||
&["operation"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_LAG_FLUSH_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_lag_flush_total",
|
||||
"Total L1 cache flushes triggered by broadcast channel lag",
|
||||
&["component"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_FENCE_OPERATIONS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_fence_operations_total",
|
||||
"Total number of cache version-fence operations",
|
||||
&["domain", "operation", "result"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_DB_FALLBACK_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_db_fallback_total",
|
||||
"Total number of strong cache reads that fell back to PostgreSQL",
|
||||
&["domain", "reason"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_STALE_WRITE_REJECT_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_stale_write_reject_total",
|
||||
"Total number of stale version-aware cache writes rejected",
|
||||
&["cache_type", "level"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_FENCE_PENDING: std::sync::LazyLock<GaugeVec> = std::sync::LazyLock::new(|| {
|
||||
gauge_vec(
|
||||
"cache_fence_pending",
|
||||
"Whether a cache version fence domain currently has a pending write",
|
||||
&["domain"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_FENCE_REPAIR_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"cache_fence_repair_total",
|
||||
"Total number of read-time cache fence repair outcomes",
|
||||
&["domain", "result"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CACHE_FENCE_DB_COMPARE: std::sync::LazyLock<GaugeVec> = std::sync::LazyLock::new(|| {
|
||||
gauge_vec(
|
||||
"cache_fence_db_compare",
|
||||
"Latest cache fence patrol comparison with PostgreSQL version (1 when observed)",
|
||||
&["domain", "relation"],
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,112 @@
|
||||
use super::*;
|
||||
|
||||
pub static CLUSTER_CONNECTIONS: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_connections_total",
|
||||
"Current number of active connections on this cluster node",
|
||||
)
|
||||
});
|
||||
|
||||
pub static NODE_ACTIVE_ROOMS: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_node_active_rooms",
|
||||
"Current number of active rooms on this node",
|
||||
)
|
||||
});
|
||||
|
||||
pub static REALTIME_EVENTS_PUBLISHED: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_realtime_events_published_total",
|
||||
"Total realtime events published",
|
||||
&["event_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static REALTIME_EVENTS_RECEIVED: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_realtime_events_received_total",
|
||||
"Total realtime events received from other nodes",
|
||||
&["event_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static REALTIME_EVENTS_DROPPED: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_realtime_events_dropped_total",
|
||||
"Total realtime events dropped",
|
||||
&["reason"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static CLUSTER_HEARTBEAT_FAILURES: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_heartbeat_failures",
|
||||
"Consecutive Redis heartbeat failures for network partition detection",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LEADER_ELECTION_STATE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_leader_election_state",
|
||||
"Leader election state (1 = leader, 0 = follower)",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LEADER_ELECTION_EPOCH: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_leader_election_epoch",
|
||||
"Leader election epoch (fencing token), incremented on each leadership acquisition",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LEADER_ELECTION_CONSECUTIVE_FAILURES: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_leader_election_consecutive_failures",
|
||||
"Consecutive leader election failures (network partition or backend outage detection)",
|
||||
)
|
||||
});
|
||||
|
||||
pub static CLUSTER_EPOCH_MISMATCH_QUARANTINE: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_epoch_mismatch_quarantine",
|
||||
"Epoch mismatch quarantine state (1 = quarantined due to split-brain, 0 = normal)",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LEADER_ELECTION_MODE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_leader_election_mode",
|
||||
"Leader election mode (0=standalone, 1=redis, 2=k8s_lease)",
|
||||
)
|
||||
});
|
||||
|
||||
pub static DISTRIBUTED_COUNTER_TTL_REFRESHES: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_cluster_distributed_counter_ttl_refreshes_total",
|
||||
"Total distributed counter TTL refresh operations",
|
||||
&["result"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static DISTRIBUTED_COUNTER_TTL_KEYS_REFRESHED: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_distributed_counter_ttl_keys_refreshed",
|
||||
"Number of keys refreshed in the last TTL refresh cycle",
|
||||
)
|
||||
});
|
||||
|
||||
pub static DISTRIBUTED_COUNTER_TTL_CONSECUTIVE_FAILURES: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_cluster_distributed_counter_ttl_consecutive_failures",
|
||||
"Consecutive TTL refresh failures (alert when >= 3)",
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,30 @@
|
||||
use super::*;
|
||||
|
||||
pub static DB_CONNECTIONS_ACTIVE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"db_connections_active",
|
||||
"Current number of active database connections",
|
||||
)
|
||||
});
|
||||
|
||||
pub static DB_POOL_UTILIZATION: std::sync::LazyLock<GaugeVec> = std::sync::LazyLock::new(|| {
|
||||
gauge_vec(
|
||||
"db_pool_utilization_ratio",
|
||||
"Database connection pool utilization ratio (active/max)",
|
||||
&["pool"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static DB_POOL_SIZE_MAX: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"db_pool_size_max",
|
||||
"Maximum number of connections in the pool",
|
||||
)
|
||||
});
|
||||
|
||||
pub static DB_CONNECTIONS_IDLE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"db_connections_idle",
|
||||
"Number of idle connections in the pool",
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,38 @@
|
||||
use super::*;
|
||||
|
||||
pub static EMAIL_DELIVERY_QUEUE_DEPTH: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"email_delivery_queue_depth",
|
||||
"Number of queued email delivery jobs",
|
||||
)
|
||||
});
|
||||
|
||||
pub static EMAIL_DELIVERY_IN_FLIGHT: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"email_delivery_in_flight",
|
||||
"Number of email delivery jobs currently being processed",
|
||||
)
|
||||
});
|
||||
|
||||
pub static EMAIL_DELIVERY_JOBS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"email_delivery_jobs_total",
|
||||
"Total email delivery job transitions",
|
||||
&["kind", "status"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static EMAIL_DELIVERY_DURATION_SECONDS: std::sync::LazyLock<HistogramVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
histogram_vec(
|
||||
HistogramOpts::new(
|
||||
"email_delivery_duration_seconds",
|
||||
"Email delivery processing duration in seconds",
|
||||
)
|
||||
.buckets(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0]),
|
||||
&["kind", "status"],
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,35 @@
|
||||
use super::*;
|
||||
|
||||
pub static FILE_OBJECT_DELETE_ATTEMPTS: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_file_object_delete_attempts_total",
|
||||
"Total file object delete attempts",
|
||||
&["origin", "backend"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static FILE_OBJECT_DELETE_FAILURES: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_file_object_delete_failures_total",
|
||||
"Total file object delete failures",
|
||||
&["origin", "backend"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static FILE_CLEANUP_JOBS_DUE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"synctv_file_cleanup_jobs_due",
|
||||
"File cleanup jobs due for retry",
|
||||
)
|
||||
});
|
||||
|
||||
pub static FILE_CLEANUP_JOBS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"synctv_file_cleanup_jobs_total",
|
||||
"Total file cleanup retry job actions",
|
||||
&["action", "origin", "backend"],
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,73 @@
|
||||
use prometheus::{Histogram, HistogramTimer, IntGauge};
|
||||
|
||||
/// Keeps an integer gauge balanced across early returns, cancellation, and panics.
|
||||
#[derive(Debug)]
|
||||
#[must_use = "the guard must be held for as long as the measured operation is active"]
|
||||
pub struct GaugeGuard {
|
||||
gauge: IntGauge,
|
||||
}
|
||||
|
||||
impl GaugeGuard {
|
||||
pub fn increment(gauge: &IntGauge) -> Self {
|
||||
gauge.inc();
|
||||
Self {
|
||||
gauge: gauge.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GaugeGuard {
|
||||
fn drop(&mut self) {
|
||||
self.gauge.dec();
|
||||
}
|
||||
}
|
||||
|
||||
/// Measures an operation's active count and duration through one lexical lifetime.
|
||||
#[derive(Debug)]
|
||||
#[must_use = "the guard must be held until the measured operation completes"]
|
||||
pub struct InFlightTimer {
|
||||
_active: GaugeGuard,
|
||||
_duration: HistogramTimer,
|
||||
}
|
||||
|
||||
impl InFlightTimer {
|
||||
pub fn start(active: &IntGauge, duration: &Histogram) -> Self {
|
||||
Self {
|
||||
_active: GaugeGuard::increment(active),
|
||||
_duration: duration.start_timer(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn gauge_guard_balances_the_gauge_when_dropped() {
|
||||
let gauge = IntGauge::new("guard_test_gauge", "test gauge").expect("valid gauge");
|
||||
|
||||
{
|
||||
let _guard = GaugeGuard::increment(&gauge);
|
||||
assert_eq!(gauge.get(), 1);
|
||||
}
|
||||
|
||||
assert_eq!(gauge.get(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_flight_timer_records_duration_and_balances_the_gauge() {
|
||||
let gauge = IntGauge::new("timer_test_gauge", "test gauge").expect("valid gauge");
|
||||
let histogram =
|
||||
Histogram::with_opts(prometheus::HistogramOpts::new("timer_test", "test timer"))
|
||||
.expect("valid histogram");
|
||||
|
||||
{
|
||||
let _guard = InFlightTimer::start(&gauge, &histogram);
|
||||
assert_eq!(gauge.get(), 1);
|
||||
}
|
||||
|
||||
assert_eq!(gauge.get(), 0);
|
||||
assert_eq!(histogram.get_sample_count(), 1);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,128 @@
|
||||
use super::*;
|
||||
|
||||
const WEBSOCKET_SUCCESS: &str = "success";
|
||||
|
||||
/// Total HTTP requests, labeled by method, path, and status code.
|
||||
pub static HTTP_REQUESTS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"http_requests_total",
|
||||
"Total number of HTTP requests",
|
||||
&["method", "path", "status"],
|
||||
)
|
||||
});
|
||||
|
||||
/// HTTP request duration in seconds, labeled by method and path.
|
||||
pub static HTTP_REQUEST_DURATION_SECONDS: std::sync::LazyLock<HistogramVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
histogram_vec(
|
||||
HistogramOpts::new(
|
||||
"http_request_duration_seconds",
|
||||
"HTTP request duration in seconds (P50/P95/P99)",
|
||||
)
|
||||
.buckets(vec![
|
||||
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
]),
|
||||
&["method", "path"],
|
||||
)
|
||||
});
|
||||
|
||||
/// Number of in-flight HTTP requests.
|
||||
pub static HTTP_REQUESTS_IN_FLIGHT: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"http_requests_in_flight",
|
||||
"Number of HTTP requests currently being processed",
|
||||
)
|
||||
});
|
||||
|
||||
/// Active WebSocket connections.
|
||||
pub static WEBSOCKET_CONNECTIONS_ACTIVE: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"websocket_connections_active",
|
||||
"Number of active WebSocket connections",
|
||||
)
|
||||
});
|
||||
|
||||
/// Total WebSocket connections opened, labeled by connection outcome.
|
||||
pub static WEBSOCKET_CONNECTIONS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"websocket_connections_total",
|
||||
"Total number of WebSocket connections opened",
|
||||
&["status"],
|
||||
)
|
||||
});
|
||||
|
||||
/// Total WebSocket errors, labeled by error type.
|
||||
pub static WEBSOCKET_ERRORS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"websocket_errors_total",
|
||||
"Total number of WebSocket errors",
|
||||
&["error_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub fn start_request() -> GaugeGuard {
|
||||
GaugeGuard::increment(&HTTP_REQUESTS_IN_FLIGHT)
|
||||
}
|
||||
|
||||
pub fn record_request(method: &str, path: &str, status: u16, elapsed: std::time::Duration) {
|
||||
let status = status.to_string();
|
||||
HTTP_REQUESTS_TOTAL
|
||||
.with_label_values(&[method, path, &status])
|
||||
.inc();
|
||||
HTTP_REQUEST_DURATION_SECONDS
|
||||
.with_label_values(&[method, path])
|
||||
.observe(elapsed.as_secs_f64());
|
||||
}
|
||||
|
||||
pub fn track_websocket_connection() -> GaugeGuard {
|
||||
WEBSOCKET_CONNECTIONS_TOTAL
|
||||
.with_label_values(&[WEBSOCKET_SUCCESS])
|
||||
.inc();
|
||||
GaugeGuard::increment(&WEBSOCKET_CONNECTIONS_ACTIVE)
|
||||
}
|
||||
|
||||
pub fn record_websocket_error(error_type: &'static str) {
|
||||
WEBSOCKET_ERRORS_TOTAL
|
||||
.with_label_values(&[error_type])
|
||||
.inc();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn request_facade_uses_bounded_route_labels() {
|
||||
let requests = HTTP_REQUESTS_TOTAL
|
||||
.with_label_values(&["GET", "/items/{item_id}", "200"])
|
||||
.get();
|
||||
let observations = HTTP_REQUEST_DURATION_SECONDS
|
||||
.with_label_values(&["GET", "/items/{item_id}"])
|
||||
.get_sample_count();
|
||||
|
||||
record_request(
|
||||
"GET",
|
||||
"/items/{item_id}",
|
||||
200,
|
||||
std::time::Duration::from_millis(5),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
HTTP_REQUESTS_TOTAL
|
||||
.with_label_values(&["GET", "/items/{item_id}", "200"])
|
||||
.get(),
|
||||
requests + 1
|
||||
);
|
||||
assert_eq!(
|
||||
HTTP_REQUEST_DURATION_SECONDS
|
||||
.with_label_values(&["GET", "/items/{item_id}"])
|
||||
.get_sample_count(),
|
||||
observations + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
use super::*;
|
||||
|
||||
pub static PUBLISHER_HEARTBEAT_FAILURES: std::sync::LazyLock<IntCounter> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter(
|
||||
"synctv_publisher_heartbeat_failures_total",
|
||||
"Total publisher cleanups due to heartbeat failure",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LIVESTREAM_ACTIVE_PUBLISHERS: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"livestream_active_publishers",
|
||||
"Number of active livestream publishers",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LIVESTREAM_ACTIVE_VIEWERS: std::sync::LazyLock<IntGauge> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"livestream_active_viewers",
|
||||
"Number of active livestream viewers",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LIVESTREAM_RELAY_FRAME_DROPS: std::sync::LazyLock<IntCounter> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter(
|
||||
"livestream_relay_frame_drops_total",
|
||||
"Total relay frames dropped due to backpressure",
|
||||
)
|
||||
});
|
||||
|
||||
pub static LIVESTREAM_FLV_SLOW_CLIENT_TERMINATIONS_TOTAL: std::sync::LazyLock<IntCounter> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter(
|
||||
"livestream_flv_slow_client_terminations_total",
|
||||
"Total FLV stream terminations due to slow client",
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,30 @@
|
||||
use std::{collections::HashMap, sync::Mutex};
|
||||
|
||||
use super::*;
|
||||
|
||||
pub static LOGGING_DROPPED_LINES_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"logging_dropped_lines_total",
|
||||
"Total log lines dropped by a full non-blocking queue",
|
||||
&["component"],
|
||||
)
|
||||
});
|
||||
|
||||
static LAST_OBSERVED: std::sync::LazyLock<Mutex<HashMap<String, usize>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(crate) fn sync_dropped_lines(samples: &[(String, usize)]) {
|
||||
let mut observed = LAST_OBSERVED
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for (component, current) in samples {
|
||||
let previous = observed.entry(component.clone()).or_default();
|
||||
let delta = current.saturating_sub(*previous);
|
||||
let counter = LOGGING_DROPPED_LINES_TOTAL.with_label_values(&[component]);
|
||||
if delta > 0 {
|
||||
counter.inc_by(u64::try_from(delta).unwrap_or(u64::MAX));
|
||||
}
|
||||
*previous = *current;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
use super::*;
|
||||
|
||||
pub static RATE_LIMIT_REDIS_FALLBACKS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"rate_limit_redis_fallbacks_total",
|
||||
"Total Redis errors that triggered in-memory rate limit fallback",
|
||||
&["category"],
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,166 @@
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{LazyLock, Mutex},
|
||||
};
|
||||
|
||||
use prometheus::{
|
||||
core::Collector, Encoder, GaugeVec, HistogramOpts, HistogramVec, IntCounter, IntCounterVec,
|
||||
IntGauge, Opts, Registry, TextEncoder,
|
||||
};
|
||||
|
||||
static REGISTRY: LazyLock<MetricsRegistry> = LazyLock::new(MetricsRegistry::new);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MetricsRegistry {
|
||||
inner: Registry,
|
||||
descriptors: Mutex<BTreeMap<String, MetricDescriptor>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum MetricKind {
|
||||
Counter,
|
||||
Gauge,
|
||||
Histogram,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(super) struct MetricDescriptor {
|
||||
pub kind: MetricKind,
|
||||
pub labels: Vec<String>,
|
||||
}
|
||||
|
||||
impl MetricsRegistry {
|
||||
fn new() -> Self {
|
||||
let registry = Self {
|
||||
inner: Registry::new(),
|
||||
descriptors: Mutex::new(BTreeMap::new()),
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
registry.register_collector(
|
||||
prometheus::process_collector::ProcessCollector::for_self(),
|
||||
"process",
|
||||
);
|
||||
registry
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn empty() -> Self {
|
||||
Self {
|
||||
inner: Registry::new(),
|
||||
descriptors: Mutex::new(BTreeMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn register<T>(&self, metric: T, name: &str, kind: MetricKind) -> T
|
||||
where
|
||||
T: Collector + Clone + 'static,
|
||||
{
|
||||
let descriptors = metric.desc();
|
||||
self.register_collector(metric.clone(), name);
|
||||
let mut registered = self
|
||||
.descriptors
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for descriptor in descriptors {
|
||||
registered.insert(
|
||||
descriptor.fq_name.clone(),
|
||||
MetricDescriptor {
|
||||
kind,
|
||||
labels: descriptor.variable_labels.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
metric
|
||||
}
|
||||
|
||||
fn register_collector<T>(&self, collector: T, name: &str)
|
||||
where
|
||||
T: Collector + 'static,
|
||||
{
|
||||
self.inner
|
||||
.register(Box::new(collector))
|
||||
.unwrap_or_else(|error| panic!("registering Prometheus metric `{name}`: {error}"));
|
||||
}
|
||||
|
||||
fn gather(&self) -> Result<String, MetricsError> {
|
||||
let mut buffer = Vec::new();
|
||||
TextEncoder::new().encode(&self.inner.gather(), &mut buffer)?;
|
||||
Ok(String::from_utf8(buffer)?)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum MetricsError {
|
||||
#[error("failed to encode Prometheus metrics: {0}")]
|
||||
Encode(#[from] prometheus::Error),
|
||||
#[error("Prometheus text encoder produced invalid UTF-8: {0}")]
|
||||
InvalidUtf8(#[from] std::string::FromUtf8Error),
|
||||
}
|
||||
|
||||
fn register<T>(metric: T, name: &str, kind: MetricKind) -> T
|
||||
where
|
||||
T: Collector + Clone + 'static,
|
||||
{
|
||||
REGISTRY.register(metric, name, kind)
|
||||
}
|
||||
|
||||
pub(super) fn int_counter(name: &str, help: &str) -> IntCounter {
|
||||
let metric = IntCounter::new(name, help)
|
||||
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||
register(metric, name, MetricKind::Counter)
|
||||
}
|
||||
|
||||
pub(super) fn int_gauge(name: &str, help: &str) -> IntGauge {
|
||||
let metric = IntGauge::new(name, help)
|
||||
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||
register(metric, name, MetricKind::Gauge)
|
||||
}
|
||||
|
||||
pub(super) fn int_counter_vec(name: &str, help: &str, labels: &[&str]) -> IntCounterVec {
|
||||
let metric = IntCounterVec::new(Opts::new(name, help), labels)
|
||||
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||
register(metric, name, MetricKind::Counter)
|
||||
}
|
||||
|
||||
pub(super) fn gauge_vec(name: &str, help: &str, labels: &[&str]) -> GaugeVec {
|
||||
let metric = GaugeVec::new(Opts::new(name, help), labels)
|
||||
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||
register(metric, name, MetricKind::Gauge)
|
||||
}
|
||||
|
||||
pub(super) fn histogram_vec(opts: HistogramOpts, labels: &[&str]) -> HistogramVec {
|
||||
let name = opts.common_opts.fq_name();
|
||||
let metric = HistogramVec::new(opts, labels)
|
||||
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||
register(metric, &name, MetricKind::Histogram)
|
||||
}
|
||||
|
||||
pub(super) fn gather() -> Result<String, MetricsError> {
|
||||
REGISTRY.gather()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn descriptors() -> BTreeMap<String, MetricDescriptor> {
|
||||
REGISTRY
|
||||
.descriptors
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "registering Prometheus metric `duplicate_metric`")]
|
||||
fn registry_fails_fast_on_duplicate_metric_names() {
|
||||
let registry = MetricsRegistry::empty();
|
||||
let first = IntGauge::new("duplicate_metric", "first definition").expect("valid metric");
|
||||
let duplicate =
|
||||
IntGauge::new("duplicate_metric", "second definition").expect("valid metric");
|
||||
|
||||
registry.register(first, "duplicate_metric", MetricKind::Gauge);
|
||||
registry.register(duplicate, "duplicate_metric", MetricKind::Gauge);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
use super::*;
|
||||
|
||||
pub static REMOTE_TRANSPORT_REQUESTS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"grpc_requests_total",
|
||||
"Total number of remote transport requests",
|
||||
&["service", "method", "status"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static REMOTE_TRANSPORT_REQUEST_DURATION: std::sync::LazyLock<HistogramVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
histogram_vec(
|
||||
HistogramOpts::new(
|
||||
"grpc_request_duration_seconds",
|
||||
"Remote transport request duration in seconds",
|
||||
),
|
||||
&["service", "method", "status"],
|
||||
)
|
||||
});
|
||||
|
||||
pub fn record(service: &str, method: &str, status: &str, elapsed: std::time::Duration) {
|
||||
let labels = &[service, method, status];
|
||||
REMOTE_TRANSPORT_REQUESTS_TOTAL
|
||||
.with_label_values(labels)
|
||||
.inc();
|
||||
REMOTE_TRANSPORT_REQUEST_DURATION
|
||||
.with_label_values(labels)
|
||||
.observe(elapsed.as_secs_f64());
|
||||
}
|
||||
@ -0,0 +1,83 @@
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum RelayProtocol {
|
||||
Hls,
|
||||
Rtmp,
|
||||
}
|
||||
|
||||
impl RelayProtocol {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Hls => "hls",
|
||||
Self::Rtmp => "rtmp",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static STREAM_RELAY_DURATION: std::sync::LazyLock<HistogramVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
histogram_vec(
|
||||
HistogramOpts::new(
|
||||
"stream_relay_duration_seconds",
|
||||
"Stream relay operation duration in seconds",
|
||||
),
|
||||
&["stream_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub static ACTIVE_RELAY_STREAMS: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||
int_gauge(
|
||||
"active_relay_streams",
|
||||
"Current number of active relay streams",
|
||||
)
|
||||
});
|
||||
|
||||
pub static STREAM_ERRORS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"stream_errors_total",
|
||||
"Total number of stream errors",
|
||||
&["stream_type", "error_type"],
|
||||
)
|
||||
});
|
||||
|
||||
pub fn track_relay(protocol: RelayProtocol) -> InFlightTimer {
|
||||
InFlightTimer::start(
|
||||
&ACTIVE_RELAY_STREAMS,
|
||||
&STREAM_RELAY_DURATION.with_label_values(&[protocol.as_str()]),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn record_error(protocol: RelayProtocol, error: &str) {
|
||||
let error_type = if error.contains("timeout") {
|
||||
"timeout"
|
||||
} else if error.contains("connection") {
|
||||
"connection"
|
||||
} else {
|
||||
"other"
|
||||
};
|
||||
STREAM_ERRORS
|
||||
.with_label_values(&[protocol.as_str(), error_type])
|
||||
.inc();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn relay_error_classification_has_a_bounded_value_set() {
|
||||
let timeout = STREAM_ERRORS.with_label_values(&["rtmp", "timeout"]);
|
||||
let connection = STREAM_ERRORS.with_label_values(&["rtmp", "connection"]);
|
||||
let other = STREAM_ERRORS.with_label_values(&["rtmp", "other"]);
|
||||
let before = (timeout.get(), connection.get(), other.get());
|
||||
|
||||
record_error(RelayProtocol::Rtmp, "request timeout");
|
||||
record_error(RelayProtocol::Rtmp, "connection reset");
|
||||
record_error(RelayProtocol::Rtmp, "codec failure with id 123");
|
||||
|
||||
assert_eq!(timeout.get(), before.0 + 1);
|
||||
assert_eq!(connection.get(), before.1 + 1);
|
||||
assert_eq!(other.get(), before.2 + 1);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
use super::*;
|
||||
|
||||
pub static STREAMHUB_RESTARTS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||
std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"streamhub_restarts_total",
|
||||
"Total number of StreamHub event loop restarts",
|
||||
&["reason"],
|
||||
)
|
||||
});
|
||||
@ -0,0 +1,9 @@
|
||||
use super::*;
|
||||
|
||||
pub static TASK_PANICS_TOTAL: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||
int_counter_vec(
|
||||
"spawned_task_panics_total",
|
||||
"Total number of spawned task panics caught by spawn_monitored",
|
||||
&["task_name"],
|
||||
)
|
||||
});
|
||||
Loading…
Reference in New Issue