mirror of https://github.com/synctv-org/synctv
refactor(metrics): centralize registry and lifecycle tracking (#446)
## Summary - split Prometheus definitions into domain-owned modules backed by one registry - eagerly initialize every metric family and fail startup on invalid or duplicate definitions - add reusable RAII guards for gauges and relay durations across cancellation, panic, and retry paths - centralize HTTP, WebSocket, gRPC, and relay recording behind bounded-label helpers - enforce the 65-metric bilingual catalog from registered descriptors - persist completed explicit-message deletion and user-ban moderation steps - update the pinned Rust toolchain and CI jobs to `nightly-2026-08-25` - make the CI Clippy job fail on every Rust warning with `-D warnings` - keep Helm configuration-validation builds visible to avoid silent-run termination - separate Rust caches by artifact type and let Helm reuse codegen artifacts from the Build job - work around the nightly global next-solver memory regression and remove obsolete rustc recursion-limit overrides - preserve workspace Rust flags in Docker builds while applying linker flags to the final binary ## Compatibility - preserve all existing metric names, HELP text, label order, and histogram buckets - keep route-template and bounded status/error labels - return HTTP 500 when Prometheus exposition encoding fails - keep the latest nightly while restoring the previous coherence-only trait-solver behavior ## Nightly memory regression The failed CI jobs were terminated by the runner while compiling `synctv-api-http` (SIGTERM, exit 143). Local peak-RSS measurements for that crate were: | Configuration | Peak RSS | | --- | ---: | | `nightly-2026-08-21` | 6.62 GB | | `nightly-2026-08-25`, global next solver | 10.76 GB | | `nightly-2026-08-25`, `-Znext-solver=coherence` | 6.64 GB | Rust enabled `-Znext-solver=globally` by default on nightly in [rust-lang/rust#160619](https://github.com/rust-lang/rust/pull/160619). The same memory blow-up is tracked in [rust-lang/rust#161748](https://github.com/rust-lang/rust/issues/161748), and the official tracking issue documents `-Znext-solver=coherence` as the temporary opt-out. The workspace now applies that option in `.cargo/config.toml`. The Dockerfile previously set `RUSTFLAGS` for linker options, which overrode the workspace configuration and re-enabled the global solver inside image builds. It now uses `cargo rustc` to pass linker flags only to the final binary, preserving the workspace solver setting for every crate. All rustc `#![recursion_limit = "256"]` attributes were removed. The default limit passes with the coherence-only solver, confirming that a higher recursion limit was unrelated to the CI termination. ## Validation - `make build-workspace` - `make clippy-check` (`--workspace --all-targets -- -D warnings`) - `cargo check --locked -p synctv-api-http` - `cargo check --locked -p synctv-api-common -p synctv-api-grpc` - `cargo test -p synctv-api-common --lib` (655 passed, 165 Docker tests ignored) - `cargo test -p synctv-core metrics --lib` (9 passed) - `make fmt-check` - `docker build --check .` - `make validate-helm` - `actionlint .github/workflows/ci.yml .github/workflows/helm-ci.yml` - `npm run validate` in `docs` (124 pages, 353 links, Astro 0 errors/warnings)dependabot/github_actions/actions/cache-6
zijiren
4 weeks ago
committed by
GitHub
parent
7fb7f6c64f
commit
2445da8630
@ -1,2 +1,3 @@
|
|||||||
[build]
|
[build]
|
||||||
rustflags = ["-Zthreads=8", "-Zshare-generics=y"]
|
# The nightly global next solver exceeds CI memory on synctv-api-http; see rust-lang/rust#161748.
|
||||||
|
rustflags = ["-Zthreads=8", "-Zshare-generics=y", "-Znext-solver=coherence"]
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
[toolchain]
|
[toolchain]
|
||||||
channel = "nightly-2026-08-21"
|
channel = "nightly-2026-08-25"
|
||||||
profile = "minimal"
|
profile = "minimal"
|
||||||
components = ["clippy", "rustfmt"]
|
components = ["clippy", "rustfmt"]
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,21 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static ROOMS_ACTIVE: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| int_gauge("rooms_active", "Number of currently active rooms"));
|
||||||
|
|
||||||
|
pub static USERS_ONLINE: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| int_gauge("users_online", "Number of currently online users"));
|
||||||
|
|
||||||
|
pub static STREAMS_ACTIVE: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| int_gauge("streams_active", "Number of active live streams"));
|
||||||
|
|
||||||
|
pub static WEBRTC_PEERS_ACTIVE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"webrtc_peers_active",
|
||||||
|
"Number of active WebRTC peer connections",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CHAT_MESSAGES_TOTAL: std::sync::LazyLock<IntCounter> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter("chat_messages_total", "Total number of chat messages sent")
|
||||||
|
});
|
||||||
@ -0,0 +1,114 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static CACHE_HITS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_hits_total",
|
||||||
|
"Total number of cache hits",
|
||||||
|
&["cache_type", "level"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_MISSES: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_misses_total",
|
||||||
|
"Total number of cache misses",
|
||||||
|
&["cache_type", "level"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_EVICTIONS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_evictions_total",
|
||||||
|
"Total number of cache evictions",
|
||||||
|
&["cache_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_ERRORS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_errors_total",
|
||||||
|
"Total number of cache operation errors",
|
||||||
|
&["cache_type", "operation"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_INVALIDATIONS: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_invalidations_total",
|
||||||
|
"Total number of cache invalidations",
|
||||||
|
&["cache_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_OPERATION_DURATION: std::sync::LazyLock<HistogramVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
histogram_vec(
|
||||||
|
HistogramOpts::new(
|
||||||
|
"cache_operation_duration_seconds",
|
||||||
|
"Duration of cache operations in seconds",
|
||||||
|
),
|
||||||
|
&["operation"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_LAG_FLUSH_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_lag_flush_total",
|
||||||
|
"Total L1 cache flushes triggered by broadcast channel lag",
|
||||||
|
&["component"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_FENCE_OPERATIONS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_fence_operations_total",
|
||||||
|
"Total number of cache version-fence operations",
|
||||||
|
&["domain", "operation", "result"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_DB_FALLBACK_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_db_fallback_total",
|
||||||
|
"Total number of strong cache reads that fell back to PostgreSQL",
|
||||||
|
&["domain", "reason"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_STALE_WRITE_REJECT_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_stale_write_reject_total",
|
||||||
|
"Total number of stale version-aware cache writes rejected",
|
||||||
|
&["cache_type", "level"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_FENCE_PENDING: std::sync::LazyLock<GaugeVec> = std::sync::LazyLock::new(|| {
|
||||||
|
gauge_vec(
|
||||||
|
"cache_fence_pending",
|
||||||
|
"Whether a cache version fence domain currently has a pending write",
|
||||||
|
&["domain"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_FENCE_REPAIR_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"cache_fence_repair_total",
|
||||||
|
"Total number of read-time cache fence repair outcomes",
|
||||||
|
&["domain", "result"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CACHE_FENCE_DB_COMPARE: std::sync::LazyLock<GaugeVec> = std::sync::LazyLock::new(|| {
|
||||||
|
gauge_vec(
|
||||||
|
"cache_fence_db_compare",
|
||||||
|
"Latest cache fence patrol comparison with PostgreSQL version (1 when observed)",
|
||||||
|
&["domain", "relation"],
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,112 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static CLUSTER_CONNECTIONS: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_connections_total",
|
||||||
|
"Current number of active connections on this cluster node",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static NODE_ACTIVE_ROOMS: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_node_active_rooms",
|
||||||
|
"Current number of active rooms on this node",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static REALTIME_EVENTS_PUBLISHED: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_realtime_events_published_total",
|
||||||
|
"Total realtime events published",
|
||||||
|
&["event_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static REALTIME_EVENTS_RECEIVED: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_realtime_events_received_total",
|
||||||
|
"Total realtime events received from other nodes",
|
||||||
|
&["event_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static REALTIME_EVENTS_DROPPED: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_realtime_events_dropped_total",
|
||||||
|
"Total realtime events dropped",
|
||||||
|
&["reason"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CLUSTER_HEARTBEAT_FAILURES: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_heartbeat_failures",
|
||||||
|
"Consecutive Redis heartbeat failures for network partition detection",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LEADER_ELECTION_STATE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_leader_election_state",
|
||||||
|
"Leader election state (1 = leader, 0 = follower)",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LEADER_ELECTION_EPOCH: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_leader_election_epoch",
|
||||||
|
"Leader election epoch (fencing token), incremented on each leadership acquisition",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LEADER_ELECTION_CONSECUTIVE_FAILURES: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_leader_election_consecutive_failures",
|
||||||
|
"Consecutive leader election failures (network partition or backend outage detection)",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static CLUSTER_EPOCH_MISMATCH_QUARANTINE: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_epoch_mismatch_quarantine",
|
||||||
|
"Epoch mismatch quarantine state (1 = quarantined due to split-brain, 0 = normal)",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LEADER_ELECTION_MODE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_leader_election_mode",
|
||||||
|
"Leader election mode (0=standalone, 1=redis, 2=k8s_lease)",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static DISTRIBUTED_COUNTER_TTL_REFRESHES: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_cluster_distributed_counter_ttl_refreshes_total",
|
||||||
|
"Total distributed counter TTL refresh operations",
|
||||||
|
&["result"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static DISTRIBUTED_COUNTER_TTL_KEYS_REFRESHED: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_distributed_counter_ttl_keys_refreshed",
|
||||||
|
"Number of keys refreshed in the last TTL refresh cycle",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static DISTRIBUTED_COUNTER_TTL_CONSECUTIVE_FAILURES: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_cluster_distributed_counter_ttl_consecutive_failures",
|
||||||
|
"Consecutive TTL refresh failures (alert when >= 3)",
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,30 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static DB_CONNECTIONS_ACTIVE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"db_connections_active",
|
||||||
|
"Current number of active database connections",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static DB_POOL_UTILIZATION: std::sync::LazyLock<GaugeVec> = std::sync::LazyLock::new(|| {
|
||||||
|
gauge_vec(
|
||||||
|
"db_pool_utilization_ratio",
|
||||||
|
"Database connection pool utilization ratio (active/max)",
|
||||||
|
&["pool"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static DB_POOL_SIZE_MAX: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"db_pool_size_max",
|
||||||
|
"Maximum number of connections in the pool",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static DB_CONNECTIONS_IDLE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"db_connections_idle",
|
||||||
|
"Number of idle connections in the pool",
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static EMAIL_DELIVERY_QUEUE_DEPTH: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"email_delivery_queue_depth",
|
||||||
|
"Number of queued email delivery jobs",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static EMAIL_DELIVERY_IN_FLIGHT: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"email_delivery_in_flight",
|
||||||
|
"Number of email delivery jobs currently being processed",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static EMAIL_DELIVERY_JOBS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"email_delivery_jobs_total",
|
||||||
|
"Total email delivery job transitions",
|
||||||
|
&["kind", "status"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static EMAIL_DELIVERY_DURATION_SECONDS: std::sync::LazyLock<HistogramVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
histogram_vec(
|
||||||
|
HistogramOpts::new(
|
||||||
|
"email_delivery_duration_seconds",
|
||||||
|
"Email delivery processing duration in seconds",
|
||||||
|
)
|
||||||
|
.buckets(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0]),
|
||||||
|
&["kind", "status"],
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,35 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static FILE_OBJECT_DELETE_ATTEMPTS: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_file_object_delete_attempts_total",
|
||||||
|
"Total file object delete attempts",
|
||||||
|
&["origin", "backend"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static FILE_OBJECT_DELETE_FAILURES: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_file_object_delete_failures_total",
|
||||||
|
"Total file object delete failures",
|
||||||
|
&["origin", "backend"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static FILE_CLEANUP_JOBS_DUE: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"synctv_file_cleanup_jobs_due",
|
||||||
|
"File cleanup jobs due for retry",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static FILE_CLEANUP_JOBS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"synctv_file_cleanup_jobs_total",
|
||||||
|
"Total file cleanup retry job actions",
|
||||||
|
&["action", "origin", "backend"],
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,73 @@
|
|||||||
|
use prometheus::{Histogram, HistogramTimer, IntGauge};
|
||||||
|
|
||||||
|
/// Keeps an integer gauge balanced across early returns, cancellation, and panics.
|
||||||
|
#[derive(Debug)]
|
||||||
|
#[must_use = "the guard must be held for as long as the measured operation is active"]
|
||||||
|
pub struct GaugeGuard {
|
||||||
|
gauge: IntGauge,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GaugeGuard {
|
||||||
|
pub fn increment(gauge: &IntGauge) -> Self {
|
||||||
|
gauge.inc();
|
||||||
|
Self {
|
||||||
|
gauge: gauge.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for GaugeGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.gauge.dec();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Measures an operation's active count and duration through one lexical lifetime.
|
||||||
|
#[derive(Debug)]
|
||||||
|
#[must_use = "the guard must be held until the measured operation completes"]
|
||||||
|
pub struct InFlightTimer {
|
||||||
|
_active: GaugeGuard,
|
||||||
|
_duration: HistogramTimer,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InFlightTimer {
|
||||||
|
pub fn start(active: &IntGauge, duration: &Histogram) -> Self {
|
||||||
|
Self {
|
||||||
|
_active: GaugeGuard::increment(active),
|
||||||
|
_duration: duration.start_timer(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauge_guard_balances_the_gauge_when_dropped() {
|
||||||
|
let gauge = IntGauge::new("guard_test_gauge", "test gauge").expect("valid gauge");
|
||||||
|
|
||||||
|
{
|
||||||
|
let _guard = GaugeGuard::increment(&gauge);
|
||||||
|
assert_eq!(gauge.get(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(gauge.get(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn in_flight_timer_records_duration_and_balances_the_gauge() {
|
||||||
|
let gauge = IntGauge::new("timer_test_gauge", "test gauge").expect("valid gauge");
|
||||||
|
let histogram =
|
||||||
|
Histogram::with_opts(prometheus::HistogramOpts::new("timer_test", "test timer"))
|
||||||
|
.expect("valid histogram");
|
||||||
|
|
||||||
|
{
|
||||||
|
let _guard = InFlightTimer::start(&gauge, &histogram);
|
||||||
|
assert_eq!(gauge.get(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(gauge.get(), 0);
|
||||||
|
assert_eq!(histogram.get_sample_count(), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,128 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const WEBSOCKET_SUCCESS: &str = "success";
|
||||||
|
|
||||||
|
/// Total HTTP requests, labeled by method, path, and status code.
|
||||||
|
pub static HTTP_REQUESTS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"http_requests_total",
|
||||||
|
"Total number of HTTP requests",
|
||||||
|
&["method", "path", "status"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
/// HTTP request duration in seconds, labeled by method and path.
|
||||||
|
pub static HTTP_REQUEST_DURATION_SECONDS: std::sync::LazyLock<HistogramVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
histogram_vec(
|
||||||
|
HistogramOpts::new(
|
||||||
|
"http_request_duration_seconds",
|
||||||
|
"HTTP request duration in seconds (P50/P95/P99)",
|
||||||
|
)
|
||||||
|
.buckets(vec![
|
||||||
|
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||||
|
]),
|
||||||
|
&["method", "path"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Number of in-flight HTTP requests.
|
||||||
|
pub static HTTP_REQUESTS_IN_FLIGHT: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"http_requests_in_flight",
|
||||||
|
"Number of HTTP requests currently being processed",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Active WebSocket connections.
|
||||||
|
pub static WEBSOCKET_CONNECTIONS_ACTIVE: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"websocket_connections_active",
|
||||||
|
"Number of active WebSocket connections",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Total WebSocket connections opened, labeled by connection outcome.
|
||||||
|
pub static WEBSOCKET_CONNECTIONS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"websocket_connections_total",
|
||||||
|
"Total number of WebSocket connections opened",
|
||||||
|
&["status"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Total WebSocket errors, labeled by error type.
|
||||||
|
pub static WEBSOCKET_ERRORS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"websocket_errors_total",
|
||||||
|
"Total number of WebSocket errors",
|
||||||
|
&["error_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub fn start_request() -> GaugeGuard {
|
||||||
|
GaugeGuard::increment(&HTTP_REQUESTS_IN_FLIGHT)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_request(method: &str, path: &str, status: u16, elapsed: std::time::Duration) {
|
||||||
|
let status = status.to_string();
|
||||||
|
HTTP_REQUESTS_TOTAL
|
||||||
|
.with_label_values(&[method, path, &status])
|
||||||
|
.inc();
|
||||||
|
HTTP_REQUEST_DURATION_SECONDS
|
||||||
|
.with_label_values(&[method, path])
|
||||||
|
.observe(elapsed.as_secs_f64());
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn track_websocket_connection() -> GaugeGuard {
|
||||||
|
WEBSOCKET_CONNECTIONS_TOTAL
|
||||||
|
.with_label_values(&[WEBSOCKET_SUCCESS])
|
||||||
|
.inc();
|
||||||
|
GaugeGuard::increment(&WEBSOCKET_CONNECTIONS_ACTIVE)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_websocket_error(error_type: &'static str) {
|
||||||
|
WEBSOCKET_ERRORS_TOTAL
|
||||||
|
.with_label_values(&[error_type])
|
||||||
|
.inc();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn request_facade_uses_bounded_route_labels() {
|
||||||
|
let requests = HTTP_REQUESTS_TOTAL
|
||||||
|
.with_label_values(&["GET", "/items/{item_id}", "200"])
|
||||||
|
.get();
|
||||||
|
let observations = HTTP_REQUEST_DURATION_SECONDS
|
||||||
|
.with_label_values(&["GET", "/items/{item_id}"])
|
||||||
|
.get_sample_count();
|
||||||
|
|
||||||
|
record_request(
|
||||||
|
"GET",
|
||||||
|
"/items/{item_id}",
|
||||||
|
200,
|
||||||
|
std::time::Duration::from_millis(5),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
HTTP_REQUESTS_TOTAL
|
||||||
|
.with_label_values(&["GET", "/items/{item_id}", "200"])
|
||||||
|
.get(),
|
||||||
|
requests + 1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
HTTP_REQUEST_DURATION_SECONDS
|
||||||
|
.with_label_values(&["GET", "/items/{item_id}"])
|
||||||
|
.get_sample_count(),
|
||||||
|
observations + 1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static PUBLISHER_HEARTBEAT_FAILURES: std::sync::LazyLock<IntCounter> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter(
|
||||||
|
"synctv_publisher_heartbeat_failures_total",
|
||||||
|
"Total publisher cleanups due to heartbeat failure",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LIVESTREAM_ACTIVE_PUBLISHERS: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"livestream_active_publishers",
|
||||||
|
"Number of active livestream publishers",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LIVESTREAM_ACTIVE_VIEWERS: std::sync::LazyLock<IntGauge> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"livestream_active_viewers",
|
||||||
|
"Number of active livestream viewers",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LIVESTREAM_RELAY_FRAME_DROPS: std::sync::LazyLock<IntCounter> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter(
|
||||||
|
"livestream_relay_frame_drops_total",
|
||||||
|
"Total relay frames dropped due to backpressure",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static LIVESTREAM_FLV_SLOW_CLIENT_TERMINATIONS_TOTAL: std::sync::LazyLock<IntCounter> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter(
|
||||||
|
"livestream_flv_slow_client_terminations_total",
|
||||||
|
"Total FLV stream terminations due to slow client",
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,30 @@
|
|||||||
|
use std::{collections::HashMap, sync::Mutex};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static LOGGING_DROPPED_LINES_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"logging_dropped_lines_total",
|
||||||
|
"Total log lines dropped by a full non-blocking queue",
|
||||||
|
&["component"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
static LAST_OBSERVED: std::sync::LazyLock<Mutex<HashMap<String, usize>>> =
|
||||||
|
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||||
|
|
||||||
|
pub(crate) fn sync_dropped_lines(samples: &[(String, usize)]) {
|
||||||
|
let mut observed = LAST_OBSERVED
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
for (component, current) in samples {
|
||||||
|
let previous = observed.entry(component.clone()).or_default();
|
||||||
|
let delta = current.saturating_sub(*previous);
|
||||||
|
let counter = LOGGING_DROPPED_LINES_TOTAL.with_label_values(&[component]);
|
||||||
|
if delta > 0 {
|
||||||
|
counter.inc_by(u64::try_from(delta).unwrap_or(u64::MAX));
|
||||||
|
}
|
||||||
|
*previous = *current;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static RATE_LIMIT_REDIS_FALLBACKS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"rate_limit_redis_fallbacks_total",
|
||||||
|
"Total Redis errors that triggered in-memory rate limit fallback",
|
||||||
|
&["category"],
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,166 @@
|
|||||||
|
use std::{
|
||||||
|
collections::BTreeMap,
|
||||||
|
sync::{LazyLock, Mutex},
|
||||||
|
};
|
||||||
|
|
||||||
|
use prometheus::{
|
||||||
|
core::Collector, Encoder, GaugeVec, HistogramOpts, HistogramVec, IntCounter, IntCounterVec,
|
||||||
|
IntGauge, Opts, Registry, TextEncoder,
|
||||||
|
};
|
||||||
|
|
||||||
|
static REGISTRY: LazyLock<MetricsRegistry> = LazyLock::new(MetricsRegistry::new);
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct MetricsRegistry {
|
||||||
|
inner: Registry,
|
||||||
|
descriptors: Mutex<BTreeMap<String, MetricDescriptor>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(super) enum MetricKind {
|
||||||
|
Counter,
|
||||||
|
Gauge,
|
||||||
|
Histogram,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub(super) struct MetricDescriptor {
|
||||||
|
pub kind: MetricKind,
|
||||||
|
pub labels: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MetricsRegistry {
|
||||||
|
fn new() -> Self {
|
||||||
|
let registry = Self {
|
||||||
|
inner: Registry::new(),
|
||||||
|
descriptors: Mutex::new(BTreeMap::new()),
|
||||||
|
};
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
registry.register_collector(
|
||||||
|
prometheus::process_collector::ProcessCollector::for_self(),
|
||||||
|
"process",
|
||||||
|
);
|
||||||
|
registry
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Registry::new(),
|
||||||
|
descriptors: Mutex::new(BTreeMap::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register<T>(&self, metric: T, name: &str, kind: MetricKind) -> T
|
||||||
|
where
|
||||||
|
T: Collector + Clone + 'static,
|
||||||
|
{
|
||||||
|
let descriptors = metric.desc();
|
||||||
|
self.register_collector(metric.clone(), name);
|
||||||
|
let mut registered = self
|
||||||
|
.descriptors
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
for descriptor in descriptors {
|
||||||
|
registered.insert(
|
||||||
|
descriptor.fq_name.clone(),
|
||||||
|
MetricDescriptor {
|
||||||
|
kind,
|
||||||
|
labels: descriptor.variable_labels.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
metric
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register_collector<T>(&self, collector: T, name: &str)
|
||||||
|
where
|
||||||
|
T: Collector + 'static,
|
||||||
|
{
|
||||||
|
self.inner
|
||||||
|
.register(Box::new(collector))
|
||||||
|
.unwrap_or_else(|error| panic!("registering Prometheus metric `{name}`: {error}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn gather(&self) -> Result<String, MetricsError> {
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
TextEncoder::new().encode(&self.inner.gather(), &mut buffer)?;
|
||||||
|
Ok(String::from_utf8(buffer)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum MetricsError {
|
||||||
|
#[error("failed to encode Prometheus metrics: {0}")]
|
||||||
|
Encode(#[from] prometheus::Error),
|
||||||
|
#[error("Prometheus text encoder produced invalid UTF-8: {0}")]
|
||||||
|
InvalidUtf8(#[from] std::string::FromUtf8Error),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register<T>(metric: T, name: &str, kind: MetricKind) -> T
|
||||||
|
where
|
||||||
|
T: Collector + Clone + 'static,
|
||||||
|
{
|
||||||
|
REGISTRY.register(metric, name, kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn int_counter(name: &str, help: &str) -> IntCounter {
|
||||||
|
let metric = IntCounter::new(name, help)
|
||||||
|
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||||
|
register(metric, name, MetricKind::Counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn int_gauge(name: &str, help: &str) -> IntGauge {
|
||||||
|
let metric = IntGauge::new(name, help)
|
||||||
|
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||||
|
register(metric, name, MetricKind::Gauge)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn int_counter_vec(name: &str, help: &str, labels: &[&str]) -> IntCounterVec {
|
||||||
|
let metric = IntCounterVec::new(Opts::new(name, help), labels)
|
||||||
|
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||||
|
register(metric, name, MetricKind::Counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn gauge_vec(name: &str, help: &str, labels: &[&str]) -> GaugeVec {
|
||||||
|
let metric = GaugeVec::new(Opts::new(name, help), labels)
|
||||||
|
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||||
|
register(metric, name, MetricKind::Gauge)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn histogram_vec(opts: HistogramOpts, labels: &[&str]) -> HistogramVec {
|
||||||
|
let name = opts.common_opts.fq_name();
|
||||||
|
let metric = HistogramVec::new(opts, labels)
|
||||||
|
.unwrap_or_else(|error| panic!("defining Prometheus metric `{name}`: {error}"));
|
||||||
|
register(metric, &name, MetricKind::Histogram)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn gather() -> Result<String, MetricsError> {
|
||||||
|
REGISTRY.gather()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(super) fn descriptors() -> BTreeMap<String, MetricDescriptor> {
|
||||||
|
REGISTRY
|
||||||
|
.descriptors
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[should_panic(expected = "registering Prometheus metric `duplicate_metric`")]
|
||||||
|
fn registry_fails_fast_on_duplicate_metric_names() {
|
||||||
|
let registry = MetricsRegistry::empty();
|
||||||
|
let first = IntGauge::new("duplicate_metric", "first definition").expect("valid metric");
|
||||||
|
let duplicate =
|
||||||
|
IntGauge::new("duplicate_metric", "second definition").expect("valid metric");
|
||||||
|
|
||||||
|
registry.register(first, "duplicate_metric", MetricKind::Gauge);
|
||||||
|
registry.register(duplicate, "duplicate_metric", MetricKind::Gauge);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,31 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static REMOTE_TRANSPORT_REQUESTS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"grpc_requests_total",
|
||||||
|
"Total number of remote transport requests",
|
||||||
|
&["service", "method", "status"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static REMOTE_TRANSPORT_REQUEST_DURATION: std::sync::LazyLock<HistogramVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
histogram_vec(
|
||||||
|
HistogramOpts::new(
|
||||||
|
"grpc_request_duration_seconds",
|
||||||
|
"Remote transport request duration in seconds",
|
||||||
|
),
|
||||||
|
&["service", "method", "status"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub fn record(service: &str, method: &str, status: &str, elapsed: std::time::Duration) {
|
||||||
|
let labels = &[service, method, status];
|
||||||
|
REMOTE_TRANSPORT_REQUESTS_TOTAL
|
||||||
|
.with_label_values(labels)
|
||||||
|
.inc();
|
||||||
|
REMOTE_TRANSPORT_REQUEST_DURATION
|
||||||
|
.with_label_values(labels)
|
||||||
|
.observe(elapsed.as_secs_f64());
|
||||||
|
}
|
||||||
@ -0,0 +1,83 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum RelayProtocol {
|
||||||
|
Hls,
|
||||||
|
Rtmp,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RelayProtocol {
|
||||||
|
const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Hls => "hls",
|
||||||
|
Self::Rtmp => "rtmp",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub static STREAM_RELAY_DURATION: std::sync::LazyLock<HistogramVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
histogram_vec(
|
||||||
|
HistogramOpts::new(
|
||||||
|
"stream_relay_duration_seconds",
|
||||||
|
"Stream relay operation duration in seconds",
|
||||||
|
),
|
||||||
|
&["stream_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static ACTIVE_RELAY_STREAMS: std::sync::LazyLock<IntGauge> = std::sync::LazyLock::new(|| {
|
||||||
|
int_gauge(
|
||||||
|
"active_relay_streams",
|
||||||
|
"Current number of active relay streams",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static STREAM_ERRORS: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"stream_errors_total",
|
||||||
|
"Total number of stream errors",
|
||||||
|
&["stream_type", "error_type"],
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub fn track_relay(protocol: RelayProtocol) -> InFlightTimer {
|
||||||
|
InFlightTimer::start(
|
||||||
|
&ACTIVE_RELAY_STREAMS,
|
||||||
|
&STREAM_RELAY_DURATION.with_label_values(&[protocol.as_str()]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_error(protocol: RelayProtocol, error: &str) {
|
||||||
|
let error_type = if error.contains("timeout") {
|
||||||
|
"timeout"
|
||||||
|
} else if error.contains("connection") {
|
||||||
|
"connection"
|
||||||
|
} else {
|
||||||
|
"other"
|
||||||
|
};
|
||||||
|
STREAM_ERRORS
|
||||||
|
.with_label_values(&[protocol.as_str(), error_type])
|
||||||
|
.inc();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn relay_error_classification_has_a_bounded_value_set() {
|
||||||
|
let timeout = STREAM_ERRORS.with_label_values(&["rtmp", "timeout"]);
|
||||||
|
let connection = STREAM_ERRORS.with_label_values(&["rtmp", "connection"]);
|
||||||
|
let other = STREAM_ERRORS.with_label_values(&["rtmp", "other"]);
|
||||||
|
let before = (timeout.get(), connection.get(), other.get());
|
||||||
|
|
||||||
|
record_error(RelayProtocol::Rtmp, "request timeout");
|
||||||
|
record_error(RelayProtocol::Rtmp, "connection reset");
|
||||||
|
record_error(RelayProtocol::Rtmp, "codec failure with id 123");
|
||||||
|
|
||||||
|
assert_eq!(timeout.get(), before.0 + 1);
|
||||||
|
assert_eq!(connection.get(), before.1 + 1);
|
||||||
|
assert_eq!(other.get(), before.2 + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static STREAMHUB_RESTARTS_TOTAL: std::sync::LazyLock<IntCounterVec> =
|
||||||
|
std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"streamhub_restarts_total",
|
||||||
|
"Total number of StreamHub event loop restarts",
|
||||||
|
&["reason"],
|
||||||
|
)
|
||||||
|
});
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub static TASK_PANICS_TOTAL: std::sync::LazyLock<IntCounterVec> = std::sync::LazyLock::new(|| {
|
||||||
|
int_counter_vec(
|
||||||
|
"spawned_task_panics_total",
|
||||||
|
"Total number of spawned task panics caught by spawn_monitored",
|
||||||
|
&["task_name"],
|
||||||
|
)
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue