steel_core/server/
tick_overload.rs

1//! Skipping ticks the server is too far behind to replay.
2
3use std::time::{Duration, Instant};
4
5const OVERLOAD_THRESHOLD: Duration = Duration::from_secs(1);
6const OVERLOAD_THRESHOLD_TICKS: u32 = 20;
7const OVERLOAD_WARNING_INTERVAL: Duration = Duration::from_secs(10);
8const OVERLOAD_WARNING_INTERVAL_TICKS: u32 = 100;
9
10/// Tracks how far behind the loop is and drops the backlog past the threshold.
11pub(super) struct TickOverloadGuard {
12    last_report: Option<Instant>,
13}
14
15impl TickOverloadGuard {
16    pub(super) const fn new() -> Self {
17        Self { last_report: None }
18    }
19
20    pub(super) const fn restart_report_gap(&mut self, now: Instant) {
21        self.last_report = Some(now);
22    }
23
24    /// Skips the ticks the loop is behind by, if it is far enough behind and the
25    /// last report is old enough, and returns how many were skipped.
26    pub(super) fn skip_backlog_if_overloaded(
27        &mut self,
28        now: Instant,
29        next_tick_time: &mut Instant,
30        nanoseconds_per_tick: u64,
31    ) -> u64 {
32        if nanoseconds_per_tick == 0 {
33            return 0;
34        }
35
36        let tick = Duration::from_nanos(nanoseconds_per_tick);
37        let behind = now.saturating_duration_since(*next_tick_time);
38        let threshold = OVERLOAD_THRESHOLD + tick * OVERLOAD_THRESHOLD_TICKS;
39        let report_gap = OVERLOAD_WARNING_INTERVAL + tick * OVERLOAD_WARNING_INTERVAL_TICKS;
40        let reported_recently = self.last_report.is_some_and(|last_report| {
41            next_tick_time.saturating_duration_since(last_report) < report_gap
42        });
43        if behind <= threshold || reported_recently {
44            return 0;
45        }
46
47        let behind_nanos = u64::try_from(behind.as_nanos()).unwrap_or(u64::MAX);
48        let ticks_behind = behind_nanos / nanoseconds_per_tick;
49        log::warn!(
50            "Can't keep up! Is the server overloaded? Running {}ms or {ticks_behind} ticks behind",
51            behind.as_millis()
52        );
53
54        *next_tick_time += Duration::from_nanos(ticks_behind * nanoseconds_per_tick);
55        self.last_report = Some(*next_tick_time);
56        ticks_behind
57    }
58}
59
60#[cfg(test)]
61mod tests {
62    use super::*;
63
64    const NANOS_PER_TICK: u64 = 50_000_000;
65    const TICK: Duration = Duration::from_nanos(NANOS_PER_TICK);
66
67    fn running_behind(behind: Duration) -> (Instant, Instant, TickOverloadGuard) {
68        let next_tick_time = Instant::now();
69        (
70            next_tick_time + behind,
71            next_tick_time,
72            TickOverloadGuard::new(),
73        )
74    }
75
76    #[test]
77    fn on_time_loop_keeps_its_schedule() {
78        let (now, mut next_tick_time, mut guard) = running_behind(Duration::ZERO);
79        next_tick_time += TICK;
80
81        let skipped = guard.skip_backlog_if_overloaded(now, &mut next_tick_time, NANOS_PER_TICK);
82
83        assert_eq!(skipped, 0, "a loop that is not behind skips nothing");
84        assert_eq!(next_tick_time, now + TICK, "the schedule is left alone");
85    }
86
87    #[test]
88    fn small_backlog_is_still_replayed() {
89        let (now, mut next_tick_time, mut guard) = running_behind(Duration::from_millis(500));
90
91        let skipped = guard.skip_backlog_if_overloaded(now, &mut next_tick_time, NANOS_PER_TICK);
92
93        assert_eq!(skipped, 0, "a backlog under the threshold is not dropped");
94    }
95
96    #[test]
97    fn large_backlog_is_skipped_and_resyncs_the_clock() {
98        let (now, mut next_tick_time, mut guard) = running_behind(Duration::from_secs(5));
99
100        let skipped = guard.skip_backlog_if_overloaded(now, &mut next_tick_time, NANOS_PER_TICK);
101
102        assert_eq!(skipped, 100, "five seconds at 20 ticks per second");
103        assert_eq!(
104            next_tick_time, now,
105            "the loop resumes on the wall clock instead of replaying the backlog"
106        );
107    }
108
109    #[test]
110    fn a_second_report_waits_for_the_report_gap() {
111        let (now, mut next_tick_time, mut guard) = running_behind(Duration::from_secs(5));
112        assert_eq!(
113            guard.skip_backlog_if_overloaded(now, &mut next_tick_time, NANOS_PER_TICK),
114            100
115        );
116
117        next_tick_time = now;
118        let later = now + Duration::from_secs(5);
119        let skipped = guard.skip_backlog_if_overloaded(later, &mut next_tick_time, NANOS_PER_TICK);
120
121        assert_eq!(skipped, 0, "reports and skips are rate limited together");
122    }
123
124    #[test]
125    fn a_sprint_restarts_the_report_gap() {
126        let (now, mut next_tick_time, mut guard) = running_behind(Duration::from_secs(5));
127        guard.restart_report_gap(next_tick_time);
128
129        let skipped = guard.skip_backlog_if_overloaded(now, &mut next_tick_time, NANOS_PER_TICK);
130
131        assert_eq!(skipped, 0, "a backlog right after a sprint is replayed");
132    }
133
134    #[test]
135    fn a_backlog_is_dropped_again_once_the_report_gap_has_passed() {
136        let (now, mut next_tick_time, mut guard) = running_behind(Duration::from_secs(5));
137        assert_eq!(
138            guard.skip_backlog_if_overloaded(now, &mut next_tick_time, NANOS_PER_TICK),
139            100
140        );
141
142        let gap = OVERLOAD_WARNING_INTERVAL + TICK * OVERLOAD_WARNING_INTERVAL_TICKS;
143        next_tick_time = now + gap;
144        let later = next_tick_time + Duration::from_secs(5);
145        let skipped = guard.skip_backlog_if_overloaded(later, &mut next_tick_time, NANOS_PER_TICK);
146
147        assert_eq!(
148            skipped, 100,
149            "the gap has run out, so the backlog is dropped"
150        );
151    }
152
153    #[test]
154    fn a_zero_length_tick_is_ignored() {
155        let (now, mut next_tick_time, mut guard) = running_behind(Duration::from_secs(5));
156
157        let skipped = guard.skip_backlog_if_overloaded(now, &mut next_tick_time, 0);
158
159        assert_eq!(skipped, 0, "a zero tick length cannot be divided by");
160    }
161}
steel_core/server/tick_overload.rs

steel_core/server/
tick_overload.rs
