Flag hung seats and memory pressure

A turn that has run 15 minutes while producing under 50 tokens a minute is a
stalled API stream: flagged on the page with an Esc button, and notified once.
The container's memory use is shown in the header and notified near the limit.

Claude-Session: https://claude.ai/code/session_01QhLkBGEip8wQdaZ54z4AqJ

server.py

@@ -53,6 +53,39 @@ def clean(line):
     return FRAME.sub("", line).strip()
 
 
+SPINNER = re.compile(r"\((?:(\d+)h )?(?:(\d+)m )?(\d+)s · [↓↑] ([\d.]+)(k?) tokens")
+STALL_MIN = float(os.environ.get("RIG_POCKET_STALL_MIN", "15"))
+STALL_RATE = float(os.environ.get("RIG_POCKET_STALL_RATE", "50"))  # tokens per minute
+MEM_WARN = float(os.environ.get("RIG_POCKET_MEM_WARN", "0.9"))
+
+
+def stalled(screen):
+    """Minutes of a turn that has gone on long while barely producing tokens (a hung API stream), else None."""
+    m = None
+    for m in SPINNER.finditer(screen):
+        pass
+    if not m:
+        return None
+    h, mins, secs, tokens, k = m.groups()
+    minutes = int(h or 0) * 60 + int(mins or 0) + int(secs) / 60
+    tokens = float(tokens) * (1000 if k else 1)
+    if minutes >= STALL_MIN and tokens / minutes < STALL_RATE:
+        return round(minutes)
+    return None
+
+
+def memory():
+    """Memory use of this container (cgroup v2), as a fraction of its limit, or None."""
+    try:
+        cur = int(Path("/sys/fs/cgroup/memory.current").read_text())
+        limit = Path("/sys/fs/cgroup/memory.max").read_text().strip()
+        if limit == "max":
+            limit = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
+        return {"used": cur, "limit": int(limit), "fraction": round(cur / int(limit), 3)}
+    except (OSError, ValueError, StopIteration):
+        return None
+
+
 def parse_ask(screen, idle):
     """What a seat is waiting for, read off its screen.
 
@@ -152,6 +185,7 @@ def collect():
                 "lastActivity": n.get("lastActivity"),
                 "waiting": ask is not None,
                 "ask": ask,
+                "stalled": stalled(screen) if screen and not ask else None,
             })
         rigs.append({"name": r["name"], "status": r.get("status"), "uptime": r.get("uptime"),
                      "attention": r.get("attentionCount", 0), "seats": seats})
@@ -172,7 +206,7 @@ def collect():
             commits.append({"hash": full, "short": short, "when": when, "refs": refs, "subject": subject})
         repos.append({"name": os.path.basename(p), "commits": commits})
 
-    return {"updated": time.time(), "rigs": rigs, "queue": queue, "repos": repos}
+    return {"updated": time.time(), "rigs": rigs, "queue": queue, "repos": repos, "memory": memory()}
 
 
 def notify(title, body, priority=3, tags=""):
@@ -201,6 +235,8 @@ class Watcher:
         self.notified_waiting = set()
         self.queue_state = None
         self.notified_gates = set()
+        self.notified_stalled = set()
+        self.mem_warned = False
 
     def events(self, snap):
         out = []
@@ -241,6 +277,20 @@ class Watcher:
                     out.append((f"{n} venter på svar", ask_text(s.get("ask")), 4, "warning"))
                 if not s["waiting"]:
                     self.notified_waiting.discard(n)
+                if s.get("stalled") and n not in self.notified_stalled:
+                    self.notified_stalled.add(n)
+                    out.append((f"{n} ser ud til at hænge",
+                                f"Samme tur i {s['stalled']} min uden at producere noget. Åbn og send Esc.", 4, "hourglass"))
+                if not s.get("stalled"):
+                    self.notified_stalled.discard(n)
+
+        mem = snap.get("memory")
+        if mem and mem["fraction"] >= MEM_WARN and not self.mem_warned:
+            self.mem_warned = True
+            out.append(("Containeren er ved at løbe tør for hukommelse",
+                        f"{mem['used'] / 2**30:.1f} af {mem['limit'] / 2**30:.1f} GB brugt. Agenterne kan gå i stå.", 5, "rotating_light"))
+        elif mem and mem["fraction"] < MEM_WARN - 0.1:
+            self.mem_warned = False
         return out