Sfoglia il codice sorgente

Add agi runtime manager

Toby Chui 2 settimane fa
parent
commit
f265c2d2b9

+ 48 - 0
src/agi_runtime_manager.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"net/http"
+
+	prout "imuslab.com/arozos/mod/prouter"
+	"imuslab.com/arozos/mod/utils"
+)
+
+/*
+	AGI Runtime Manager
+
+	Registers the "AGI Runtimes" tab in System Settings > Developer Options
+	and exposes two authenticated endpoints:
+
+	  GET  /system/ajgi/runtime/list    – list running VMs (filtered by role)
+	  POST /system/ajgi/runtime/stop    – force-stop a VM by execid
+
+	Regular users may list and stop their own VMs.
+	Admins may list and stop any VM.
+*/
+
+func AGIRuntimeManagerInit() {
+	// Register the settings tab in the "Advance" (Developer Options) group.
+	// RequireAdmin: false so regular users can view their own running scripts.
+	registerSetting(settingModule{
+		Name:         "AGI Runtimes",
+		Desc:         "Monitor and force-stop running AGI script VM instances",
+		IconPath:     "SystemAO/advance/img/small_icon.png",
+		Group:        "Advance",
+		StartDir:     "SystemAO/advance/agi_runtime.html",
+		RequireAdmin: false,
+	})
+
+	// Authenticated, non-admin router so every logged-in user can reach these
+	// endpoints. Permission filtering is enforced inside the handler methods.
+	authRouter := prout.NewModuleRouter(prout.RouterOption{
+		ModuleName:  "System Settings",
+		AdminOnly:   false,
+		UserHandler: userHandler,
+		DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
+			utils.SendErrorResponse(w, "Permission Denied")
+		},
+	})
+
+	authRouter.HandleFunc("/system/ajgi/runtime/list", AGIGateway.HandleListRuntimes)
+	authRouter.HandleFunc("/system/ajgi/runtime/stop", AGIGateway.HandleForceStopRuntime)
+}

+ 47 - 2
src/mod/agi/agi.go

@@ -80,6 +80,7 @@ type Gateway struct {
 	Option           *AgiSysInfo
 	endpointStats    map[string]*EndpointStats // per-UUID execution statistics (in-memory)
 	statsMux         sync.RWMutex              // guards endpointStats
+	vmReg            *vmRegistry               // live VM lifecycle registry
 }
 
 func NewGateway(option AgiSysInfo) (*Gateway, error) {
@@ -90,6 +91,7 @@ func NewGateway(option AgiSysInfo) (*Gateway, error) {
 		LoadedAGILibrary: map[string]AgiLibInjectionIntergface{},
 		Option:           &option,
 		endpointStats:    make(map[string]*EndpointStats),
+		vmReg:            newVMRegistry(),
 	}
 
 	//Start all WebApps Registration
@@ -290,10 +292,37 @@ func (g *Gateway) ExecuteAGIScript(scriptContent string, fsh *filesystem.FileSys
 
 	//Create a new vm for this request
 	vm := otto.New()
-	//Inject standard libs into the vm
-	g.injectStandardLibs(vm, scriptFile, scriptScope)
+	vm.Interrupt = make(chan func(), 1) // required for force-stop support
+	//Inject standard libs into the vm; capture execID for registry correlation
+	execID := g.injectStandardLibs(vm, scriptFile, scriptScope)
 	g.injectUserFunctions(vm, fsh, scriptFile, scriptScope, thisuser, w, r)
 
+	username := ""
+	if thisuser != nil {
+		username = thisuser.Username
+	}
+
+	// Register in the VM lifecycle registry so it can be listed and force-stopped
+	g.vmReg.register(&VMRecord{
+		ExecID:      execID,
+		ScriptFile:  scriptFile,
+		Username:    username,
+		StartTime:   time.Now(),
+		interruptCh: vm.Interrupt,
+	})
+	defer func() {
+		g.vmReg.unregister(execID)
+		if caught := recover(); caught != nil {
+			if caught == errForceStop {
+				logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] VM %s force-stopped (script: %s, user: %s)", execID, scriptFile, username), nil)
+				w.WriteHeader(http.StatusServiceUnavailable)
+				w.Write([]byte("503 - Script execution was force-terminated"))
+			} else {
+				panic(caught) // re-panic anything we don't own
+			}
+		}
+	}()
+
 	//Detect cotent type
 	contentType := r.Header.Get("Content-type")
 	if strings.Contains(contentType, "application/json") {
@@ -390,8 +419,18 @@ func (g *Gateway) ExecuteAGIScriptAsUser(fsh *filesystem.FileSystemHandler, scri
 	//Inject interrupt Channel
 	vm.Interrupt = make(chan func(), 1)
 
+	// Register in the VM lifecycle registry
+	g.vmReg.register(&VMRecord{
+		ExecID:      execID,
+		ScriptFile:  scriptFile,
+		Username:    targetUser.Username,
+		StartTime:   time.Now(),
+		interruptCh: vm.Interrupt,
+	})
+
 	//Create a panic recovery logic
 	defer func() {
+		g.vmReg.unregister(execID)
 		if caught := recover(); caught != nil {
 			if caught == errTimeout {
 				logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] Execution timeout: %s (user: %s)", scriptFile, targetUser.Username), nil)
@@ -399,6 +438,12 @@ func (g *Gateway) ExecuteAGIScriptAsUser(fsh *filesystem.FileSystemHandler, scri
 			} else if caught == errExitcall {
 				//Exit gracefully
 				return
+			} else if caught == errForceStop {
+				logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] VM %s force-stopped (script: %s, user: %s)", execID, scriptFile, targetUser.Username), nil)
+				if w != nil {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					w.Write([]byte("503 - Script execution was force-terminated"))
+				}
 			} else {
 				//Something screwed. Return Internal Server Error
 				logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] VM crash in %s (user: %s): %v", scriptFile, targetUser.Username, caught), nil)

+ 158 - 0
src/mod/agi/agi.vm_registry.go

@@ -0,0 +1,158 @@
+package agi
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"imuslab.com/arozos/mod/utils"
+)
+
+/*
+AGI VM Registry
+
+Tracks every Otto VM that is actively executing a script so that
+administrators (and users for their own scripts) can inspect running
+VMs and force-stop any that are stuck in an infinite loop or
+otherwise unresponsive.
+
+Force-stop works by sending a function into the VM's interrupt
+channel; Otto checks that channel between JS operations and, when a
+value is found, calls the function — which panics with errForceStop.
+The panic is caught by a deferred recovery block in each Execute*
+function and results in a 503 response rather than a goroutine crash.
+*/
+
+// errForceStop is the sentinel value panicked inside a forcibly stopped VM.
+var errForceStop = errors.New("errForceStop")
+
+// VMRecord holds metadata about one live AGI VM instance.
+type VMRecord struct {
+	ExecID      string
+	ScriptFile  string
+	Username    string
+	StartTime   time.Time
+	interruptCh chan func() // alias to vm.Interrupt — never nil after registration
+}
+
+// VMInfo is the JSON-serialisable view of a VMRecord sent to API callers.
+type VMInfo struct {
+	ExecID         string `json:"execID"`
+	ScriptFile     string `json:"scriptFile"`
+	Username       string `json:"username"`
+	StartTime      int64  `json:"startTime"`      // Unix seconds
+	ElapsedSeconds int64  `json:"elapsedSeconds"` // seconds since StartTime
+}
+
+func toVMInfo(rec *VMRecord) VMInfo {
+	return VMInfo{
+		ExecID:         rec.ExecID,
+		ScriptFile:     rec.ScriptFile,
+		Username:       rec.Username,
+		StartTime:      rec.StartTime.Unix(),
+		ElapsedSeconds: int64(time.Since(rec.StartTime).Seconds()),
+	}
+}
+
+// vmRegistry is a goroutine-safe map of execID → *VMRecord.
+type vmRegistry struct {
+	mu      sync.RWMutex
+	records map[string]*VMRecord
+}
+
+func newVMRegistry() *vmRegistry {
+	return &vmRegistry{records: make(map[string]*VMRecord)}
+}
+
+// register adds a record.  Called just before vm.Run() in each Execute* path.
+func (r *vmRegistry) register(rec *VMRecord) {
+	r.mu.Lock()
+	r.records[rec.ExecID] = rec
+	r.mu.Unlock()
+}
+
+// unregister removes a record.  Always called via defer so it fires even on panic.
+func (r *vmRegistry) unregister(execID string) {
+	r.mu.Lock()
+	delete(r.records, execID)
+	r.mu.Unlock()
+}
+
+// list returns VMInfo for every VM visible to the requester.
+// Admins see all; regular users see only their own records.
+func (r *vmRegistry) list(requesterUsername string, isAdmin bool) []VMInfo {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	result := make([]VMInfo, 0, len(r.records))
+	for _, rec := range r.records {
+		if isAdmin || rec.Username == requesterUsername {
+			result = append(result, toVMInfo(rec))
+		}
+	}
+	return result
+}
+
+// forceStop sends an interrupt to the VM with the given execID.
+// Regular users may only stop their own VMs; admins may stop any.
+func (r *vmRegistry) forceStop(execID, requesterUsername string, isAdmin bool) error {
+	r.mu.RLock()
+	rec, ok := r.records[execID]
+	r.mu.RUnlock()
+	if !ok {
+		return errors.New("VM not found: " + execID)
+	}
+	if !isAdmin && rec.Username != requesterUsername {
+		return errors.New("permission denied: you can only stop your own VMs")
+	}
+	select {
+	case rec.interruptCh <- func() { panic(errForceStop) }:
+		log.Printf("[AGI] VM %s (script: %s, user: %s) force-stopped by %s",
+			execID, rec.ScriptFile, rec.Username, requesterUsername)
+		return nil
+	default:
+		return errors.New("interrupt channel full — VM may already be stopping")
+	}
+}
+
+// ── HTTP Handlers ──────────────────────────────────────────────────────────
+
+// HandleListRuntimes returns the list of running VMs visible to the caller.
+// GET /system/ajgi/runtime/list
+func (g *Gateway) HandleListRuntimes(w http.ResponseWriter, r *http.Request) {
+	thisuser, err := g.Option.UserHandler.GetUserInfoFromRequest(w, r)
+	if err != nil {
+		http.Error(w, "401 Unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	infos := g.vmReg.list(thisuser.Username, thisuser.IsAdmin())
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(infos)
+}
+
+// HandleForceStopRuntime terminates a VM identified by the execid POST parameter.
+// POST /system/ajgi/runtime/stop
+func (g *Gateway) HandleForceStopRuntime(w http.ResponseWriter, r *http.Request) {
+	thisuser, err := g.Option.UserHandler.GetUserInfoFromRequest(w, r)
+	if err != nil {
+		http.Error(w, "401 Unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	execID, err := utils.PostPara(r, "execid")
+	if err != nil {
+		utils.SendErrorResponse(w, "missing execid parameter")
+		return
+	}
+
+	if stopErr := g.vmReg.forceStop(execID, thisuser.Username, thisuser.IsAdmin()); stopErr != nil {
+		utils.SendErrorResponse(w, stopErr.Error())
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write([]byte(`{"ok":true}`))
+}

+ 349 - 0
src/web/SystemAO/advance/agi_runtime.html

@@ -0,0 +1,349 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>AGI Runtimes</title>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <link rel="stylesheet" href="../../script/semantic/semantic.css">
+        <script src="../../script/jquery.min.js"></script>
+        <script src="../../script/semantic/semantic.js"></script>
+        <!-- applocale must be loaded before the inline script so NewAppLocale() is defined -->
+        <script src="../../script/applocale.js"></script>
+        <style>
+            #sys-setting-page { padding: 20px; }
+
+            /* ── Sortable column headers ──────────────────── */
+            th.sortable {
+                cursor: pointer;
+                user-select: none;
+                white-space: nowrap;
+            }
+            th.sortable:hover { background: rgba(0,0,0,0.04) !important; }
+            th.sortable .sort-icon {
+                display: inline-block;
+                margin-left: 4px;
+                opacity: 0.35;
+                font-style: normal;
+                font-size: 0.82em;
+            }
+            th.sortable.asc  .sort-icon,
+            th.sortable.desc .sort-icon { opacity: 1; }
+            th.sortable.asc  .sort-icon::after { content: " ↑"; }
+            th.sortable.desc .sort-icon::after { content: " ↓"; }
+            th.sortable:not(.asc):not(.desc) .sort-icon::after { content: " ↕"; }
+
+            /* ── Running-time colour coding ───────────────── */
+            .elapsed-warn   { color: #e67e22; font-weight: 600; }
+            .elapsed-danger { color: #e74c3c; font-weight: 600; }
+
+            /* ── Exec-ID mono ─────────────────────────────── */
+            .execid-cell {
+                font-family: 'Cascadia Code', 'Consolas', monospace;
+                font-size: 0.82em;
+                color: #555;
+            }
+
+            /* ── Refresh bar ──────────────────────────────── */
+            .refresh-bar {
+                display: flex;
+                align-items: center;
+                justify-content: space-between;
+                margin-bottom: 10px;
+            }
+            .refresh-bar .left { display: flex; align-items: center; gap: 14px; }
+            #countdownText { color: #aaa; font-size: 0.88em; }
+
+            /* ── Empty / error states ─────────────────────── */
+            #emptyRow td { text-align: center; color: #aaa; padding: 28px 0 !important; }
+
+            /* ── Toast notification ───────────────────────── */
+            #toast {
+                display: none;
+                position: fixed;
+                bottom: 24px;
+                right: 24px;
+                z-index: 9999;
+                min-width: 220px;
+            }
+        </style>
+    </head>
+    <body>
+        <div id="sys-setting-page">
+
+            <!-- Page header -->
+            <div class="ui basic segment" style="padding-bottom:0">
+                <div class="ui header">
+                    <i class="microchip icon"></i>
+                    <div class="content">
+                        <span locale="agi_runtime/title">AGI Runtimes</span>
+                        <div class="sub header">
+                            <span locale="agi_runtime/subtitle">Monitor and force-stop running AGI script VM instances. Regular users see their own scripts; administrators see all.</span>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+            <div class="ui divider"></div>
+
+            <!-- Toolbar -->
+            <div class="refresh-bar">
+                <div class="left">
+                    <span id="vmCountLabel"><span locale="agi_runtime/loading">Loading&hellip;</span></span>
+                    <span id="countdownText"></span>
+                </div>
+                <button class="ui tiny basic blue button" onclick="loadRuntimes()">
+                    <i class="refresh icon"></i>
+                    <span locale="agi_runtime/btn_refresh">Refresh Now</span>
+                </button>
+            </div>
+
+            <!-- Runtime table -->
+            <table class="ui celled striped table" id="runtimeTable">
+                <thead>
+                    <tr>
+                        <th class="sortable" data-key="scriptFile" onclick="sortBy('scriptFile')">
+                            <span locale="agi_runtime/col_script">Script</span>
+                            <span class="sort-icon"></span>
+                        </th>
+                        <th class="sortable" data-key="username" onclick="sortBy('username')">
+                            <span locale="agi_runtime/col_user">User</span>
+                            <span class="sort-icon"></span>
+                        </th>
+                        <th class="sortable" data-key="startTime" onclick="sortBy('startTime')">
+                            <span locale="agi_runtime/col_started">Started</span>
+                            <span class="sort-icon"></span>
+                        </th>
+                        <th class="sortable" data-key="elapsedSeconds" onclick="sortBy('elapsedSeconds')">
+                            <span locale="agi_runtime/col_running">Running</span>
+                            <span class="sort-icon"></span>
+                        </th>
+                        <th><span locale="agi_runtime/col_execid">Exec ID</span></th>
+                        <th style="width:120px"><span locale="agi_runtime/col_actions">Actions</span></th>
+                    </tr>
+                </thead>
+                <tbody id="runtimeTbody">
+                    <tr id="emptyRow">
+                        <td colspan="6"><span locale="agi_runtime/loading">Loading&hellip;</span></td>
+                    </tr>
+                </tbody>
+            </table>
+
+        </div><!-- #sys-setting-page -->
+
+        <!-- Toast -->
+        <div id="toast" class="ui message"></div>
+
+        <script>
+            /* ── Locale ─────────────────────────────────────────────
+               Use NewAppLocale() so this scoped object never collides
+               with applocale instances in other AJAX-loaded pages.     */
+            var agiRtLoc = NewAppLocale();
+
+            function loc(key, fallback) {
+                return agiRtLoc.getString(key, fallback);
+            }
+
+            /* ── State ─────────────────────────────────────────────── */
+            var vmData        = [];
+            var sortKey       = "elapsedSeconds";
+            var sortAsc       = false;          // longest-running first by default
+            var countdownSecs = 5;
+
+            // Kill any interval left over from a previous AJAX load of this page.
+            // We store the ID on window so it survives the re-declaration of local
+            // variables when the same script block is executed a second time.
+            clearInterval(window._agiRtInterval);
+            window._agiRtInterval = null;
+
+            /* ── Boot ───────────────────────────────────────────────── */
+            $(document).ready(function() {
+                agiRtLoc.init(
+                    "../locale/system_settings/agi_runtime.json",
+                    function() {
+                        agiRtLoc.translate();   // apply locale= attributes
+                        updateSortHeaders();
+                        loadRuntimes();
+                    }
+                );
+            });
+
+            /* ── Data loading ───────────────────────────────────────── */
+            function loadRuntimes() {
+                clearInterval(window._agiRtInterval);
+                $.ajax({
+                    url: "../../system/ajgi/runtime/list",
+                    method: "GET",
+                    success: function(data) {
+                        vmData = Array.isArray(data) ? data : [];
+                        renderTable();
+                        updateCountLabel();
+                        startCountdown();
+                    },
+                    error: function() {
+                        showEmptyRow(loc("agi_runtime/error_load",
+                            "Failed to load runtime list — check your permissions."));
+                        document.getElementById("vmCountLabel").textContent =
+                            loc("agi_runtime/error_load", "Error");
+                        startCountdown();
+                    }
+                });
+            }
+
+            /* ── Rendering ──────────────────────────────────────────── */
+            function renderTable() {
+                var sorted = vmData.slice().sort(function(a, b) {
+                    var va = a[sortKey], vb = b[sortKey];
+                    if (typeof va === "string") va = va.toLowerCase();
+                    if (typeof vb === "string") vb = vb.toLowerCase();
+                    if (va < vb) return sortAsc ?  -1 : 1;
+                    if (va > vb) return sortAsc ?   1 : -1;
+                    return 0;
+                });
+
+                if (sorted.length === 0) {
+                    showEmptyRow(loc("agi_runtime/empty_state",
+                        "No AGI VMs are currently running."));
+                    return;
+                }
+
+                var btnLabel = loc("agi_runtime/btn_force_stop", "Force Stop");
+
+                document.getElementById("runtimeTbody").innerHTML = sorted.map(function(vm) {
+                    var scriptName = vm.scriptFile.split("/").pop() || vm.scriptFile;
+                    var startedStr = new Date(vm.startTime * 1000).toLocaleTimeString();
+                    var elapsedStr = fmtElapsed(vm.elapsedSeconds);
+                    var elapsedCls = vm.elapsedSeconds > 1800 ? "elapsed-danger"
+                                   : vm.elapsedSeconds > 300  ? "elapsed-warn"
+                                   : "";
+                    var shortID    = vm.execID.substring(0, 8) + "…";
+
+                    return '<tr>' +
+                        '<td title="' + esc(vm.scriptFile) + '">' + esc(scriptName) + '</td>' +
+                        '<td>' + esc(vm.username) + '</td>' +
+                        '<td>' + startedStr + '</td>' +
+                        '<td class="' + elapsedCls + '">' + elapsedStr + '</td>' +
+                        '<td class="execid-cell" title="' + esc(vm.execID) + '">' + shortID + '</td>' +
+                        '<td>' +
+                            '<button class="ui mini red basic button" ' +
+                                'onclick="forceStop(\'' + esc(vm.execID) + '\',\'' + esc(scriptName) + '\')">' +
+                                '<i class="stop circle outline icon"></i> ' + esc(btnLabel) +
+                            '</button>' +
+                        '</td>' +
+                    '</tr>';
+                }).join("");
+            }
+
+            function showEmptyRow(msg) {
+                document.getElementById("runtimeTbody").innerHTML =
+                    '<tr id="emptyRow"><td colspan="6">' + esc(msg) + '</td></tr>';
+            }
+
+            function updateCountLabel() {
+                var n   = vmData.length;
+                var str = n === 0 ? loc("agi_runtime/no_vms",  "No VMs running")
+                        : n === 1 ? loc("agi_runtime/one_vm",  "1 VM running")
+                        :           loc("agi_runtime/n_vms",   "{n} VMs running").replace("{n}", n);
+                document.getElementById("vmCountLabel").textContent = str;
+            }
+
+            /* ── Sorting ────────────────────────────────────────────── */
+            function sortBy(key) {
+                if (sortKey === key) {
+                    sortAsc = !sortAsc;
+                } else {
+                    sortKey = key;
+                    sortAsc = (key !== "elapsedSeconds");
+                }
+                renderTable();
+                updateSortHeaders();
+            }
+
+            function updateSortHeaders() {
+                document.querySelectorAll("th.sortable").forEach(function(th) {
+                    th.classList.remove("asc", "desc");
+                    if (th.getAttribute("data-key") === sortKey) {
+                        th.classList.add(sortAsc ? "asc" : "desc");
+                    }
+                });
+            }
+
+            /* ── Force stop ─────────────────────────────────────────── */
+            function forceStop(execID, scriptName) {
+                var confirmTitle   = loc("agi_runtime/confirm_stop",
+                    "Force stop the following script?");
+                var confirmWarning = loc("agi_runtime/confirm_warning",
+                    "The VM will be terminated immediately and the request will return HTTP 503. Any unsaved state will be lost.");
+
+                if (!confirm(confirmTitle + "\n\n" + scriptName + "\n\n" + confirmWarning)) return;
+
+                $.ajax({
+                    url:    "../../system/ajgi/runtime/stop",
+                    method: "POST",
+                    data:   { execid: execID },
+                    success: function(data) {
+                        if (data && data.error) {
+                            showToast(loc("agi_runtime/toast_error_prefix", "Error: ") + data.error, "red");
+                        } else {
+                            showToast(loc("agi_runtime/toast_success",
+                                "VM force-stopped successfully."), "green");
+                            setTimeout(loadRuntimes, 800);
+                        }
+                    },
+                    error: function(xhr) {
+                        var msg = loc("agi_runtime/req_failed", "Request failed");
+                        try { msg = JSON.parse(xhr.responseText).error || msg; } catch(_) {}
+                        showToast(loc("agi_runtime/toast_error_prefix", "Error: ") + msg, "red");
+                    }
+                });
+            }
+
+            /* ── Auto-refresh countdown ─────────────────────────────── */
+            function startCountdown() {
+                countdownSecs = 5;
+                updateCountdownLabel();
+                window._agiRtInterval = setInterval(function() {
+                    countdownSecs--;
+                    updateCountdownLabel();
+                    if (countdownSecs <= 0) {
+                        clearInterval(window._agiRtInterval);
+                        loadRuntimes();
+                    }
+                }, 1000);
+            }
+
+            function updateCountdownLabel() {
+                document.getElementById("countdownText").textContent =
+                    loc("agi_runtime/auto_refresh", "Auto-refresh in {n}s")
+                        .replace("{n}", countdownSecs);
+            }
+
+            /* ── Toast ──────────────────────────────────────────────── */
+            function showToast(msg, color) {
+                var t = document.getElementById("toast");
+                t.className = "ui " + (color || "blue") + " message";
+                t.textContent = msg;
+                t.style.display = "block";
+                setTimeout(function() { t.style.display = "none"; }, 3500);
+            }
+
+            /* ── Helpers ────────────────────────────────────────────── */
+            function fmtElapsed(sec) {
+                sec = Math.max(0, Math.floor(sec));
+                if (sec <   60) return sec + "s";
+                if (sec < 3600) return Math.floor(sec / 60) + "m " + (sec % 60) + "s";
+                var h = Math.floor(sec / 3600);
+                var m = Math.floor((sec % 3600) / 60);
+                return h + "h " + m + "m";
+            }
+
+            function esc(s) {
+                return String(s)
+                    .replace(/&/g, "&amp;")
+                    .replace(/</g, "&lt;")
+                    .replace(/>/g, "&gt;")
+                    .replace(/"/g, "&quot;")
+                    .replace(/'/g, "&#39;");
+            }
+        </script>
+    </body>
+</html>