diff --git a/docs/Computing-Hardware.md b/docs/Computing-Hardware.md index 99459139..a44a77d8 100644 --- a/docs/Computing-Hardware.md +++ b/docs/Computing-Hardware.md @@ -17,7 +17,7 @@ Below is a list of computing resources we have available, as well as some links raven.fish.washington.edu checking… — - disk / CPU below · dashboard + disk / CPU below gannet.fish.washington.edu @@ -52,7 +52,9 @@ Checks run automatically — the timestamp shows when each host was last reached -This comes from a [daily snapshot](https://gannet.fish.washington.edu/v1_web/owlshell/bu-github/ghr.log) (`df` plus CPU load) that raven writes to gannet each morning around 7am, not from the live network check above — so it can be up to a day old, and it is flagged if the snapshot itself goes stale. Drives are listed fullest first. See the **[Raven Dashboard](Raven-Dashboard.md)** for CPU and memory history and who is using raven. +
+ +This comes from a [daily snapshot](https://gannet.fish.washington.edu/v1_web/owlshell/bu-github/ghr.log) (`df` plus CPU load) that raven writes to gannet each morning around 7am, not from the live network check above — so it can be up to a day old, and it is flagged if the snapshot itself goes stale. Drives are listed fullest first. Open **Usage history** below the table for CPU and memory over time and who is using raven (the top 5 accounts by CPU, the **Winners** list, each day). Raven keeps only its latest snapshot, so that history is collected by the status probers one day at a time. ### Gannet Health diff --git a/docs/Raven-Dashboard.md b/docs/Raven-Dashboard.md deleted file mode 100644 index 4cadc029..00000000 --- a/docs/Raven-Dashboard.md +++ /dev/null @@ -1,10 +0,0 @@ -# Raven Dashboard - -Activity on `raven.fish.washington.edu` from the [daily snapshot](https://gannet.fish.washington.edu/v1_web/owlshell/bu-github/ghr.log) a cron on raven writes to gannet each morning around 7am: CPU and memory in use, the top accounts by CPU (the **Winners** list), the processes using the most memory, and drive space. For whether raven is answering right now, see [Server Status](Computing-Hardware.md#server-status). - -
-

Loading raven's latest snapshot…

-
- -!!! note - Every value is a single reading taken when the snapshot runs, not a daily average. A job that runs overnight and finishes by 7am will not appear. Raven only keeps the latest snapshot, so the history here is collected by the server status probers one day at a time and starts on the day this page went live. diff --git a/docs/javascripts/raven-dashboard.js b/docs/javascripts/raven-history.js similarity index 55% rename from docs/javascripts/raven-dashboard.js rename to docs/javascripts/raven-history.js index f69b8ff4..5cfb8904 100644 --- a/docs/javascripts/raven-dashboard.js +++ b/docs/javascripts/raven-history.js @@ -1,24 +1,23 @@ /* - * Raven Dashboard ([data-raven-dashboard] on docs/Raven-Dashboard.md). + * Raven usage history: a collapsible "Usage history" section under the Raven + * Disk / CPU table on Computing Hardware ([data-raven-history]), matching the + * one under Klone Allocation. * * Two inputs, both on the server-status branch (see scripts/README.md): * * internal.json / external.json doc.raven_stats, the latest parsed ghr.log - * snapshot (disks, CPU, memory, Winners, - * top memory processes). + * snapshot (for the latest Winners table). * raven_history.json one entry per day, accumulated by the * probers, since raven keeps no dated * copies of ghr.log. * - * Every number here is a single sample taken when raven's cron runs each + * Every number is a single sample taken when raven's cron runs each * morning, not a daily average. */ (function () { "use strict"; - var STALE_MS = 30 * 60 * 60 * 1000; var SOURCES = ["internal.json", "external.json"]; - var LOG_URL = "https://gannet.fish.washington.edu/v1_web/owlshell/bu-github/ghr.log"; var SVG_NS = "http://www.w3.org/2000/svg"; // Service accounts that show up in ps but are not people running jobs. var SYSTEM_USERS = ["root", "mysql", "rstudio+", "systemd+", "message+", "syslog", "daemon"]; @@ -46,15 +45,6 @@ return best; } - function relativeTime(then, now) { - var mins = Math.round((now - then) / 60000); - if (mins < 60) return mins + " min ago"; - var hrs = Math.round(mins / 60); - if (hrs < 24) return hrs + " hr ago"; - var days = Math.round(hrs / 24); - return days === 1 ? "1 day ago" : days + " days ago"; - } - function el(tag, className, text) { var node = document.createElement(tag); if (className) node.className = className; @@ -68,21 +58,6 @@ return node; } - function section(root, title) { - var s = el("section", "gh-section"); - s.appendChild(el("h2", null, title)); - root.appendChild(s); - return s; - } - - function tile(label, value, sub, cls) { - var t = el("div", "gh-tile " + (cls || "")); - t.appendChild(el("div", "gh-tile-label", label)); - t.appendChild(el("div", "gh-tile-value", value)); - if (sub) t.appendChild(el("div", "gh-tile-sub", sub)); - return t; - } - function table(headers, rows) { var t = el("table"); var thead = el("thead"); @@ -93,32 +68,13 @@ var tbody = el("tbody"); rows.forEach(function (cells) { var r = el("tr"); - cells.forEach(function (c) { - r.appendChild(c instanceof Node ? c : el("td", null, c == null ? "—" : String(c))); - }); + cells.forEach(function (c) { r.appendChild(el("td", null, c)); }); tbody.appendChild(r); }); t.appendChild(tbody); return t; } - function fullnessClass(pct) { - if (pct >= 90) return "ss-disk-critical"; - if (pct >= 75) return "ss-disk-warn"; - return ""; - } - - function barCell(pct) { - var td = el("td", "ss-disk-pct " + fullnessClass(pct)); - var bar = el("span", "ss-bar"); - var fill = el("span", "ss-bar-fill"); - fill.style.width = Math.min(pct, 100) + "%"; - bar.appendChild(fill); - td.appendChild(bar); - td.appendChild(el("span", "ss-bar-label", pct + "%")); - return td; - } - function isSystem(user) { return SYSTEM_USERS.indexOf(user) !== -1; } @@ -210,62 +166,15 @@ return chart; } - function historyNote(history) { - return note( - history.length < 2 - ? "History started on " + (history[0] ? history[0].date : "—") + - " and fills in one day at a time. Hover a point for its value." - : history.length + " daily snapshots, " + history[0].date + " to " + - history[history.length - 1].date + ". Hover a point for its value." - ); - } - - function paint(root, stats, history, now) { - root.innerHTML = ""; + function paint(container, stats, history) { + container.innerHTML = ""; var charts = window.RobertsLabCharts; - history = history || []; - - if (!stats && !history.length) { - root.appendChild(el("p", null, "No raven snapshot available yet.")); - return; - } - - if (stats) { - var stale = now - stats.generatedAt > STALE_MS; - var head = el("p", "ss-raven-meta" + (stale ? " ss-stats-stale" : "")); - head.appendChild( - document.createTextNode( - "Snapshot from " + new Date(stats.generatedAt).toLocaleString() + - " (" + relativeTime(stats.generatedAt, now) + ")" + - (stale ? " · stale — the snapshot cron on raven may have stopped" : "") + - " · " - ) - ); - var raw = el("a", null, "raw log"); - raw.href = LOG_URL; - head.appendChild(raw); - root.appendChild(head); + if (!history.length) return; - var tiles = el("div", "gh-tiles"); - tiles.appendChild(tile("CPU in use", stats.cpu_percent != null ? stats.cpu_percent + "%" : "—", "all cores")); - tiles.appendChild(tile("Memory in use", stats.memory_percent != null ? stats.memory_percent + "%" : "—", "of RAM")); - var disks = (stats.disks || []).slice().sort(function (a, b) { return b.use_percent - a.use_percent; }); - if (disks[0]) { - var c = fullnessClass(disks[0].use_percent); - tiles.appendChild( - tile("Fullest drive", disks[0].use_percent + "% full", disks[0].mount + " · " + disks[0].available + " free", - c === "ss-disk-critical" ? "gh-critical" : c === "ss-disk-warn" ? "gh-warning" : "") - ); - } - var people = (stats.winners || []).filter(function (w) { return !isSystem(w.user); }); - tiles.appendChild( - tile("Top user", people[0] ? people[0].user : "—", people[0] ? people[0].cpu + "% CPU" : "no user jobs") - ); - root.appendChild(tiles); - } + var details = el("details", "hy-history"); + details.appendChild(el("summary", null, "Usage history")); // CPU and memory over time - var load = section(root, "CPU and memory"); var dates = history.map(function (h) { return h.date; }); var series = [ ["CPU", "cpu_percent", "gh-s1"], @@ -278,6 +187,7 @@ return { name: s[0], cls: s[2], points: pts }; }).filter(function (s) { return s.points.length; }); if (charts && series.length) { + details.appendChild(el("h4", null, "CPU and memory in use")); var legend = el("div", "gh-legend"); series.forEach(function (s) { var item = el("span", "gh-legend-item"); @@ -285,26 +195,22 @@ item.appendChild(document.createTextNode(s.name)); legend.appendChild(item); }); - load.appendChild(legend); - load.appendChild( + details.appendChild(legend); + details.appendChild( charts.lineChart(dates, series, { yMax: 100, unit: "%", label: "Raven CPU and memory in use by day" }) ); - load.appendChild(historyNote(history)); - } else { - load.appendChild(note("No history yet.")); } // Winners - var win = section(root, "Winners"); - win.appendChild( + details.appendChild(el("h4", null, "Winners")); + details.appendChild( note( - "The top 5 accounts by CPU when the snapshot was taken. CPU is summed over each account's processes, " + + "The top 5 accounts by CPU at snapshot time. CPU is summed over each account's processes, " + "so 100% is one full core and values above 100% mean several cores." ) ); if (stats && stats.winners && stats.winners.length) { - win.appendChild(el("h3", null, "Latest")); - win.appendChild( + details.appendChild( table( ["User", "CPU", "Memory"], stats.winners.map(function (w) { @@ -315,56 +221,32 @@ } var grid = occurrenceGrid(history); if (grid) { - win.appendChild(el("h3", null, "Appearances in the Winners list")); - win.appendChild(grid); - win.appendChild( - note( - "One square per daily snapshot; darker means more CPU that day. System accounts are listed last in gray. " + - "Hover a square for the numbers." - ) - ); + details.appendChild(el("h4", null, "Appearances in the Winners list")); + details.appendChild(grid); } - // Memory by process - if (stats && stats.top_memory && stats.top_memory.length) { - var mem = section(root, "Top memory processes"); - mem.appendChild( - table( - ["User", "Command", "Memory"], - stats.top_memory.map(function (p) { - var cmd = el("td", "ss-mount", p.command); - return [p.user, cmd, p.mem_percent + "%"]; - }) - ) - ); - } - - // Storage - if (stats && stats.disks && stats.disks.length) { - var st = section(root, "Storage"); - st.appendChild( - table( - ["Mount", "Size", "Used", "Available", "% Full"], - stats.disks - .slice() - .sort(function (a, b) { return b.use_percent - a.use_percent; }) - .map(function (d) { - return [el("td", "ss-mount", d.mount), d.size, d.used, d.available, barCell(d.use_percent)]; - }) - ) - ); - } + details.appendChild( + note( + (history.length < 2 + ? "History started on " + history[0].date + " and fills in one day at a time. " + : history.length + " daily snapshots, " + history[0].date + " to " + history[history.length - 1].date + ". ") + + "Each value is a single reading when the snapshot runs, not a daily average. " + + "In the grid, darker means more CPU that day and system accounts are listed last in gray. " + + "Hover a point or square for its values." + ) + ); + container.appendChild(details); } function render() { - var root = document.querySelector("[data-raven-dashboard]"); - if (!root) return; - var base = root.getAttribute("data-status-base"); + var container = document.querySelector("[data-raven-history]"); + if (!container) return; + var base = container.getAttribute("data-status-base"); Promise.all( SOURCES.map(function (n) { return fetchJson(base, n); }).concat([fetchJson(base, "raven_history.json")]) ).then(function (results) { var history = results.pop(); - paint(root, latestStats(results), Array.isArray(history) ? history : [], Date.now()); + paint(container, latestStats(results), Array.isArray(history) ? history : []); }); } diff --git a/docs/javascripts/server-status.js b/docs/javascripts/server-status.js index af85538a..21bb62e5 100644 --- a/docs/javascripts/server-status.js +++ b/docs/javascripts/server-status.js @@ -88,6 +88,7 @@ if (best && best.generatedAt >= generatedAt) return; best = { cpuPercent: doc.raven_stats.cpu_percent, + memoryPercent: doc.raven_stats.memory_percent, disks: doc.raven_stats.disks || [], generatedAt: generatedAt }; @@ -199,6 +200,7 @@ if (meta) { var bits = []; if (stats.cpuPercent != null) bits.push("CPU load " + stats.cpuPercent + "%"); + if (stats.memoryPercent != null) bits.push("memory " + stats.memoryPercent + "%"); bits.push( "snapshot from " + new Date(stats.generatedAt).toLocaleString() + diff --git a/mkdocs.yml b/mkdocs.yml index 137d80fd..03ad0f7f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,7 +34,7 @@ extra_javascript: - javascripts/server-status.js - javascripts/gannet-health.js - javascripts/hyak-alloc.js - - javascripts/raven-dashboard.js + - javascripts/raven-history.js markdown_extensions: - admonition @@ -89,7 +89,6 @@ nav: - Code Snippets: 'code_Snippets.md' - Computing Hardware: 'Computing-Hardware.md' - Gannet Dashboard: 'Gannet-Dashboard.md' - - Raven Dashboard: 'Raven-Dashboard.md' - Lab Software: 'Lab-Software.md' - 'Klone Guides': diff --git a/scripts/README.md b/scripts/README.md index 9eb2df52..bed4e4ad 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -90,12 +90,12 @@ per-mount use %, Winners), replaced if the same day is seen again, capped at running the checker, so the previous copy is on disk and is carried forward when the branch is re-orphaned. Losing that branch loses the history. -The **Raven Dashboard** page (`docs/Raven-Dashboard.md`, -`docs/javascripts/raven-dashboard.js`, `[data-raven-dashboard]`) reads -`raven_history.json` plus the latest `raven_stats` and draws CPU/memory over -time, a per-user grid of appearances in the Winners list, and current -process and drive tables. It uses the shared line chart from -`gannet-health.js`. +A collapsible **Usage history** section under the Raven Disk / CPU table +on Computing Hardware (`docs/javascripts/raven-history.js`, +`[data-raven-history]`) reads `raven_history.json` plus the latest +`raven_stats` and draws CPU/memory over time, the latest Winners table, and a +per-user grid of appearances in the Winners list. It uses the shared line +chart from `gannet-health.js`. ## Gannet's daily health report