diff --git a/modules/performance-statistics-ext/report/js/systemViewTab.js b/modules/performance-statistics-ext/report/js/systemViewTab.js
index 20ba62803..a18b330d8 100644
--- a/modules/performance-statistics-ext/report/js/systemViewTab.js
+++ b/modules/performance-statistics-ext/report/js/systemViewTab.js
@@ -18,45 +18,117 @@
const sysViewSearchNodesSelect = $('#sysViewSearchNodes');
const searchViewsSelect = $('#searchViews');
-function generateColumns(viewName, nodeId) {
- const keys = [];
+function getSystemViewMeta(viewName) {
+ return `data/system-views/${viewName}/meta.json`;
+}
- if (nodeId === "total")
- keys.push("viewNodeId");
-
- const hasSchema = Object.values(REPORT_DATA['systemView']).some(nodeData => {
- if (nodeData[viewName]) {
- keys.push(...nodeData[viewName]['schema']);
- return true;
- }
- return false;
+function getSystemViewChunk(viewName, nodeId, chunkIdx) {
+ const chunkName = String(chunkIdx).padStart(6, '0');
+ return `data/system-views/${viewName}/${nodeId}/chunk-${chunkName}.json`;
+}
+
+function loadSystemViewMeta(viewName) {
+ return $.getJSON(getSystemViewMeta(viewName)).then(meta => {
+ meta.nodeOrder = Object.keys(meta.nodes).sort();
+ meta.totalCount = meta.nodeOrder.reduce((sum, nodeId) => sum + Number(meta.nodes[nodeId]), 0);
+ meta.totalSegments = buildTotalSegments(meta);
+ return meta;
});
+}
+
+function loadSystemViewChunk(viewName, nodeId, chunkIdx) {
+ return $.getJSON(getSystemViewChunk(viewName, nodeId, chunkIdx)).then(chunk => {
+ const rows = (chunk && Array.isArray(chunk.rows)) ? chunk.rows : [];
+
+ return rows;
+ });
+}
+
+function buildTotalSegments(meta) {
+ let offset = 0;
+
+ return meta.nodeOrder.map(nodeId => {
+ const count = Number(meta.nodes[nodeId]);
+ const start = offset;
+ const end = offset + count;
+
+ offset = end;
+
+ return { nodeId, start, end };
+ }).filter(segment => segment.end > segment.start);
+}
+
+function fetchRowsForRange(viewName, nodeId, meta, start, end) {
+ if (end <= start)
+ return Promise.resolve([]);
+
+ const chunkSize = meta.chunkSize;
+ const startChunk = Math.floor(start / chunkSize);
+ const endChunk = Math.floor((end - 1) / chunkSize);
+ const chunkRequests = [];
+
+ for (let idx = startChunk; idx <= endChunk; idx++)
+ chunkRequests.push(loadSystemViewChunk(viewName, nodeId, idx));
+
+ return Promise.all(chunkRequests).then(chunks => {
+ const rows = [];
- if (!hasSchema)
- return null;
+ chunks.forEach((chunkRows, index) => {
+ const chunkIdx = startChunk + index;
+ const chunkStart = chunkIdx * chunkSize;
+ const sliceStart = Math.max(start, chunkStart) - chunkStart;
+ const sliceEnd = Math.min(end, chunkStart + chunkRows.length) - chunkStart;
- return keys.map((key, index) => ({
- field: index,
- title: key,
- sortable: true
- }));
+ if (sliceEnd > sliceStart)
+ rows.push(...chunkRows.slice(sliceStart, sliceEnd));
+ });
+
+ return rows;
+ });
}
-function generateRows(viewName, nodeId) {
- if (nodeId !== "total") {
- const view = REPORT_DATA['systemView'][nodeId][viewName];
- if (view)
- return view['data'];
+function fetchTotalRows(viewName, meta, start, end) {
+ if (end <= start)
+ return Promise.resolve([]);
+
+ const requests = [];
+
+ meta.totalSegments.forEach(segment => {
+ if (end <= segment.start || start >= segment.end)
+ return;
+
+ const localStart = Math.max(start, segment.start) - segment.start;
+ const localEnd = Math.min(end, segment.end) - segment.start;
+
+ requests.push({ nodeId: segment.nodeId, localStart, localEnd });
+ });
+
+ return Promise.all(requests.map(req => {
+ return fetchRowsForRange(viewName, req.nodeId, meta, req.localStart, req.localEnd)
+ .then(rows => rows.map(row => [req.nodeId, ...row]));
+ })).then(chunks => chunks.flat());
+}
+function generateColumns(meta, nodeId) {
+ if (!meta || !Array.isArray(meta.columns) || meta.columns.length === 0)
return [];
- }
- return Object.entries(REPORT_DATA['systemView']).flatMap(([nodeId, nodeData]) => {
- if (!nodeData[viewName])
- return [];
+ const columns = [];
+
+ if (nodeId === "total")
+ columns.push({ field: 0, title: "nodeId", sortable: false });
- return nodeData[viewName]['data'].map(row => [nodeId, ...row]);
+ meta.columns.forEach((key, index) => {
+ const field = nodeId === "total" ? index + 1 : index;
+
+ columns.push({
+ field: field,
+ title: key,
+ sortable: false
+ });
});
+
+ return columns;
}
function drawSystemViewsTable() {
@@ -66,22 +138,55 @@ function drawSystemViewsTable() {
const nodeId = sysViewSearchNodesSelect.val();
const viewName = searchViewsSelect.val();
- const columns = generateColumns(viewName, nodeId);
- const rows = generateRows(viewName, nodeId);
+ loadSystemViewMeta(viewName).then(viewMeta => {
+ const columns = generateColumns(viewMeta, nodeId);
+ const table = getOrCreateSystemViewTable(div);
+ const totalRows = nodeId === "total" ? viewMeta.totalCount : viewMeta.nodes[nodeId];
+
+ $(table).bootstrapTable({
+ formatNoMatches: () => `The "${viewName}" system view is empty on node "${nodeId}".`,
+ pagination: true,
+ sidePagination: "server",
+ search: false,
+ columns: columns,
+ sortOrder: 'desc',
+ ajax: function (params) {
+ const limit = params.data.limit;
+ const offset = params.data.offset;
+ const end = Math.min(offset + limit, totalRows);
+
+ if (totalRows === 0) {
+ params.success({total: 0, rows: []});
+ return;
+ }
+
+ const loader = nodeId === "total"
+ ? fetchTotalRows(viewName, viewMeta, offset, end)
+ : fetchRowsForRange(viewName, nodeId, viewMeta, offset, end);
+
+ loader.then(rows => {
+ console.log('[systemView] Loaded rows', {viewName, nodeId, count: rows.length});
+ params.success({total: totalRows, rows: rows});
+ }).catch(err => {
+ console.error('[systemView] Failed to load rows', viewName, nodeId, err);
+ params.error(err);
+ });
+ }
+ });
+ });
+}
- const table = document.createElement('table');
+function getOrCreateSystemViewTable(div) {
+ let table = document.getElementById('systemViewTable');
- table.id = 'systemViewTable';
- div.appendChild(table);
+ if (!table) {
+ div.innerHTML = "";
+ table = document.createElement('table');
+ table.id = 'systemViewTable';
+ div.appendChild(table);
+ }
- $(table).bootstrapTable({
- formatNoMatches: () => `The "${viewName}" system view is empty on node "${nodeId}".`,
- pagination: true,
- search: true,
- columns: columns,
- data: rows,
- sortOrder: 'desc'
- });
+ return table;
}
buildSelectNodesSystemView(sysViewSearchNodesSelect, drawSystemViewsTable);
diff --git a/modules/performance-statistics-ext/report/js/utils.js b/modules/performance-statistics-ext/report/js/utils.js
index 0e0f20b3d..7d0fdb421 100644
--- a/modules/performance-statistics-ext/report/js/utils.js
+++ b/modules/performance-statistics-ext/report/js/utils.js
@@ -89,17 +89,14 @@ function buildSelectNodes(el, onSelect) {
function buildSelectNodesSystemView(el, onSelect) {
el.append('
- * Example:
- *
- * {
- * $nodeId: {
- * $systemViewTable:
- * schema: [ $column1, $column2 ],
- * data:[
- * [ $value1, $value2 ],
- * [ $value3, $value4 ]
- * ]
- * }
- * }
- *
+ * Writes system view tables into chunked JSON files.
*/
public class SystemViewHandler implements IgnitePerformanceStatisticsHandler {
+ /** Number of rows per chunk. */
+ private static final int CHUNK_SIZE = 2_000;
+
+ /** Base directory for system view data. */
+ private final Path baseDir;
+
+ /** Per-view state in traversal order. */
+ private final Map views = new HashMap<>();
+
+ /** Current view being written. */
+ private ViewState currentView;
+
+ /** All node IDs seen in system views. */
+ private final Set nodeIds = new HashSet<>();
+
/** */
- private final ObjectNode resNode = MAPPER.createObjectNode();
+ public SystemViewHandler(Path baseDir) throws IOException {
+ this.baseDir = baseDir;
+ Files.createDirectories(baseDir);
+ }
/** {@inheritDoc} */
@Override public void systemView(UUID nodeId, String viewName, List schema, List data) {
- JsonNode gridNode = resNode.get(nodeId.toString());
+ String nodeIdStr = nodeId.toString();
+
+ nodeIds.add(nodeIdStr);
+
+ try {
+ if (currentView == null || !currentView.name.equals(viewName)) {
+ if (currentView != null)
+ currentView.finishCurrentNode();
- if (gridNode == null) {
- gridNode = MAPPER.createObjectNode();
- resNode.set(nodeId.toString(), gridNode);
+ currentView = views.get(viewName);
+
+ if (currentView == null) {
+ currentView = new ViewState(viewName, schema);
+ views.put(viewName, currentView);
+ }
+ }
+
+ currentView.addRow(nodeIdStr, data);
}
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
- JsonNode viewNode = gridNode.get(viewName);
+ /** {@inheritDoc} */
+ @Override public Map results() {
+ try {
+ for (ViewState view : views.values())
+ view.finish();
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
- if (viewNode == null) {
- viewNode = MAPPER.createObjectNode();
- ((ObjectNode)gridNode).set(viewName, viewNode);
+ ObjectNode meta = MAPPER.createObjectNode();
- ArrayNode schemaNode = MAPPER.createArrayNode();
- schema.forEach(schemaNode::add);
- ((ObjectNode)viewNode).set("schema", schemaNode);
+ ArrayNode viewsNode = meta.putArray("views");
+ views.values().forEach(state -> viewsNode.add(state.name));
- ArrayNode dataNode = MAPPER.createArrayNode();
- ((ObjectNode)viewNode).set("data", dataNode);
+ ArrayNode nodesNode = meta.putArray("nodes");
+ nodeIds.forEach(nodesNode::add);
+
+ return Collections.singletonMap("systemViewMeta", meta);
+ }
+
+ /** Holds state for a single view of every node. */
+ private class ViewState {
+ /** */
+ private final String name;
+
+ /** */
+ private final List schema;
+
+ /** */
+ private final Path viewDir;
+
+ /** */
+ private final Map nodeRowCounts = new TreeMap<>();
+
+ /** */
+ private String currentNodeId;
+
+ /** */
+ private NodeState currentNode;
+
+ /** */
+ private ViewState(String viewName, List schema) throws IOException {
+ this.name = viewName;
+ this.schema = schema;
+ this.viewDir = baseDir.resolve(viewName);
+
+ Files.createDirectories(viewDir);
}
- ArrayNode dataNode = (ArrayNode)viewNode.get("data");
+ /** */
+ public void addRow(String nodeId, List data) throws IOException {
+ if (currentNode == null || !nodeId.equals(currentNodeId)) {
+ finishCurrentNode();
+ currentNodeId = nodeId;
+ currentNode = new NodeState(viewDir.resolve(nodeId));
+ }
+
+ currentNode.addRow(data);
+ }
+
+ /** */
+ public void finish() throws IOException {
+ finishCurrentNode();
+
+ ObjectNode meta = MAPPER.createObjectNode();
+ meta.put("chunkSize", CHUNK_SIZE);
+
+ ArrayNode columnsNode = meta.putArray("columns");
+ schema.forEach(columnsNode::add);
- ArrayNode rowNode = MAPPER.createArrayNode();
- data.forEach(attr -> rowNode.add(String.valueOf(attr)));
- dataNode.add(rowNode);
+ ObjectNode nodesNode = meta.putObject("nodes");
+ nodeRowCounts.forEach(nodesNode::put);
+
+ Path metaFile = viewDir.resolve("meta.json");
+ MAPPER.writerWithDefaultPrettyPrinter().writeValue(new File(metaFile.toString()), meta);
+ }
+
+ /** */
+ private void finishCurrentNode() throws IOException {
+ if (currentNode == null)
+ return;
+
+ currentNode.closeChunk();
+
+ nodeRowCounts.put(currentNodeId, currentNode.rowCount);
+
+ currentNode = null;
+ currentNodeId = null;
+ }
}
- /** {@inheritDoc} */
- @Override public Map results() {
- return Collections.singletonMap("systemView", resNode);
+ /** */
+ private static class NodeState {
+ /** */
+ private final Path nodeDir;
+
+ /** */
+ private int chunkIdx;
+
+ /** */
+ private int chunkRowCount;
+
+ /** */
+ private int rowCount;
+
+ /** */
+ private JsonGenerator generator;
+
+ /** */
+ private NodeState(Path nodeDir) throws IOException {
+ this.nodeDir = nodeDir;
+ Files.createDirectories(nodeDir);
+ }
+
+ /** */
+ public void addRow(List data) {
+ try {
+ ensureChunk();
+
+ generator.writeStartArray();
+ for (Object attr : data)
+ generator.writeString(String.valueOf(attr));
+ generator.writeEndArray();
+
+ chunkRowCount++;
+ rowCount++;
+
+ if (chunkRowCount >= CHUNK_SIZE)
+ closeChunk();
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /** */
+ public void ensureChunk() throws IOException {
+ if (generator != null)
+ return;
+
+ Path chunkFile = nodeDir.resolve(String.format("chunk-%06d.json", chunkIdx));
+
+ generator = MAPPER.getFactory().createGenerator(Files.newBufferedWriter(chunkFile));
+
+ generator.writeStartObject();
+ generator.writeArrayFieldStart("rows");
+ }
+
+ /** */
+ public void closeChunk() throws IOException {
+ if (generator == null)
+ return;
+
+ generator.writeEndArray();
+ generator.writeEndObject();
+ generator.close();
+
+ generator = null;
+ chunkRowCount = 0;
+ chunkIdx++;
+ }
}
}
diff --git a/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java b/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java
index f7ca62b1e..c4b51c3d0 100644
--- a/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java
+++ b/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java
@@ -18,6 +18,9 @@
package org.apache.ignite.internal.performancestatistics;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -62,6 +65,7 @@
import static java.util.Collections.singletonMap;
import static org.apache.ignite.cache.query.IndexQueryCriteriaBuilder.gt;
+import static org.apache.ignite.internal.performancestatistics.util.Utils.MAPPER;
import static org.apache.ignite.internal.processors.performancestatistics.AbstractPerformanceStatisticsTest.waitForStatisticsEnabled;
import static org.apache.ignite.internal.processors.performancestatistics.FilePerformanceStatisticsWriter.PERF_STAT_DIR;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
@@ -256,8 +260,10 @@ public void testQueryHandlerAggregation() throws Exception {
/** */
@Test
- public void testSystemViewHandler() {
- SystemViewHandler sysViewHnd = new SystemViewHandler();
+ public void testSystemViewHandler() throws IOException {
+ Path outDir = Files.createTempDirectory("sys-view-test");
+
+ SystemViewHandler sysViewHnd = new SystemViewHandler(outDir);
int nodesNumber = 10;
int viewsNumber = 10;
@@ -278,22 +284,32 @@ public void testSystemViewHandler() {
}
}
- JsonNode res = sysViewHnd.results().get("systemView");
+ JsonNode res = sysViewHnd.results().get("systemViewMeta");
+ assertNotNull(res);
- for (int id = 0; id < nodesNumber; id++) {
- UUID nodeId = new UUID(0, id);
+ for (int i = 0; i < viewsNumber; i++) {
+ Path viewDir = outDir.resolve("view" + i);
- JsonNode nodeRes = res.get(nodeId.toString());
+ JsonNode meta = MAPPER.readTree(viewDir.resolve("meta.json").toFile());
- for (int i = 0; i < viewsNumber; i++) {
- ObjectNode view = (ObjectNode)nodeRes.get("view" + i);
+ ArrayNode schemaNode = (ArrayNode)meta.get("columns");
+
+ assertEquals(schema.get(0), schemaNode.get(0).asText());
+ assertEquals(schema.get(1), schemaNode.get(1).asText());
+
+ ObjectNode nodesNode = (ObjectNode)meta.get("nodes");
+
+ for (int id = 0; id < nodesNumber; id++) {
+ UUID nodeId = new UUID(0, id);
+ String nodeIdStr = nodeId.toString();
+
+ assertEquals(1, nodesNode.get(nodeIdStr).asInt());
- ArrayNode schemaNode = (ArrayNode)view.get("schema");
+ Path chunkPath = viewDir.resolve(nodeIdStr).resolve("chunk-000000.json");
- assertEquals(schemaNode.get(0).asText(), schema.get(0));
- assertEquals(schemaNode.get(1).asText(), schema.get(1));
+ JsonNode chunk = MAPPER.readTree(chunkPath.toFile());
- ArrayNode rowNode = (ArrayNode)view.get("data").get(0);
+ ArrayNode rowNode = (ArrayNode)chunk.get("rows").get(0);
assertEquals(Integer.toString(i), rowNode.get(0).asText());
assertEquals(Integer.toString(i), rowNode.get(1).asText());