From 017ba59bd4356f4f108244f125afd4ca98637526 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Thu, 30 Jul 2026 09:02:15 -0500 Subject: [PATCH 1/5] feat: add v10dev and default-aggregation support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CDM v10dev with a new default-aggregation field on metric_desc that controls how metrics are aggregated across breakout dimensions. Four aggregation types are supported: sum — duration-weighted sum (current behavior, default fallback) avg — duration-weighted average divided by metric count max — maximum value across all metric documents min — minimum value across all metric documents The default-aggregation field is also accepted on v9dev for forward compatibility — post-processors can emit it before users switch to v10dev without breaking indexing. The aggregation dispatch is in calcAvg: getDefaultAggregation() queries the metric_desc for the field and the query template and result computation branch accordingly. Also fixes pre-existing regex bugs in getCdmVerFromIndex and getInstancesInfo that used [\d+] (single digit) instead of \d+ (one or more digits), which would have broken any multi-digit version. Co-Authored-By: Claude Opus 4.6 (1M context) --- VERSION | 2 +- queries/cdmq/add-run-worker.js | 2 +- queries/cdmq/add-run.js | 6 +- queries/cdmq/cdm.js | 531 ++++++++++++++++++++++------ queries/cdmq/create-index.js | 6 +- queries/cdmq/delete-run.js | 6 +- queries/cdmq/get-instances-info.js | 6 +- queries/cdmq/get-metric-data.js | 6 +- queries/cdmq/get-primary-periods.js | 6 +- queries/cdmq/get-result-summary.js | 2 +- queries/cdmq/server.js | 6 +- templates/metric_desc.base | 1 + 12 files changed, 449 insertions(+), 131 deletions(-) diff --git a/VERSION b/VERSION index 7e93dd3f..729b463c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v8dev +v10dev diff --git a/queries/cdmq/add-run-worker.js b/queries/cdmq/add-run-worker.js index d2cee5d4..f4233c3b 100755 --- a/queries/cdmq/add-run-worker.js +++ b/queries/cdmq/add-run-worker.js @@ -89,7 +89,7 @@ module.exports = async ({ instance, filePath, docTypes, mode }) => { // "run":{"run-uuid":"c0e04edb-ddbc-4081-8bbc-9b9e84e6538d"}} if (Object.keys(action).includes('index') && Object.keys(action['index']).includes('_index')) { const indexName = action['index']['_index']; - const regExp = /^cdm-*(v7dev|v8dev|v9dev)-([^@]+)(@\d\d\d\d\.\d\d)*$/; + const regExp = /^cdm-*(v7dev|v8dev|v9dev|v10dev)-([^@]+)(@\d\d\d\d\.\d\d)*$/; const matches = regExp.exec(indexName); if (matches) { const cdmVer = matches[1]; diff --git a/queries/cdmq/add-run.js b/queries/cdmq/add-run.js index f2ff0190..c4cde8a7 100755 --- a/queries/cdmq/add-run.js +++ b/queries/cdmq/add-run.js @@ -29,10 +29,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -119,7 +119,7 @@ async function main() { .option('--dir ') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); // If the user does not specify any hosts, assume localhost:9200 is used diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index f9e303b8..92e566d3 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -5,7 +5,8 @@ var bigQuerySize = 262144; const docTypes = { v7dev: ['run', 'tag', 'iteration', 'param', 'sample', 'period', 'metric_desc', 'metric_data'], v8dev: ['run', 'tag', 'iteration', 'param', 'sample', 'period', 'metric_desc', 'metric_data'], - v9dev: ['run', 'tag', 'iteration', 'param', 'sample', 'period', 'metric_desc', 'metric_data', 'metric_def'] + v9dev: ['run', 'tag', 'iteration', 'param', 'sample', 'period', 'metric_desc', 'metric_data', 'metric_def'], + v10dev: ['run', 'tag', 'iteration', 'param', 'sample', 'period', 'metric_desc', 'metric_data', 'metric_def'] }; exports.docTypes = docTypes; const supportedCdmVersions = Object.keys(docTypes); @@ -83,7 +84,7 @@ function createGetFromMget(mgetFunc, wrapParamIndex, unwrap = (r) => r[0]) { // INDEX DEFINITIONS // -------------------------------------------------------------------------------------------------------------- -var indexDefs = { v7dev: {}, v8dev: {}, v9dev: {} }; +var indexDefs = { v7dev: {}, v8dev: {}, v9dev: {}, v10dev: {} }; // Most index mappings inherit mappings from other indices. Copies of these indices // are done with JSON.parse(JSON.stringify(src_index)) to facilitate deep copies. @@ -361,6 +362,9 @@ indexDefs['v8dev']['metric_desc']['mappings']['properties']['metric_desc'] = { } }; indexDefs['v9dev']['metric_desc'] = deepClone(indexDefs['v8dev']['metric_desc']); +indexDefs['v9dev']['metric_desc']['mappings']['properties']['metric_desc']['properties']['default-aggregation'] = { + type: 'keyword' +}; // TODO: add new names for cdmv9 @@ -403,6 +407,21 @@ indexDefs['v8dev']['metric_data']['mappings']['properties']['metric_data'] = { }; indexDefs['v9dev']['metric_data'] = deepClone(indexDefs['v8dev']['metric_data']); +// v10dev: adds default-aggregation field to metric_desc for per-metric aggregation control +indexDefs['v10dev']['run_micro'] = deepClone(indexDefs['v9dev']['run_micro']); +indexDefs['v10dev']['run'] = deepClone(indexDefs['v9dev']['run']); +indexDefs['v10dev']['tag'] = deepClone(indexDefs['v9dev']['tag']); +indexDefs['v10dev']['iteration'] = deepClone(indexDefs['v9dev']['iteration']); +indexDefs['v10dev']['param'] = deepClone(indexDefs['v9dev']['param']); +indexDefs['v10dev']['sample'] = deepClone(indexDefs['v9dev']['sample']); +indexDefs['v10dev']['period'] = deepClone(indexDefs['v9dev']['period']); +indexDefs['v10dev']['metric_desc'] = deepClone(indexDefs['v9dev']['metric_desc']); +indexDefs['v10dev']['metric_desc']['mappings']['properties']['metric_desc']['properties']['default-aggregation'] = { + type: 'keyword' +}; +indexDefs['v10dev']['metric_def'] = deepClone(indexDefs['v9dev']['metric_def']); +indexDefs['v10dev']['metric_data'] = deepClone(indexDefs['v9dev']['metric_data']); + exports.indexDefs = indexDefs; // -------------------------------------------------------------------------------------------------------------- @@ -444,10 +463,11 @@ exports.debuglog = debuglog; // -------------------------------------------------------------------------------------------------------------- getCdmVerFromIndex = function (index) { - var regExp = /^cdm-*v([\d+])dev-(.+)/; + var regExp = /^cdm-*v(\d+)dev-(.+)/; var matches = regExp.exec(index); var retMsg = ''; var retCode = 0; + var cdmVer; if (matches) { cdmVer = 'v' + matches[1] + 'dev'; } else { @@ -606,8 +626,8 @@ function getDocType(index) { } } - if (cdmVer == 'v9dev') { - var regExp = /^cdm-v9dev-([^@]+)(@\d\d\d\d\.\d\d|\*)/; + if (cdmVer == 'v9dev' || cdmVer == 'v10dev') { + var regExp = /^cdm-v\d+dev-([^@]+)(@\d\d\d\d\.\d\d|\*)/; var matches = regExp.exec(index); if (matches) { docType = matches[1]; @@ -619,7 +639,7 @@ function getDocType(index) { return createResponse(retCode, retMsg); } } else { - retMsg = 'ERROR: index name [' + index + '] does not match cdmv9 format'; + retMsg = 'ERROR: index name [' + index + '] does not match cdm ' + cdmVer + ' format'; retCode = 4; return createResponse(retCode, retMsg); } @@ -637,12 +657,12 @@ function getIndexBaseName(instance) { //debuglog('cdmver: [' + cdmVer + ']'); if (cdmVer == 'v7dev' || cdmVer == 'v8dev') { return 'cdm' + cdmVer + '-'; - } else if (cdmVer == 'v9dev') { - // v9dev adds a '-' after 'cdm' because of a [lab admin] naming convention + } else if (cdmVer == 'v9dev' || cdmVer == 'v10dev') { + // v9dev+ adds a '-' after 'cdm' because of a [lab admin] naming convention // used for shared opensearch. Therefore, you will find that v7dev // and v8dev cannot be used for some [lab managed] opensearch instances with // same naming requirement. - return 'cdm-v9dev-'; + return 'cdm-' + cdmVer + '-'; } else { console.log('CDM version [' + instance['ver'] + '] is not supported, exiting'); } @@ -660,7 +680,9 @@ function getIndexName(docType, instance, yearDotMonth) { // yearDotMonth may be comma-separated suffixes (e.g., "@2025.01,@2025.02"). // Expand each suffix into a full index name with baseName+docType. var suffixes = yearDotMonth.split(','); - var names = suffixes.map(function (s) { return baseName + docType + s; }); + var names = suffixes.map(function (s) { + return baseName + docType + s; + }); var fullName = names.join(','); checkCreateIndex(instance, fullName); return fullName; @@ -747,13 +769,21 @@ async function fetchBatchedData(instance, reqs, batchSize = 16) { //console.log POST ' + req.url + ' (' + bodyLen + ' bytes)'); if (process.env.CDM_LOG_OS_CURL) { var curlBody = req.body.replace(/'/g, "'\\''"); - console.log('[' + new Date().toISOString() + '] [OS-CURL] curl -s -X POST "' + req.url + '" -H "Content-Type: application/json" -d $\'' + curlBody + '\''); + console.log( + '[' + + new Date().toISOString() + + '] [OS-CURL] curl -s -X POST "' + + req.url + + '" -H "Content-Type: application/json" -d $\'' + + curlBody + + "'" + ); } // Use native fetch instead of then-request (which spawns child processes via sync-rpc) const response = await fetch(req.url, { method: 'POST', body: req.body, - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json' } }); var osElapsed = Date.now() - osReqStart; //console.log POST ' + req.url + ' status=' + response.status + ' in ' + osElapsed + 'ms'); @@ -1164,7 +1194,12 @@ mgetBreakoutValues = async function (instance, runIds, source, type, breakoutNam var result = {}; for (var i = 0; i < breakoutNames.length; i++) { var values = []; - if (responses[i] && responses[i].aggregations && responses[i].aggregations.source && Array.isArray(responses[i].aggregations.source.buckets)) { + if ( + responses[i] && + responses[i].aggregations && + responses[i].aggregations.source && + Array.isArray(responses[i].aggregations.source.buckets) + ) { responses[i].aggregations.source.buckets.forEach(function (bucket) { values.push(String(bucket.key)); }); @@ -1453,7 +1488,7 @@ getInstancesInfo = function (instances) { var name = index['index']; if (/^cdm/.exec(name)) { debuglog('index:\n' + JSON.stringify(index, null, 2)); - const match = name.match(/^cdm[-]{0,1}(v[\d+]dev)/); + const match = name.match(/^cdm-?(v\d+dev)/); const cdmver = match[1]; if (!Object.keys(instances[inst_idx]['indices']).includes(cdmver)) { instances[inst_idx]['indices'][cdmver] = []; @@ -1579,7 +1614,11 @@ buildYearDotMonthRange = function (instance, docType, start, end) { // For multiple months, return comma-separated suffixes. // getIndexName will expand each one with baseName+docType. - return filtered.map(function (m) { return '@' + m; }).join(','); + return filtered + .map(function (m) { + return '@' + m; + }) + .join(','); }; exports.buildYearDotMonthRange = buildYearDotMonthRange; @@ -1589,31 +1628,76 @@ exports.buildYearDotMonthRange = buildYearDotMonthRange; // -------------------------------------------------------------------------------------------------------------- getDistinctNames = async function (instance, yearDotMonth) { - return await mSearch(instance, 'run', yearDotMonth, [], [], null, { source: { terms: { field: 'run.name', size: 10000 } } }, 0); + return await mSearch( + instance, + 'run', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'run.name', size: 10000 } } }, + 0 + ); }; exports.getDistinctNames = getDistinctNames; // -------------------------------------------------------------------------------------------------------------- getDistinctEmails = async function (instance, yearDotMonth) { - return await mSearch(instance, 'run', yearDotMonth, [], [], null, { source: { terms: { field: 'run.email', size: 10000 } } }, 0); + return await mSearch( + instance, + 'run', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'run.email', size: 10000 } } }, + 0 + ); }; exports.getDistinctEmails = getDistinctEmails; // -------------------------------------------------------------------------------------------------------------- getDistinctRunIds = async function (instance, yearDotMonth) { - return await mSearch(instance, 'run', yearDotMonth, [], [], null, { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, 0); + return await mSearch( + instance, + 'run', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, + 0 + ); }; exports.getDistinctRunIds = getDistinctRunIds; // -------------------------------------------------------------------------------------------------------------- getDistinctBenchmarks = async function (instance, yearDotMonth) { - return await mSearch(instance, 'run', yearDotMonth, [], [], null, { source: { terms: { field: 'run.benchmark', size: 10000 } } }, 0); + return await mSearch( + instance, + 'run', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'run.benchmark', size: 10000 } } }, + 0 + ); }; exports.getDistinctBenchmarks = getDistinctBenchmarks; // -------------------------------------------------------------------------------------------------------------- getDistinctTagNames = async function (instance, yearDotMonth) { - return await mSearch(instance, 'tag', yearDotMonth, [], [], null, { source: { terms: { field: 'tag.name', size: 10000 } } }, 0); + return await mSearch( + instance, + 'tag', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'tag.name', size: 10000 } } }, + 0 + ); }; exports.getDistinctTagNames = getDistinctTagNames; @@ -1621,13 +1705,31 @@ exports.getDistinctTagNames = getDistinctTagNames; getDistinctTagValues = async function (instance, yearDotMonth, tagName) { var termKeys = tagName ? ['tag.name'] : []; var values = tagName ? [[tagName]] : []; - return await mSearch(instance, 'tag', yearDotMonth, termKeys, values, null, { source: { terms: { field: 'tag.val', size: 10000 } } }, 0); + return await mSearch( + instance, + 'tag', + yearDotMonth, + termKeys, + values, + null, + { source: { terms: { field: 'tag.val', size: 10000 } } }, + 0 + ); }; exports.getDistinctTagValues = getDistinctTagValues; // -------------------------------------------------------------------------------------------------------------- getDistinctParamArgs = async function (instance, yearDotMonth) { - return await mSearch(instance, 'param', yearDotMonth, [], [], null, { source: { terms: { field: 'param.arg', size: 10000 } } }, 0); + return await mSearch( + instance, + 'param', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'param.arg', size: 10000 } } }, + 0 + ); }; exports.getDistinctParamArgs = getDistinctParamArgs; @@ -1635,13 +1737,31 @@ exports.getDistinctParamArgs = getDistinctParamArgs; getDistinctParamValues = async function (instance, yearDotMonth, paramArg) { var termKeys = paramArg ? ['param.arg'] : []; var values = paramArg ? [[paramArg]] : []; - return await mSearch(instance, 'param', yearDotMonth, termKeys, values, null, { source: { terms: { field: 'param.val', size: 10000 } } }, 0); + return await mSearch( + instance, + 'param', + yearDotMonth, + termKeys, + values, + null, + { source: { terms: { field: 'param.val', size: 10000 } } }, + 0 + ); }; exports.getDistinctParamValues = getDistinctParamValues; // -------------------------------------------------------------------------------------------------------------- getDistinctPrimaryMetrics = async function (instance, yearDotMonth) { - return await mSearch(instance, 'iteration', yearDotMonth, [], [], null, { source: { terms: { field: 'iteration.primary-metric', size: 10000 } } }, 0); + return await mSearch( + instance, + 'iteration', + yearDotMonth, + [], + [], + null, + { source: { terms: { field: 'iteration.primary-metric', size: 10000 } } }, + 0 + ); }; exports.getDistinctPrimaryMetrics = getDistinctPrimaryMetrics; @@ -1651,7 +1771,16 @@ exports.getDistinctPrimaryMetrics = getDistinctPrimaryMetrics; // -------------------------------------------------------------------------------------------------------------- getRunIdsByParam = async function (instance, yearDotMonth, paramArg, paramVal) { - return await mSearch(instance, 'param', yearDotMonth, ['param.arg', 'param.val'], [[paramArg], [paramVal]], null, { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, 0); + return await mSearch( + instance, + 'param', + yearDotMonth, + ['param.arg', 'param.val'], + [[paramArg], [paramVal]], + null, + { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, + 0 + ); }; exports.getRunIdsByParam = getRunIdsByParam; @@ -1667,13 +1796,31 @@ getRunIdsByTag = async function (instance, yearDotMonth, tagName, tagVal) { termKeys.push('tag.val'); values.push([tagVal]); } - return await mSearch(instance, 'tag', yearDotMonth, termKeys, values, null, { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, 0); + return await mSearch( + instance, + 'tag', + yearDotMonth, + termKeys, + values, + null, + { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, + 0 + ); }; exports.getRunIdsByTag = getRunIdsByTag; // -------------------------------------------------------------------------------------------------------------- getRunIdsByPrimaryMetric = async function (instance, yearDotMonth, primaryMetric) { - return await mSearch(instance, 'iteration', yearDotMonth, ['iteration.primary-metric'], [[primaryMetric]], null, { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, 0); + return await mSearch( + instance, + 'iteration', + yearDotMonth, + ['iteration.primary-metric'], + [[primaryMetric]], + null, + { source: { terms: { field: 'run.run-uuid', size: 10000 } } }, + 0 + ); }; exports.getRunIdsByPrimaryMetric = getRunIdsByPrimaryMetric; @@ -2749,15 +2896,19 @@ function buildAggregateLabel(bp, maxLen) { maxLen = maxLen || 30; if (bp.values && bp.values.length > 0) { var vals = bp.values.slice().sort(function (a, b) { - var na = Number(a), nb = Number(b); + var na = Number(a), + nb = Number(b); if (!isNaN(na) && !isNaN(nb)) return na - nb; return a < b ? -1 : a > b ? 1 : 0; }); - var allNumeric = vals.every(function (v) { return !isNaN(Number(v)); }); + var allNumeric = vals.every(function (v) { + return !isNaN(Number(v)); + }); if (allNumeric) { var nums = vals.map(Number); var ranges = []; - var start = nums[0], end = nums[0]; + var start = nums[0], + end = nums[0]; for (var i = 1; i < nums.length; i++) { if (nums[i] === end + 1) { end = nums[i]; @@ -3007,7 +3158,15 @@ getMetricGroupsFromBreakouts = async function (instance, sets, yearDotMonth) { }); var mdStart = Date.now(); var responses = await esJsonArrRequest(instance, 'metric_desc', '/_msearch', jsonArr, yearDotMonth); - console.log('[' + new Date().toISOString() + '] [OS-METRIC-DESC] ' + (jsonArr.length / 2) + ' query(ies) completed in ' + (Date.now() - mdStart) + 'ms'); + console.log( + '[' + + new Date().toISOString() + + '] [OS-METRIC-DESC] ' + + jsonArr.length / 2 + + ' query(ies) completed in ' + + (Date.now() - mdStart) + + 'ms' + ); var metricGroupIdsByLabelSets = []; var metricGroupTermsSets = []; @@ -3034,7 +3193,11 @@ getMetricGroupsFromBreakouts = async function (instance, sets, yearDotMonth) { if (aggregatedPositions.length > 0) { var oldLabels = Object.keys(metricGroupTermsByLabel); if (oldLabels.length === 0) { - var synLabel = aggregatedPositions.map(function (ap) { return ap.segment; }).join('-'); + var synLabel = aggregatedPositions + .map(function (ap) { + return ap.segment; + }) + .join('-'); metricGroupTermsByLabel[synLabel] = ''; } else { var updated = {}; @@ -3121,10 +3284,16 @@ sendMetricReq = async function ( const indexjson = '{"index": "' + indexName + '" }'; const q1Prefix = '{"size":0,"query":{"bool":{"filter":[{"range":{"metric_data.end":{"lte":"'; const q1Mid = '"}}},{"range":{"metric_data.begin":{"gte":"'; - const q1Suffix = '"}}},{"terms":{"metric_desc.metric_desc-uuid":' + metricIdsArrayStr + '}}]}},"aggs":{"metric_avg":{"weighted_avg":{"value":{"field":"metric_data.value"},"weight":{"field":"metric_data.duration"}}}}}'; + const q1Suffix = + '"}}},{"terms":{"metric_desc.metric_desc-uuid":' + + metricIdsArrayStr + + '}}]}},"aggs":{"metric_avg":{"weighted_avg":{"value":{"field":"metric_data.value"},"weight":{"field":"metric_data.duration"}}}}}'; const q2Prefix = '{"size":0,"query":{"bool":{"filter":[{"range":{"metric_data.end":{"lte":"'; const q2Mid = '"}}},{"range":{"metric_data.begin":{"gte":"'; - const q2Suffix = '"}}},{"terms":{"metric_desc.metric_desc-uuid":' + metricIdsArrayStr + '}}]}},"aggs":{"total_weight":{"sum":{"field":"metric_data.duration"}}}}'; + const q2Suffix = + '"}}},{"terms":{"metric_desc.metric_desc-uuid":' + + metricIdsArrayStr + + '}}]}},"aggs":{"total_weight":{"sum":{"field":"metric_data.duration"}}}}'; // Pre-build boundary query templates per chunk of metricIds const chunkSize = 10000; @@ -3132,12 +3301,18 @@ sendMetricReq = async function ( for (let i = 0; i < metricIds.length; i += chunkSize) { const slicedMetricIdsStr = buildMetricIdsArray(metricIds.slice(i, i + chunkSize)); boundaryTemplates.push({ - q3Prefix: '{"size":' + bigQuerySize + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gt":"', + q3Prefix: + '{"size":' + + bigQuerySize + + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gt":"', q3Mid: '"}}},{"range":{"metric_data.begin":{"lte":"', q3Suffix: '"}}},{"terms":{"metric_desc.metric_desc-uuid":' + slicedMetricIdsStr + '}}]}}}', - q4Prefix: '{"size":' + bigQuerySize + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gte":', + q4Prefix: + '{"size":' + + bigQuerySize + + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gte":', q4Mid: '}}},{"range":{"metric_data.begin":{"lt":', - q4Suffix: '}}},{"terms":{"metric_desc.metric_desc-uuid":' + slicedMetricIdsStr + '}}]}}}', + q4Suffix: '}}},{"terms":{"metric_desc.metric_desc-uuid":' + slicedMetricIdsStr + '}}]}}}' }); } @@ -3147,14 +3322,20 @@ sendMetricReq = async function ( // Request 1: Weighted average for documents fully within range let reqjson = q1Prefix + thisEnd + q1Mid + thisBegin + q1Suffix; - jsonArr[wi] = indexjson; jsonArr[wi + 1] = reqjson; wi += 2; - jsonArrTracker[ti] = { label, set, begin: thisBegin, end: thisEnd, numMetricIds: metricIds.length }; ti++; + jsonArr[wi] = indexjson; + jsonArr[wi + 1] = reqjson; + wi += 2; + jsonArrTracker[ti] = { label, set, begin: thisBegin, end: thisEnd, numMetricIds: metricIds.length }; + ti++; jsonArrEstimatedBytes += (indexjson.length + reqjson.length) * 2; // Request 2: Total weight reqjson = q2Prefix + thisEnd + q2Mid + thisBegin + q2Suffix; - jsonArr[wi] = indexjson; jsonArr[wi + 1] = reqjson; wi += 2; - jsonArrTracker[ti] = {}; ti++; + jsonArr[wi] = indexjson; + jsonArr[wi + 1] = reqjson; + wi += 2; + jsonArrTracker[ti] = {}; + ti++; jsonArrEstimatedBytes += (indexjson.length + reqjson.length) * 2; // Requests 3 & 4: Documents partially outside range @@ -3162,14 +3343,20 @@ sendMetricReq = async function ( const t = boundaryTemplates[bt]; // Request 3: End after range reqjson = t.q3Prefix + thisEnd + t.q3Mid + thisEnd + t.q3Suffix; - jsonArr[wi] = indexjson; jsonArr[wi + 1] = reqjson; wi += 2; - jsonArrTracker[ti] = {}; ti++; + jsonArr[wi] = indexjson; + jsonArr[wi + 1] = reqjson; + wi += 2; + jsonArrTracker[ti] = {}; + ti++; jsonArrEstimatedBytes += (indexjson.length + reqjson.length) * 2; // Request 4: Begin before range reqjson = t.q4Prefix + thisBegin + t.q4Mid + thisBegin + t.q4Suffix; - jsonArr[wi] = indexjson; jsonArr[wi + 1] = reqjson; wi += 2; - jsonArrTracker[ti] = {}; ti++; + jsonArr[wi] = indexjson; + jsonArr[wi + 1] = reqjson; + wi += 2; + jsonArrTracker[ti] = {}; + ti++; jsonArrEstimatedBytes += (indexjson.length + reqjson.length) * 2; } @@ -3248,7 +3435,42 @@ sendMetricReq = async function ( }; // -------------------------------------------------------------------------------------------------------------- -calcAvg = function (thisBegin, thisEnd, responses, jsonArrIdx, jsonArrTracker, numMetricIds, values) { +getDefaultAggregation = function (instance, run, source, type, yearDotMonth) { + var q = { + size: 1, + _source: ['metric_desc.default-aggregation'], + query: { + bool: { + filter: [ + { term: { 'run.run-uuid': run } }, + { term: { 'metric_desc.source': source } }, + { term: { 'metric_desc.type': type } } + ] + } + } + }; + var resp = esRequest(instance, 'metric_desc', '/_search', q, yearDotMonth); + var data = JSON.parse(resp.getBody()); + if (data.hits && data.hits.hits && data.hits.hits.length > 0) { + var md = data.hits.hits[0]._source.metric_desc; + if (md && md['default-aggregation']) { + return md['default-aggregation']; + } + } + return 'sum'; +}; + +// -------------------------------------------------------------------------------------------------------------- +calcAvg = function ( + thisBegin, + thisEnd, + responses, + jsonArrIdx, + jsonArrTracker, + numMetricIds, + defaultAggregation, + values +) { debuglog('calcAvg start'); debuglog( 'calcAvg jsonArrIdx: [' + @@ -3269,52 +3491,37 @@ calcAvg = function (thisBegin, thisEnd, responses, jsonArrIdx, jsonArrTracker, n var aggAvgTimesWeight; var newWeight; debuglog('calcAvg responses[' + jsonArrIdx / 2 + ']:' + JSON.stringify(responses[jsonArrIdx / 2], null, 2)); - aggAvg = responses[jsonArrIdx / 2].aggregations.metric_avg.value; - if (isDefined(aggAvg)) { - // We have the weighted average for documents that don't overlap the time range, - // but we need to combine that with the documents that are partially outside - // the time range. We need to know the total weight from the documents we - // just finished in order to add the new documents and recompute the new weighted - // average. - aggWeight = responses[jsonArrIdx / 2 + 1].aggregations.total_weight.value; - aggAvgTimesWeight = aggAvg * aggWeight; - } else { - // It is possible that the aggregation returned no results because all of the documents - // were partially outside the time domain. This can happen when - // 1) A metric does not change during the entire test, and therefore only 1 document - // is created with a huge duration with begin before the time range and after after the - // time range. - // 2) The time domain we have is really small because the resolution we are using is - // very big. - // - // In eithr case, we have to set the average and total_weight to 0, and then the - // recompuation of the weighted average [with the last two requests in this set, finding - // all of th docs that are partially in the time domain] will work. - aggAvg = 0; - aggWeight = 0; + + var useMaxMin = defaultAggregation === 'max' || defaultAggregation === 'min'; + var aggExtreme; + + if (useMaxMin) { + var aggKey = defaultAggregation === 'max' ? 'metric_max' : 'metric_min'; + aggExtreme = responses[jsonArrIdx / 2].aggregations[aggKey].value; aggAvgTimesWeight = 0; + } else { + aggAvg = responses[jsonArrIdx / 2].aggregations.metric_avg.value; + if (isDefined(aggAvg)) { + aggWeight = responses[jsonArrIdx / 2 + 1].aggregations.total_weight.value; + aggAvgTimesWeight = aggAvg * aggWeight; + } else { + aggAvg = 0; + aggWeight = 0; + aggAvgTimesWeight = 0; + } } - // Process the remaining responses in the 'set'. These are typically 2 or more documents. - // Since these docs have a time range partially outside the time range we want, - // we have to get a new, reduced duration and use that to agment our weighted average. + // Collect partial documents (time range overlaps window boundaries). + // Consolidate by _id to avoid double-counting docs that span both boundaries. var sumValueTimesWeight = 0; var sumWeight = 0; - // It is possible to have the same document returned from these remaining queries. - // This can happen when the document's begin is before $this_begin *and* the document's end - // if after $this_end. - // You must not process the document twice. Perform a consolidation by organizing by the - // returned document's '_id' var partialDocs = {}; var k; delete responses[jsonArrIdx / 2]; delete responses[jsonArrIdx / 2 + 1]; delete jsonArrTracker[jsonArrIdx / 2]; delete jsonArrTracker[jsonArrIdx / 2 + 1]; - jsonArrIdx += 4; //advance to the non-aggreation responses - // There can be 1 to many multiples of 2 of these types of responses here. - // We know these type of responses have ended when the next response does - // have an aggregation in it. + jsonArrIdx += 4; while (jsonArrIdx / 2 < responses.length && !Object.keys(responses[jsonArrIdx / 2]).includes('aggregations')) { if (responses[jsonArrIdx / 2].hits.total.value !== responses[jsonArrIdx / 2].hits.hits.length) { console.log( @@ -3339,22 +3546,38 @@ calcAvg = function (thisBegin, thisEnd, responses, jsonArrIdx, jsonArrTracker, n delete jsonArrTracker[jsonArrIdx / 2]; jsonArrIdx += 2; } - // Now we can process the partialDocs - Object.keys(partialDocs).forEach((id) => { - //var docDuration = partialDocs[id].duration; - var docDuration = partialDocs[id].end - partialDocs[id].begin; - if (partialDocs[id].begin < thisBegin) { - docDuration -= thisBegin - partialDocs[id].begin; + + var result; + if (useMaxMin) { + var partialValues = Object.keys(partialDocs).map((id) => partialDocs[id].value); + if (defaultAggregation === 'max') { + var candidates = partialValues; + if (isDefined(aggExtreme) && aggExtreme !== null) candidates.push(aggExtreme); + result = candidates.length > 0 ? Math.max.apply(null, candidates) : 0; + } else { + var candidates = partialValues; + if (isDefined(aggExtreme) && aggExtreme !== null) candidates.push(aggExtreme); + result = candidates.length > 0 ? Math.min.apply(null, candidates) : 0; } - if (partialDocs[id].end > thisEnd) { - docDuration -= partialDocs[id].end - thisEnd; + } else { + Object.keys(partialDocs).forEach((id) => { + var docDuration = partialDocs[id].end - partialDocs[id].begin; + if (partialDocs[id].begin < thisBegin) { + docDuration -= thisBegin - partialDocs[id].begin; + } + if (partialDocs[id].end > thisEnd) { + docDuration -= partialDocs[id].end - thisEnd; + } + var valueTimesWeight = partialDocs[id].value * docDuration; + sumValueTimesWeight += valueTimesWeight; + sumWeight += docDuration; + }); + result = (aggAvgTimesWeight + sumValueTimesWeight) / totalWeightTimesMetrics; + if (defaultAggregation !== 'avg') { + result *= numMetricIds; } - var valueTimesWeight = partialDocs[id].value * docDuration; - sumValueTimesWeight += valueTimesWeight; - sumWeight += docDuration; - }); - var result = (aggAvgTimesWeight + sumValueTimesWeight) / totalWeightTimesMetrics; - result *= numMetricIds; + } + var dataSample = {}; dataSample.begin = thisBegin; dataSample.end = thisEnd; @@ -3386,7 +3609,16 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel totalLabels += Object.keys(metricGroupIdsByLabelSets[idx]).length; } var resolution = sets[0] ? Number(sets[0].resolution) : 1; - console.log('[' + new Date().toISOString() + '] [PERF] getMetricDataFromIdsSets: ' + metricGroupIdsByLabelSets.length + ' set(s), ' + totalLabels + ' label(s), resolution=' + resolution); + console.log( + '[' + + new Date().toISOString() + + '] [PERF] getMetricDataFromIdsSets: ' + + metricGroupIdsByLabelSets.length + + ' set(s), ' + + totalLabels + + ' label(s), resolution=' + + resolution + ); for (var idx = 0; idx < metricGroupIdsByLabelSets.length; idx++) { var begin = Number(sets[idx].begin); @@ -3399,17 +3631,56 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel // Build time-range templates ONCE for all labels in this set. // Each template has prefix/suffix pairs for the 4 query types, // with __IDS__ as placeholder for the metric UUID list. + var defaultAggregation = sets[idx].defaultAggregation || 'sum'; var timeRangeTemplates = []; var thisBegin = begin; var thisEnd = begin + duration; + var baseFilter = + '[{"range":{"metric_data.end":{"lte":"' + + thisEnd + + '"}}},{"range":{"metric_data.begin":{"gte":"' + + thisBegin + + '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]'; while (true) { + var filter = + '[{"range":{"metric_data.end":{"lte":"' + + thisEnd + + '"}}},{"range":{"metric_data.begin":{"gte":"' + + thisBegin + + '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]'; + var q1Agg; + if (defaultAggregation === 'max') { + q1Agg = '"aggs":{"metric_max":{"max":{"field":"metric_data.value"}}}'; + } else if (defaultAggregation === 'min') { + q1Agg = '"aggs":{"metric_min":{"min":{"field":"metric_data.value"}}}'; + } else { + q1Agg = + '"aggs":{"metric_avg":{"weighted_avg":{"value":{"field":"metric_data.value"},"weight":{"field":"metric_data.duration"}}}}'; + } timeRangeTemplates.push({ thisBegin: thisBegin, thisEnd: thisEnd, - q1: '{"size":0,"query":{"bool":{"filter":[{"range":{"metric_data.end":{"lte":"' + thisEnd + '"}}},{"range":{"metric_data.begin":{"gte":"' + thisBegin + '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]}},"aggs":{"metric_avg":{"weighted_avg":{"value":{"field":"metric_data.value"},"weight":{"field":"metric_data.duration"}}}}}', - q2: '{"size":0,"query":{"bool":{"filter":[{"range":{"metric_data.end":{"lte":"' + thisEnd + '"}}},{"range":{"metric_data.begin":{"gte":"' + thisBegin + '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]}},"aggs":{"total_weight":{"sum":{"field":"metric_data.duration"}}}}', - q3: '{"size":' + bigQuerySize + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gt":"' + thisEnd + '"}}},{"range":{"metric_data.begin":{"lte":"' + thisEnd + '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]}}}', - q4: '{"size":' + bigQuerySize + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gte":' + thisBegin + '}}},{"range":{"metric_data.begin":{"lt":' + thisBegin + '}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]}}}', + q1: '{"size":0,"query":{"bool":{"filter":' + filter + '}},' + q1Agg + '}', + q2: + '{"size":0,"query":{"bool":{"filter":' + + filter + + '}},"aggs":{"total_weight":{"sum":{"field":"metric_data.duration"}}}}', + q3: + '{"size":' + + bigQuerySize + + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gt":"' + + thisEnd + + '"}}},{"range":{"metric_data.begin":{"lte":"' + + thisEnd + + '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]}}}', + q4: + '{"size":' + + bigQuerySize + + ',"_source":["metric_data.begin","metric_data.end","metric_data.value"],"query":{"bool":{"filter":[{"range":{"metric_data.end":{"gte":' + + thisBegin + + '}}},{"range":{"metric_data.begin":{"lt":' + + thisBegin + + '}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]}}}' }); thisBegin = thisEnd + 1; thisEnd += duration + 1; @@ -3433,14 +3704,23 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel for (var t = 0; t < timeRangeTemplates.length; t++) { var tmpl = timeRangeTemplates[t]; jsonArr.push(indexjson, tmpl.q1.replace('__IDS__', metricIdsStr)); - jsonArrTracker.push({ label: label, set: idx, begin: tmpl.thisBegin, end: tmpl.thisEnd, numMetricIds: metricIds.length }); + jsonArrTracker.push({ + label: label, + set: idx, + begin: tmpl.thisBegin, + end: tmpl.thisEnd, + numMetricIds: metricIds.length + }); jsonArr.push(indexjson, tmpl.q2.replace('__IDS__', metricIdsStr)); jsonArrTracker.push({}); // Boundary queries — chunk metricIds if > 10000 var chunkSize = 10000; for (var ci = 0; ci < metricIds.length; ci += chunkSize) { - var slicedIdsStr = ci === 0 && metricIds.length <= chunkSize ? metricIdsStr : ('["' + metricIds.slice(ci, ci + chunkSize).join('","') + '"]'); + var slicedIdsStr = + ci === 0 && metricIds.length <= chunkSize + ? metricIdsStr + : '["' + metricIds.slice(ci, ci + chunkSize).join('","') + '"]'; jsonArr.push(indexjson, tmpl.q3.replace('__IDS__', slicedIdsStr)); jsonArrTracker.push({}); jsonArr.push(indexjson, tmpl.q4.replace('__IDS__', slicedIdsStr)); @@ -3450,7 +3730,7 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel const lastLabelInSet = k + 1 >= sortedKeys.length; const lastPass = idx + 1 >= metricGroupIdsByLabelSets.length && lastLabelInSet; - var shouldFlush = lastLabelInSet || ((k + 1) % flushLabelsEvery === 0); + var shouldFlush = lastLabelInSet || (k + 1) % flushLabelsEvery === 0; if (shouldFlush && jsonArr.length > 0) { var esStart = Date.now(); @@ -3465,14 +3745,39 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel while (jsonArrIdx < responses.length * 2) { var trackerIdx = jsonArrIdx / 2; var tracker = jsonArrTracker[trackerIdx]; - if (!tracker || tracker.label === undefined) { jsonArrIdx += 2; continue; } + if (!tracker || tracker.label === undefined) { + jsonArrIdx += 2; + continue; + } var setIdx = tracker.set; var trackerLabel = tracker.label; if (!valueSets[setIdx]) valueSets[setIdx] = {}; if (!valueSets[setIdx][trackerLabel]) valueSets[setIdx][trackerLabel] = []; var prevIdx = jsonArrIdx; - jsonArrIdx = calcAvg(tracker.begin, tracker.end, responses, jsonArrIdx, jsonArrTracker, tracker.numMetricIds, valueSets[setIdx][trackerLabel]); - console.log('[' + new Date().toISOString() + '] [DEBUG] calcAvg: label="' + trackerLabel + '", set=' + setIdx + ', jsonArrIdx ' + prevIdx + '->' + jsonArrIdx + ', values=' + valueSets[setIdx][trackerLabel].length); + jsonArrIdx = calcAvg( + tracker.begin, + tracker.end, + responses, + jsonArrIdx, + jsonArrTracker, + tracker.numMetricIds, + defaultAggregation, + valueSets[setIdx][trackerLabel] + ); + console.log( + '[' + + new Date().toISOString() + + '] [DEBUG] calcAvg: label="' + + trackerLabel + + '", set=' + + setIdx + + ', jsonArrIdx ' + + prevIdx + + '->' + + jsonArrIdx + + ', values=' + + valueSets[setIdx][trackerLabel].length + ); } //console.log in ' + (Date.now()-calcStart) + 'ms'); @@ -3487,7 +3792,9 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel //} } } - console.log('[' + new Date().toISOString() + '] [PERF] getMetricDataFromIdsSets total: ' + (Date.now()-funcStart) + 'ms'); + console.log( + '[' + new Date().toISOString() + '] [PERF] getMetricDataFromIdsSets total: ' + (Date.now() - funcStart) + 'ms' + ); return valueSets; }; @@ -3676,6 +3983,16 @@ getMetricDataSets = async function (instance, sets, yearDotMonth) { } } + for (var idx = 0; idx < sets.length; idx++) { + sets[idx].defaultAggregation = getDefaultAggregation( + instance, + sets[idx].run, + sets[idx].source, + sets[idx].type, + yearDotMonth + ); + } + var dataSets = await getMetricDataFromIdsSets(instance, sets, metricGroupIdsByLabelSets, yearDotMonth); if (dataSets.length != sets.length) { diff --git a/queries/cdmq/create-index.js b/queries/cdmq/create-index.js index 1193fc43..58f0f0a2 100755 --- a/queries/cdmq/create-index.js +++ b/queries/cdmq/create-index.js @@ -24,10 +24,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -37,7 +37,7 @@ async function main() { .version('0.1.0') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) .option('--index ', 'An index name matching cdm naming format, for example, cdm-v9dev-run@202505>') .parse(process.argv); diff --git a/queries/cdmq/delete-run.js b/queries/cdmq/delete-run.js index 146619d4..238937f7 100644 --- a/queries/cdmq/delete-run.js +++ b/queries/cdmq/delete-run.js @@ -23,10 +23,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -36,7 +36,7 @@ program .option('--run ') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); // If the user does not specify any hosts, assume localhost:9200 is used diff --git a/queries/cdmq/get-instances-info.js b/queries/cdmq/get-instances-info.js index 024bf902..b7b413c2 100755 --- a/queries/cdmq/get-instances-info.js +++ b/queries/cdmq/get-instances-info.js @@ -24,10 +24,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -37,7 +37,7 @@ async function main() { .version('0.1.0') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); // If the user does not specify any hosts, assume localhost:9200 is used diff --git a/queries/cdmq/get-metric-data.js b/queries/cdmq/get-metric-data.js index a52069f9..be0b103f 100644 --- a/queries/cdmq/get-metric-data.js +++ b/queries/cdmq/get-metric-data.js @@ -105,10 +105,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { program.instances[program.instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -134,7 +134,7 @@ async function main() { save_userpass ) .option( - '--ver ', + '--ver ', 'The Common Data Model version to use for the most recent --host (passed to server)', save_ver ) diff --git a/queries/cdmq/get-primary-periods.js b/queries/cdmq/get-primary-periods.js index a2ed2040..d5fd6f7a 100644 --- a/queries/cdmq/get-primary-periods.js +++ b/queries/cdmq/get-primary-periods.js @@ -30,10 +30,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -43,7 +43,7 @@ program .option('--run ') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) .option('--output-dir , if not used, output is to console only') .option('--output-format , fmta[,fmtb]', 'one or more output formats: txt, json, yaml', list, []) .parse(process.argv); diff --git a/queries/cdmq/get-result-summary.js b/queries/cdmq/get-result-summary.js index 73302aed..daedf53c 100644 --- a/queries/cdmq/get-result-summary.js +++ b/queries/cdmq/get-result-summary.js @@ -111,7 +111,7 @@ program ) .option('--host ', 'Ignored (accepted for backward compatibility)') .option('--userpass ', 'Ignored (accepted for backward compatibility)') - .option('--ver ', 'Ignored (accepted for backward compatibility)') + .option('--ver ', 'Ignored (accepted for backward compatibility)') .option('--user ', 'Filter by run name') .option('--email ', 'Filter by email') .option('--harness ', 'Filter by harness') diff --git a/queries/cdmq/server.js b/queries/cdmq/server.js index 6e03d0c8..9d3cd8ea 100755 --- a/queries/cdmq/server.js +++ b/queries/cdmq/server.js @@ -83,10 +83,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v[789]dev$/.exec(ver)) { + if (/^v([789]|10)dev$/.exec(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, or v9dev, not: ' + ver); + console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); process.exit(1); } } @@ -95,7 +95,7 @@ program .version('1.0.0') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); const options = program.opts(); diff --git a/templates/metric_desc.base b/templates/metric_desc.base index 04ed6e38..d2911e83 100644 --- a/templates/metric_desc.base +++ b/templates/metric_desc.base @@ -1,6 +1,7 @@ "metric_desc": { "properties": { "metric_desc-uuid": { "type": "keyword" }, + "default-aggregation": { "type": "keyword" }, "class": { "type": "keyword" }, "type": { "type": "keyword" }, "source": { "type": "keyword" }, From 9aefc47a7adc702430bef61eab5fe3125723330c Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Thu, 30 Jul 2026 09:26:06 -0500 Subject: [PATCH 2/5] refactor: consolidate version validation into cdm.js Add isValidCdmVersion() and cdmVersionOptionDesc() exports to cdm.js so all CLI scripts validate versions against the single supportedCdmVersions list instead of duplicating a regex and version enumeration in each file. Adding a new CDM version now only requires updating docTypes in cdm.js. Co-Authored-By: Claude Opus 4.6 (1M context) --- queries/cdmq/add-run-worker.js | 4 +- queries/cdmq/add-run.js | 6 +- queries/cdmq/cdm.js | 9 +- queries/cdmq/create-index.js | 6 +- queries/cdmq/delete-run.js | 6 +- queries/cdmq/get-instances-info.js | 6 +- queries/cdmq/get-metric-data.js | 6 +- queries/cdmq/get-primary-periods.js | 6 +- queries/cdmq/get-result-summary.js | 3 +- queries/cdmq/server.js | 192 ++++++++++++++++++++-------- 10 files changed, 168 insertions(+), 76 deletions(-) diff --git a/queries/cdmq/add-run-worker.js b/queries/cdmq/add-run-worker.js index f4233c3b..473cd714 100755 --- a/queries/cdmq/add-run-worker.js +++ b/queries/cdmq/add-run-worker.js @@ -89,7 +89,9 @@ module.exports = async ({ instance, filePath, docTypes, mode }) => { // "run":{"run-uuid":"c0e04edb-ddbc-4081-8bbc-9b9e84e6538d"}} if (Object.keys(action).includes('index') && Object.keys(action['index']).includes('_index')) { const indexName = action['index']['_index']; - const regExp = /^cdm-*(v7dev|v8dev|v9dev|v10dev)-([^@]+)(@\d\d\d\d\.\d\d)*$/; + const regExp = new RegExp( + '^cdm-*(' + cdm.supportedCdmVersions.join('|') + ')-([^@]+)(@\\d\\d\\d\\d\\.\\d\\d)*$' + ); const matches = regExp.exec(indexName); if (matches) { const cdmVer = matches[1]; diff --git a/queries/cdmq/add-run.js b/queries/cdmq/add-run.js index c4cde8a7..8c0ac521 100755 --- a/queries/cdmq/add-run.js +++ b/queries/cdmq/add-run.js @@ -29,10 +29,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -119,7 +119,7 @@ async function main() { .option('--dir ') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option(cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); // If the user does not specify any hosts, assume localhost:9200 is used diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index 92e566d3..1153ce7d 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -11,6 +11,13 @@ const docTypes = { exports.docTypes = docTypes; const supportedCdmVersions = Object.keys(docTypes); exports.supportedCdmVersions = supportedCdmVersions; +function isValidCdmVersion(ver) { + return supportedCdmVersions.includes(ver); +} +exports.isValidCdmVersion = isValidCdmVersion; +exports.cdmVersionOptionDesc = function () { + return '--ver <' + supportedCdmVersions.join('|') + '>'; +}; const debugOut = 0; const indexSettings = { number_of_shards: 1, @@ -1503,7 +1510,7 @@ getInstancesInfo = function (instances) { } if (Object.keys(instances[inst_idx]['indices']).length != 0) { // If mulitple versions of indices exist, default to the latest version - // (this can be overridden with --ver after --host) + // (this can be overridden with --ver after --host) // Note: if you index a new data into a newer CDM version, that will // create those indices, and a subsequent query (without --ver) will now // default to the newer cdm version. diff --git a/queries/cdmq/create-index.js b/queries/cdmq/create-index.js index 58f0f0a2..e1df1ca2 100755 --- a/queries/cdmq/create-index.js +++ b/queries/cdmq/create-index.js @@ -24,10 +24,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -37,7 +37,7 @@ async function main() { .version('0.1.0') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option(cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host', save_ver) .option('--index ', 'An index name matching cdm naming format, for example, cdm-v9dev-run@202505>') .parse(process.argv); diff --git a/queries/cdmq/delete-run.js b/queries/cdmq/delete-run.js index 238937f7..3bca5cf7 100644 --- a/queries/cdmq/delete-run.js +++ b/queries/cdmq/delete-run.js @@ -23,10 +23,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -36,7 +36,7 @@ program .option('--run ') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option(cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); // If the user does not specify any hosts, assume localhost:9200 is used diff --git a/queries/cdmq/get-instances-info.js b/queries/cdmq/get-instances-info.js index b7b413c2..0d8dc49f 100755 --- a/queries/cdmq/get-instances-info.js +++ b/queries/cdmq/get-instances-info.js @@ -24,10 +24,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -37,7 +37,7 @@ async function main() { .version('0.1.0') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option(cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); // If the user does not specify any hosts, assume localhost:9200 is used diff --git a/queries/cdmq/get-metric-data.js b/queries/cdmq/get-metric-data.js index be0b103f..160aa6ac 100644 --- a/queries/cdmq/get-metric-data.js +++ b/queries/cdmq/get-metric-data.js @@ -105,10 +105,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { program.instances[program.instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -134,7 +134,7 @@ async function main() { save_userpass ) .option( - '--ver ', + cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host (passed to server)', save_ver ) diff --git a/queries/cdmq/get-primary-periods.js b/queries/cdmq/get-primary-periods.js index d5fd6f7a..5ac8601f 100644 --- a/queries/cdmq/get-primary-periods.js +++ b/queries/cdmq/get-primary-periods.js @@ -30,10 +30,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -43,7 +43,7 @@ program .option('--run ') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option(cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host', save_ver) .option('--output-dir , if not used, output is to console only') .option('--output-format , fmta[,fmtb]', 'one or more output formats: txt, json, yaml', list, []) .parse(process.argv); diff --git a/queries/cdmq/get-result-summary.js b/queries/cdmq/get-result-summary.js index daedf53c..d73591c5 100644 --- a/queries/cdmq/get-result-summary.js +++ b/queries/cdmq/get-result-summary.js @@ -1,4 +1,5 @@ //# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=javascript +var cdm = require('./cdm'); var yaml = require('js-yaml'); var program = require('commander'); const http = require('http'); @@ -111,7 +112,7 @@ program ) .option('--host ', 'Ignored (accepted for backward compatibility)') .option('--userpass ', 'Ignored (accepted for backward compatibility)') - .option('--ver ', 'Ignored (accepted for backward compatibility)') + .option(cdm.cdmVersionOptionDesc(), 'Ignored (accepted for backward compatibility)') .option('--user ', 'Filter by run name') .option('--email ', 'Filter by email') .option('--harness ', 'Filter by harness') diff --git a/queries/cdmq/server.js b/queries/cdmq/server.js index 9d3cd8ea..0a95e907 100755 --- a/queries/cdmq/server.js +++ b/queries/cdmq/server.js @@ -40,7 +40,9 @@ function serverError(msg, reqId) { // Returns null if valid, or an error message string if unknown fields are found. function validateBodyFields(body, knownFields) { if (!body || typeof body !== 'object') return null; - var unknown = Object.keys(body).filter(function (k) { return !knownFields.includes(k); }); + var unknown = Object.keys(body).filter(function (k) { + return !knownFields.includes(k); + }); if (unknown.length === 0) return null; var hints = unknown.map(function (u) { // suggest a known field if it differs only by a trailing 's' or missing trailing 's' @@ -48,8 +50,7 @@ function validateBodyFields(body, knownFields) { if (knownFields.includes(candidate)) return u + ' (did you mean: ' + candidate + '?)'; return u; }); - return 'Unknown field(s) in request body: ' + hints.join(', ') + - '. Known fields: ' + knownFields.join(', '); + return 'Unknown field(s) in request body: ' + hints.join(', ') + '. Known fields: ' + knownFields.join(', '); } // Per-client request counter for generating short session-like IDs @@ -83,10 +84,10 @@ function save_ver(ver) { console.log('You must specify a --host before a --ver'); process.exit(1); } - if (/^v([789]|10)dev$/.exec(ver)) { + if (cdm.isValidCdmVersion(ver)) { instances[instances.length - 1]['ver'] = ver; } else { - console.log('The version must be v7dev, v8dev, v9dev, or v10dev, not: ' + ver); + console.log('The version must be one of: ' + cdm.supportedCdmVersions.join(', ') + ', not: ' + ver); process.exit(1); } } @@ -95,7 +96,7 @@ program .version('1.0.0') .option('--host ', 'The host and optional port of the OpenSearch instance', save_host) .option('--userpass ', 'The user and password for the most recent --host', save_userpass) - .option('--ver ', 'The Common Data Model version to use for the most recent --host', save_ver) + .option(cdm.cdmVersionOptionDesc(), 'The Common Data Model version to use for the most recent --host', save_ver) .parse(process.argv); const options = program.opts(); @@ -235,7 +236,9 @@ app.get('/api/v1/runs', async (req, res) => { // Apply multi-run-ID filter if specified if (runIdFilter) { - runIds = runIds.filter(function (id) { return runIdFilter.has(id); }); + runIds = runIds.filter(function (id) { + return runIdFilter.has(id); + }); } // Helper: get run IDs from a cross-index aggregation and intersect with current set @@ -859,10 +862,14 @@ app.post('/api/v1/iterations/details', async (req, res) => { // Step 5: Assemble iteration objects for (var i = 0; i < allIterIds.length; i++) { var meta = iterToRunMap[i]; - var iterSamples = (samplesByIter[i]) || []; + var iterSamples = samplesByIter[i] || []; var iterStatuses = (statuses && statuses[i]) || []; - var passCount = iterStatuses.filter(function (s) { return s === 'pass'; }).length; - var failCount = iterStatuses.filter(function (s) { return s === 'fail'; }).length; + var passCount = iterStatuses.filter(function (s) { + return s === 'pass'; + }).length; + var failCount = iterStatuses.filter(function (s) { + return s === 'fail'; + }).length; allIterations.push({ runId: meta.runId, @@ -885,11 +892,7 @@ app.post('/api/v1/iterations/details', async (req, res) => { } serverLog( - 'POST /api/v1/iterations/details: ' + - runIds.length + - ' run(s) -> ' + - allIterations.length + - ' iteration(s)' + 'POST /api/v1/iterations/details: ' + runIds.length + ' run(s) -> ' + allIterations.length + ' iteration(s)' ); res.json({ iterations: allIterations }); } catch (error) { @@ -962,7 +965,9 @@ app.post('/api/v1/iterations/metric-values', async (req, res) => { // Get primary period IDs var primaryPeriodIds = []; - var hasPassing = passingSamplesByIter.some(function (s) { return s.length > 0; }); + var hasPassing = passingSamplesByIter.some(function (s) { + return s.length > 0; + }); if (hasPassing) { primaryPeriodIds = await cdm.mgetPrimaryPeriodId(inst, passingSamplesByIter, passingPeriodNamesByIter, ydm); if (typeof primaryPeriodIds === 'undefined') primaryPeriodIds = []; @@ -985,8 +990,8 @@ app.post('/api/v1/iterations/metric-values', async (req, res) => { if (pmParts.length < 2) continue; var pmSource = pmParts[0]; var pmType = pmParts[1]; - var iterPeriodIds = (primaryPeriodIds[i]) || []; - var iterRanges = (periodRanges[i]) || []; + var iterPeriodIds = primaryPeriodIds[i] || []; + var iterRanges = periodRanges[i] || []; for (var s = 0; s < iterPeriodIds.length; s++) { if (!iterPeriodIds[s]) continue; var range = iterRanges[s]; @@ -1070,7 +1075,10 @@ app.post('/api/v1/iterations/metric-sources', async (req, res) => { var sourcesPerRun = await cdm.mgetMetricSources(inst, runIds, ydm); if (sourcesPerRun) { sourcesPerRun.forEach(function (s) { - if (Array.isArray(s)) s.forEach(function (v) { allSources.add(v); }); + if (Array.isArray(s)) + s.forEach(function (v) { + allSources.add(v); + }); }); } } @@ -1100,11 +1108,16 @@ app.post('/api/v1/iterations/metric-types', async (req, res) => { if (invalidInstance(inst)) continue; var ydm = cdm.buildYearDotMonthRange(inst, 'run', start || null, end || null); // mgetMetricTypes needs parallel arrays of runIds and sources - var sources = runIds.map(function () { return source; }); + var sources = runIds.map(function () { + return source; + }); var typesPerRun = await cdm.mgetMetricTypes(inst, runIds, sources, ydm); if (typesPerRun) { typesPerRun.forEach(function (t) { - if (Array.isArray(t)) t.forEach(function (v) { allTypes.add(v); }); + if (Array.isArray(t)) + t.forEach(function (v) { + allTypes.add(v); + }); }); } } @@ -1131,7 +1144,9 @@ app.post('/api/v1/iterations/breakout-values', async (req, res) => { } const { runIds, start, end, source, type, breakouts } = req.body; if (!Array.isArray(runIds) || runIds.length === 0 || !source || !type || !Array.isArray(breakouts)) { - return res.status(400).json({ code: 'MISSING_PARAMS', error: 'runIds, source, type, and breakouts are required' }); + return res + .status(400) + .json({ code: 'MISSING_PARAMS', error: 'runIds, source, type, and breakouts are required' }); } getInstancesInfo(instances); var merged = {}; @@ -1142,7 +1157,9 @@ app.post('/api/v1/iterations/breakout-values', async (req, res) => { // Merge values across instances Object.keys(result).forEach(function (dim) { if (!merged[dim]) merged[dim] = new Set(); - result[dim].forEach(function (v) { merged[dim].add(v); }); + result[dim].forEach(function (v) { + merged[dim].add(v); + }); }); } // Convert Sets to sorted arrays @@ -1150,7 +1167,18 @@ app.post('/api/v1/iterations/breakout-values', async (req, res) => { Object.keys(merged).forEach(function (dim) { response[dim] = Array.from(merged[dim]).sort(); }); - serverLog('POST /api/v1/iterations/breakout-values: ' + source + '::' + type + ' -> ' + Object.keys(response).map(function (k) { return k + ':' + response[k].length; }).join(', ')); + serverLog( + 'POST /api/v1/iterations/breakout-values: ' + + source + + '::' + + type + + ' -> ' + + Object.keys(response) + .map(function (k) { + return k + ':' + response[k].length; + }) + .join(', ') + ); res.json({ breakouts: response }); } catch (error) { serverError('Error in POST /api/v1/iterations/breakout-values: ' + error); @@ -1174,8 +1202,9 @@ app.post('/api/v1/iterations/period-info', async (req, res) => { if (!Array.isArray(reqIterations) || reqIterations.length === 0) { return res.status(400).json({ code: 'MISSING_PARAMS', error: 'iterations array is required' }); } - var requestedSampleIdx = (typeof sampleIndex === 'number') ? sampleIndex : null; - var perIterSampleIdx = (typeof sampleIndex === 'object' && sampleIndex !== null && !Array.isArray(sampleIndex)) ? sampleIndex : null; + var requestedSampleIdx = typeof sampleIndex === 'number' ? sampleIndex : null; + var perIterSampleIdx = + typeof sampleIndex === 'object' && sampleIndex !== null && !Array.isArray(sampleIndex) ? sampleIndex : null; getInstancesInfo(instances); var result = {}; @@ -1184,8 +1213,12 @@ app.post('/api/v1/iterations/period-info', async (req, res) => { if (invalidInstance(inst)) continue; var ydm = cdm.buildYearDotMonthRange(inst, 'run', start || null, end || null); - var allIterIds = reqIterations.map(function (it) { return it.iterationId; }); - var iterRunIds = reqIterations.map(function (it) { return it.runId; }); + var allIterIds = reqIterations.map(function (it) { + return it.iterationId; + }); + var iterRunIds = reqIterations.map(function (it) { + return it.runId; + }); var samples = await cdm.mgetSamples(inst, allIterIds, ydm); var statuses = await cdm.mgetSampleStatuses(inst, samples || [], ydm); @@ -1207,7 +1240,9 @@ app.post('/api/v1/iterations/period-info', async (req, res) => { } var primaryPeriodIds = []; - var hasPassing = passingSamplesByIter.some(function (s) { return s.length > 0; }); + var hasPassing = passingSamplesByIter.some(function (s) { + return s.length > 0; + }); if (hasPassing) { primaryPeriodIds = await cdm.mgetPrimaryPeriodId(inst, passingSamplesByIter, passingPeriodNamesByIter, ydm); if (typeof primaryPeriodIds === 'undefined') primaryPeriodIds = []; @@ -1220,8 +1255,8 @@ app.post('/api/v1/iterations/period-info', async (req, res) => { } for (var i = 0; i < allIterIds.length; i++) { - var iterPeriodIds = (primaryPeriodIds[i]) || []; - var iterRanges = (periodRanges[i]) || []; + var iterPeriodIds = primaryPeriodIds[i] || []; + var iterRanges = periodRanges[i] || []; if (iterPeriodIds.length === 0) continue; var selIdx = 0; @@ -1240,7 +1275,7 @@ app.post('/api/v1/iterations/period-info', async (req, res) => { periodId: iterPeriodIds[selIdx], begin: range.begin, end: range.end, - runId: iterRunIds[i], + runId: iterRunIds[i] }; } } @@ -1265,13 +1300,17 @@ app.post('/api/v1/iterations/supplemental-metric', async (req, res) => { var breakoutArr = Array.isArray(breakout) ? breakout : []; var filterVal = filter || null; // sampleIndex can be a number (same for all iterations) or an object { iterationId: index } - var requestedSampleIdx = (typeof sampleIndex === 'number') ? sampleIndex : null; - var perIterSampleIdx = (typeof sampleIndex === 'object' && sampleIndex !== null && !Array.isArray(sampleIndex)) ? sampleIndex : null; + var requestedSampleIdx = typeof sampleIndex === 'number' ? sampleIndex : null; + var perIterSampleIdx = + typeof sampleIndex === 'object' && sampleIndex !== null && !Array.isArray(sampleIndex) ? sampleIndex : null; if (!source || !type) { return res.status(400).json({ code: 'MISSING_PARAMS', error: 'source and type are required' }); } // Accept either iterations (array of {iterationId, runId}) or runIds (discover iterations) - if ((!Array.isArray(reqIterations) || reqIterations.length === 0) && (!Array.isArray(runIds) || runIds.length === 0)) { + if ( + (!Array.isArray(reqIterations) || reqIterations.length === 0) && + (!Array.isArray(runIds) || runIds.length === 0) + ) { return res.status(400).json({ code: 'MISSING_PARAMS', error: 'iterations or runIds are required' }); } getInstancesInfo(instances); @@ -1325,7 +1364,9 @@ app.post('/api/v1/iterations/supplemental-metric', async (req, res) => { } var primaryPeriodIds = []; - var hasPassing = passingSamplesByIter.some(function (s) { return s.length > 0; }); + var hasPassing = passingSamplesByIter.some(function (s) { + return s.length > 0; + }); if (hasPassing) { primaryPeriodIds = await cdm.mgetPrimaryPeriodId(inst, passingSamplesByIter, passingPeriodNamesByIter, ydm); if (typeof primaryPeriodIds === 'undefined') primaryPeriodIds = []; @@ -1342,8 +1383,8 @@ app.post('/api/v1/iterations/supplemental-metric', async (req, res) => { var metricSets = []; var metricSetMap = []; for (var i = 0; i < allIterIds.length; i++) { - var iterPeriodIds = (primaryPeriodIds[i]) || []; - var iterRanges = (periodRanges[i]) || []; + var iterPeriodIds = primaryPeriodIds[i] || []; + var iterRanges = periodRanges[i] || []; if (iterPeriodIds.length === 0) continue; // Use per-iteration sample index if available, otherwise global, otherwise 0 @@ -1407,7 +1448,19 @@ app.post('/api/v1/iterations/supplemental-metric', async (req, res) => { } } - serverLog('POST /api/v1/iterations/supplemental-metric: ' + source + '::' + type + ' breakout=' + JSON.stringify(breakoutArr) + ' sampleIndex=' + requestedSampleIdx + ' -> ' + Object.keys(result).length + ' iteration(s)'); + serverLog( + 'POST /api/v1/iterations/supplemental-metric: ' + + source + + '::' + + type + + ' breakout=' + + JSON.stringify(breakoutArr) + + ' sampleIndex=' + + requestedSampleIdx + + ' -> ' + + Object.keys(result).length + + ' iteration(s)' + ); res.json({ values: result, remainingBreakouts: remainingBreakouts, sampleInfo: sampleInfo }); } catch (error) { serverError('Error in POST /api/v1/iterations/supplemental-metric: ' + error); @@ -1460,9 +1513,7 @@ app.get('/api/v1/fields/months', async (req, res) => { app.get('/api/v1/fields/run-ids', async (req, res) => { try { - var values = await getDistinctValues(instances, (inst) => - cdm.getDistinctRunIds(inst, getYdm(inst, 'run', req)) - ); + var values = await getDistinctValues(instances, (inst) => cdm.getDistinctRunIds(inst, getYdm(inst, 'run', req))); serverLog('GET /api/v1/fields/run-ids returned ' + values.length + ' value(s)'); res.json({ values: values }); } catch (error) { @@ -1473,9 +1524,7 @@ app.get('/api/v1/fields/run-ids', async (req, res) => { app.get('/api/v1/fields/names', async (req, res) => { try { - var values = await getDistinctValues(instances, (inst) => - cdm.getDistinctNames(inst, getYdm(inst, 'run', req)) - ); + var values = await getDistinctValues(instances, (inst) => cdm.getDistinctNames(inst, getYdm(inst, 'run', req))); serverLog('GET /api/v1/fields/names returned ' + values.length + ' value(s)'); res.json({ values: values }); } catch (error) { @@ -1486,9 +1535,7 @@ app.get('/api/v1/fields/names', async (req, res) => { app.get('/api/v1/fields/emails', async (req, res) => { try { - var values = await getDistinctValues(instances, (inst) => - cdm.getDistinctEmails(inst, getYdm(inst, 'run', req)) - ); + var values = await getDistinctValues(instances, (inst) => cdm.getDistinctEmails(inst, getYdm(inst, 'run', req))); serverLog('GET /api/v1/fields/emails returned ' + values.length + ' value(s)'); res.json({ values: values }); } catch (error) { @@ -1512,9 +1559,7 @@ app.get('/api/v1/fields/benchmarks', async (req, res) => { app.get('/api/v1/fields/tag-names', async (req, res) => { try { - var values = await getDistinctValues(instances, (inst) => - cdm.getDistinctTagNames(inst, getYdm(inst, 'tag', req)) - ); + var values = await getDistinctValues(instances, (inst) => cdm.getDistinctTagNames(inst, getYdm(inst, 'tag', req))); serverLog('GET /api/v1/fields/tag-names returned ' + values.length + ' value(s)'); res.json({ values: values }); } catch (error) { @@ -1582,7 +1627,18 @@ app.get('/api/v1/fields/primary-metrics', async (req, res) => { // -------------------------------------------------------------------------------------------------------------- app.post('/api/v1/metric-data', async (req, res) => { try { - var knownFields = ['run', 'period', 'begin', 'end', 'source', 'type', 'resolution', 'breakout', 'filter', 'instances']; + var knownFields = [ + 'run', + 'period', + 'begin', + 'end', + 'source', + 'type', + 'resolution', + 'breakout', + 'filter', + 'instances' + ]; var fieldErr = validateBodyFields(req.body, knownFields); if (fieldErr) { return res.status(400).json({ code: 'UNKNOWN_FIELDS', error: fieldErr }); @@ -1590,8 +1646,31 @@ app.post('/api/v1/metric-data', async (req, res) => { var { run, period, begin, end, source, type, resolution, breakout, filter, instances: reqInstances } = req.body; var reqStart = Date.now(); - var breakoutStr = Array.isArray(breakout) ? breakout.map(function (b) { return typeof b === 'object' && b.name ? b.name : b; }).join(',') : (breakout || 'none'); - serverLog('POST /api/v1/metric-data: ' + source + '::' + type + ' resolution=' + resolution + ' breakout=[' + breakoutStr + ']' + (filter ? ' filter=' + filter : '') + ' run=' + (run || 'none').toString().substring(0, 8) + '... period=' + (period || 'none').toString().substring(0, 8) + '...', req.reqId); + var breakoutStr = Array.isArray(breakout) + ? breakout + .map(function (b) { + return typeof b === 'object' && b.name ? b.name : b; + }) + .join(',') + : breakout || 'none'; + serverLog( + 'POST /api/v1/metric-data: ' + + source + + '::' + + type + + ' resolution=' + + resolution + + ' breakout=[' + + breakoutStr + + ']' + + (filter ? ' filter=' + filter : '') + + ' run=' + + (run || 'none').toString().substring(0, 8) + + '... period=' + + (period || 'none').toString().substring(0, 8) + + '...', + req.reqId + ); //serverLog(' curl: curl -s -X POST http://localhost:3000/api/v1/metric-data -H "Content-Type: application/json" -d \'' + JSON.stringify({ run: run, period: period, begin: begin, end: end, source: source, type: type, resolution: resolution, breakout: breakout, filter: filter }) + '\'', req.reqId); // Use instances from request if provided, otherwise use server's configured instances @@ -1672,7 +1751,10 @@ app.post('/api/v1/metric-data', async (req, res) => { var labelCount = metric_data && metric_data.values ? Object.keys(metric_data.values).length : 0; var elapsed = Date.now() - reqStart; - serverLog('POST /api/v1/metric-data: ' + source + '::' + type + ' -> ' + labelCount + ' label(s) in ' + elapsed + 'ms', req.reqId); + serverLog( + 'POST /api/v1/metric-data: ' + source + '::' + type + ' -> ' + labelCount + ' label(s) in ' + elapsed + 'ms', + req.reqId + ); // Return the data res.json(metric_data); From 34539bb41aaab1987dd368526ed1a037e6074203 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Thu, 30 Jul 2026 10:59:55 -0500 Subject: [PATCH 3/5] feat: add --aggregation override to get-metric queries Allow users to override the default aggregation method at query time with --aggregation . When specified, this takes priority over the default-aggregation value stored in the metric_desc document. Co-Authored-By: Claude Opus 4.6 (1M context) --- queries/cdmq/cdm.js | 18 +++++++++++------- queries/cdmq/get-metric-data.js | 5 ++++- queries/cdmq/server.js | 26 ++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index 1153ce7d..ef94ade2 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -3991,13 +3991,17 @@ getMetricDataSets = async function (instance, sets, yearDotMonth) { } for (var idx = 0; idx < sets.length; idx++) { - sets[idx].defaultAggregation = getDefaultAggregation( - instance, - sets[idx].run, - sets[idx].source, - sets[idx].type, - yearDotMonth - ); + if (sets[idx].aggregation) { + sets[idx].defaultAggregation = sets[idx].aggregation; + } else { + sets[idx].defaultAggregation = getDefaultAggregation( + instance, + sets[idx].run, + sets[idx].source, + sets[idx].type, + yearDotMonth + ); + } } var dataSets = await getMetricDataFromIdsSets(instance, sets, metricGroupIdsByLabelSets, yearDotMonth); diff --git a/queries/cdmq/get-metric-data.js b/queries/cdmq/get-metric-data.js index 160aa6ac..73918df5 100644 --- a/queries/cdmq/get-metric-data.js +++ b/queries/cdmq/get-metric-data.js @@ -9,6 +9,7 @@ // //# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=javascript +var cdm = require('./cdm'); var program = require('commander'); var sprintf = require('sprintf-js').sprintf; const http = require('http'); @@ -161,6 +162,7 @@ async function main() { '--filter ', '[optional] Filter out (do not output) metrics which do not pass the conditional. gt=greater-than, ge=greater-than-or-equal, lt=less-than, le=less-than-or-equal' ) + .option('--aggregation ', '[optional] Override the default aggregation method for this query') .option('--output-format ', 'table') .option( '--date-format ', @@ -205,7 +207,8 @@ async function main() { resolution: program.resolution, breakout: program.breakout, // Send as array to preserve complex breakout syntax filter: program.filter, - instances: program.instances.length > 0 ? program.instances : undefined // Pass instances to server if provided + aggregation: program.aggregation, + instances: program.instances.length > 0 ? program.instances : undefined }; // Fetch metric data from the API diff --git a/queries/cdmq/server.js b/queries/cdmq/server.js index 0a95e907..47cbe636 100755 --- a/queries/cdmq/server.js +++ b/queries/cdmq/server.js @@ -1637,13 +1637,34 @@ app.post('/api/v1/metric-data', async (req, res) => { 'resolution', 'breakout', 'filter', + 'aggregation', 'instances' ]; var fieldErr = validateBodyFields(req.body, knownFields); if (fieldErr) { return res.status(400).json({ code: 'UNKNOWN_FIELDS', error: fieldErr }); } - var { run, period, begin, end, source, type, resolution, breakout, filter, instances: reqInstances } = req.body; + var { + run, + period, + begin, + end, + source, + type, + resolution, + breakout, + filter, + aggregation, + instances: reqInstances + } = req.body; + + var validAggregations = ['sum', 'avg', 'max', 'min']; + if (aggregation && !validAggregations.includes(aggregation)) { + return res.status(400).json({ + code: 'INVALID_AGGREGATION', + error: "Invalid aggregation '" + aggregation + "'. Must be one of: " + validAggregations.join(', ') + }); + } var reqStart = Date.now(); var breakoutStr = Array.isArray(breakout) @@ -1738,7 +1759,8 @@ app.post('/api/v1/metric-data', async (req, res) => { end: end, resolution: resolution, breakout: breakout, - filter: filter + filter: filter, + aggregation: aggregation }; var resp = await cdm.getMetricDataSets(instance, [set], yearDotMonth); if (resp['ret-code'] != 0) { From aeab27420af66bd372d2141c8761409ebb601ba6 Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Thu, 30 Jul 2026 13:09:31 -0500 Subject: [PATCH 4/5] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20stale=20callsite=20and=20dead=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix sendMetricReq's calcAvg call to pass 'sum' as the defaultAggregation argument (legacy dead code, but correct the signature match). Remove unused baseFilter variable left over from query template refactoring. Co-Authored-By: Claude Opus 4.6 (1M context) --- queries/cdmq/cdm.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index ef94ade2..db42bc1f 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -3426,6 +3426,7 @@ sendMetricReq = async function ( jsonArrIdx, jsonArrTracker, tracker.numMetricIds, + 'sum', valueSets[setIdx][trackerLabel] ); } @@ -3642,12 +3643,6 @@ getMetricDataFromIdsSets = async function (instance, sets, metricGroupIdsByLabel var timeRangeTemplates = []; var thisBegin = begin; var thisEnd = begin + duration; - var baseFilter = - '[{"range":{"metric_data.end":{"lte":"' + - thisEnd + - '"}}},{"range":{"metric_data.begin":{"gte":"' + - thisBegin + - '"}}},{"terms":{"metric_desc.metric_desc-uuid":__IDS__}}]'; while (true) { var filter = '[{"range":{"metric_data.end":{"lte":"' + From 94dcb00f53f2b6cdcf22d1781196546baadadb6e Mon Sep 17 00:00:00 2001 From: Karl Rister Date: Thu, 30 Jul 2026 14:35:30 -0500 Subject: [PATCH 5/5] fix: address atheurer review feedback - Remove redundant default-aggregation assignment on v10dev (already inherited from v9dev deep clone) - Add validation warning in getDefaultAggregation for unrecognized stored values (falls back to sum with log) - Replace hardcoded v9dev/v10dev checks in getDocType and getIndexBaseName with isValidCdmVersion() so future versions don't need manual dispatch updates Co-Authored-By: Claude Opus 4.6 (1M context) --- queries/cdmq/cdm.js | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/queries/cdmq/cdm.js b/queries/cdmq/cdm.js index db42bc1f..243c0646 100644 --- a/queries/cdmq/cdm.js +++ b/queries/cdmq/cdm.js @@ -414,7 +414,7 @@ indexDefs['v8dev']['metric_data']['mappings']['properties']['metric_data'] = { }; indexDefs['v9dev']['metric_data'] = deepClone(indexDefs['v8dev']['metric_data']); -// v10dev: adds default-aggregation field to metric_desc for per-metric aggregation control +// v10dev inherits default-aggregation from v9dev via deep clone indexDefs['v10dev']['run_micro'] = deepClone(indexDefs['v9dev']['run_micro']); indexDefs['v10dev']['run'] = deepClone(indexDefs['v9dev']['run']); indexDefs['v10dev']['tag'] = deepClone(indexDefs['v9dev']['tag']); @@ -423,9 +423,6 @@ indexDefs['v10dev']['param'] = deepClone(indexDefs['v9dev']['param']); indexDefs['v10dev']['sample'] = deepClone(indexDefs['v9dev']['sample']); indexDefs['v10dev']['period'] = deepClone(indexDefs['v9dev']['period']); indexDefs['v10dev']['metric_desc'] = deepClone(indexDefs['v9dev']['metric_desc']); -indexDefs['v10dev']['metric_desc']['mappings']['properties']['metric_desc']['properties']['default-aggregation'] = { - type: 'keyword' -}; indexDefs['v10dev']['metric_def'] = deepClone(indexDefs['v9dev']['metric_def']); indexDefs['v10dev']['metric_data'] = deepClone(indexDefs['v9dev']['metric_data']); @@ -633,7 +630,8 @@ function getDocType(index) { } } - if (cdmVer == 'v9dev' || cdmVer == 'v10dev') { + // v9dev+ uses cdm-{ver}-{doctype}@{year}.{month} format + if (isValidCdmVersion(cdmVer)) { var regExp = /^cdm-v\d+dev-([^@]+)(@\d\d\d\d\.\d\d|\*)/; var matches = regExp.exec(index); if (matches) { @@ -664,7 +662,7 @@ function getIndexBaseName(instance) { //debuglog('cdmver: [' + cdmVer + ']'); if (cdmVer == 'v7dev' || cdmVer == 'v8dev') { return 'cdm' + cdmVer + '-'; - } else if (cdmVer == 'v9dev' || cdmVer == 'v10dev') { + } else if (isValidCdmVersion(cdmVer)) { // v9dev+ adds a '-' after 'cdm' because of a [lab admin] naming convention // used for shared opensearch. Therefore, you will find that v7dev // and v8dev cannot be used for some [lab managed] opensearch instances with @@ -3457,12 +3455,26 @@ getDefaultAggregation = function (instance, run, source, type, yearDotMonth) { } } }; + var validAggregations = ['sum', 'avg', 'max', 'min']; var resp = esRequest(instance, 'metric_desc', '/_search', q, yearDotMonth); var data = JSON.parse(resp.getBody()); if (data.hits && data.hits.hits && data.hits.hits.length > 0) { var md = data.hits.hits[0]._source.metric_desc; if (md && md['default-aggregation']) { - return md['default-aggregation']; + var agg = md['default-aggregation']; + if (!validAggregations.includes(agg)) { + console.log( + 'WARNING: metric_desc for ' + + source + + '::' + + type + + ' has unrecognized default-aggregation "' + + agg + + '", falling back to sum' + ); + return 'sum'; + } + return agg; } } return 'sum';