// author: Neil
// date: 2/1/2015
// version: 0.9
// description: Dashboard1
// inputs: action,update,debug
// resultFormat: html
// ttl: 60

// How to use this script.
// 1. Add authentication restrictions above (see REST API tutorial for examples).
// 2. Edit the spec {} structure below
//       - define "value" entries
//       - use them to define "cell" entries
//       - arrange the cells in the "layout"
// 3. Access the URL.  The dashboard should appear.  If not, try troubleshooting steps:
//    (a) try <URL>?action=spec - should dump the spec.
//    (b) try <URL>?action=data&debug=1 - should describe the queries without running them.
//    (c) try <URL>?action=data  - this will query for the data.
//    (c) try <URL>?action=data&update=true  - just the last 10 minutes of data.

var action = action||"dash";
var update = update || "no";
var debug = debug || 0;
debug = parseInt(debug);

///////////////////// Dashboard spec /////////////////////////////

var spec = {
    "title":"example dashboard page title",
    "value": {
	// Named value-specs that we will use below, fields are:
	//   select[] -- required -- query fields
	//   units[] -- required -- strings
	//   scale[] -- optional -- numbers
	//   colors[] -- optional -- colors
	"bps": {
	    "select":[ "rate(bytes)" ],
	    "scale": [8],
	    "units": ["Bits Per Second"],
	},
	"fps": {
	    "select": [ "rate(frames)" ],
	    "units": ["Packets Per Second"]
	},
	"c_bps": {
	    "select": [ "rate(ifinoctets)" ],
	    "scale": [8],
	    "units": ["Bits/sec"]
	},
	"c_errs": {
	    "select": [ "ifinerrors","ifouterrors","sum(ifinerrors ifouterrors)"],
	    "units": ["Errors In", "Errors Out", "Errors"]
	},
	"c_errs_detail": {
	    "select": [ "symbolerrors","fcserrors","alignmenterrors"],
	    "units": ["Symbol Errors", "FCS Errors", "Alignment Errors"]
	},
	"loadavg": {
	    "select": ["load_one"],
	    "units": ["Load Avg (1 min)"],
	    "colors": ["yellow"]
	},
	"loadpercpu": {
	    "select": ["loadpercpu_one"],
	    "units": ["Load Avg Per CPU (1 min)"],
	    "colors": ["yellow"]
	},
	"temp": {
	    "select": ["temperature"],
	    "units": ["C"],
	},
	"env": {
	    "select": ["power","psus","psus_failed","fans","fans_failed","temperature"],
	    "units": ["W","PSUs","PSUs Failed","Fans","Fans Failed","C"],
	},
    },
    "common": {
	// Default settings for cells below.
	"minutes": 60,        // time-window
	"group": 1,           // time-grouping
	"view": "rttraffic",  // query view
	"filter": {
	    // default filter (by view)
	    "rttraffic": null,  // e.g. "ipsource=10.0.0.0/8"
	    "ifcounters": null,
	    "events": null
	},
	"n": 5,                  // top-N
	"globaltruncate": false, // true => topN over all minutes; false => topN per time-group
	"includeother": false,   // true => include total for keys not in topN
	"nullsinkeys": false,    // true => allow some keys to be missing
	"keys": null,            // [] default query keys
	"keynames": null,        // [] display names for keys
	"values": null,          // [] from value-spec names defined above
	"table": false,          // true => HTML table; false => trend chart
	"stack": true,
    },
    "cells": {
	// Named chart/table cells.
	// Defaults inherited from "common" above.
	"chart1": {
	    "minutes": 300,
	    "keys": ["sourceaddress"],
	    "keynames": ["Source Address"],
	    "values": ["bps"],
	    "includeother": true
	},
	"chart2": {
	    "keys": ["destinationaddress"],
	    "keynames": ["Destination Address"],
	    "values": ["bps"]
	},
	"chart3": {
	    "view": "ifcounters",
	    "keys": ["agent"],
	    "keynames": ["Device"],
	    "values": ["temp"],
	    "stack": false,
	},
	"chart4": {
            "title": "Hosts load-avg per cpu",
	    "view": "host",
	    "values": ["loadpercpu"],
	    "keys": ["hostname"],
	},
	"table1": {
	    "table": true,
	    "view": "ifcounters",
	    "title": "Env Data",
	    "keys": ["agent"],
            "keynames": ["Device"],
	    "values": ["env"],
	    "n": 10
	},
	"table2": {
	    "table": true,
	    "view": "ifcounters",
	    "keys": ["interface"],
	    "values": ["c_errs_detail"],
	    "n": 10
	}
    },
    "layout": [
	// arrange cells in rows
	[ "chart1", "chart2" ],
	[ "chart3", "chart4" ],
	[ "table1", "table2" ],
    ]
};

///////////////////// shared /////////////////////////////

function get_divclass(table) { return table ? "topntable" : "trend"; }

///////////////////// actions /////////////////////////////


if(action == "spec") {
    println(JSON.stringify(spec, null, 4));
}
else if(action == "data") {
    var separator = "_SEP_";
    var otherstr = "-other-";

    // utility to scale values in a callback row
    function scaleValues(row, scale) {
	if(scale.length) {
	    for(var ss = 1; ss < scale.length; ss++) {
		var scl = scale[ss];
		var idx = row.length - scale.length + ss;
		var val = row[idx];
		if(val && scl) row[idx] = val * scl;
	    }
	}
    }

    // function used to build query-callback for a query row
    function topNTrendCallback(chart, cell, scale) {

	// utility to represent a compound key as a string
	function extractKey(row) {
	    // assume row[0] is time, and keys are next
	    var key = "";
	    var other = true;
	    if(cell.keys) {
		for(var kk = 1; kk < (1 + cell.keys.length); kk++) {
		    if(row[kk]) other = false;
		    if(kk > 1) key += separator;
		    key += row[kk] || "null";
		}
	    }
	    return other ? otherstr : key;
	}

	return function(row) {
	    var times = chart.trend.times;
	    var trend = chart.trend.trends[chart.id];
	    var tbin = chart.trend.maxpoints-1;
	    var tim = row[0].getTime();
	    var key = extractKey(row);
	    scaleValues(row, scale);
	    // take last value
	    var val = row[row.length-1];
	    if(tbin == -1 || tim != times[tbin]) {
		// start new time-bin
		chart.trend.maxpoints++;
		times.push(tim);
		trend.push({});
		tbin++;
		chart.trailingNull = true;
	    }
	    // add key-value pair to time-bin
	    var topN = trend[tbin];
	    topN[key] = val;
	    if(val) chart.trailingNull = false;
	};
    }

    // function used to build query-callback for a query row
    // where there are no keys - only one or more values
    function valueTrendCallback(chart, cell, scale, units) {

	return function(row) {
	    var times = chart.trend.times;
	    var trend = chart.trend.trends[chart.id];
	    var tbin = chart.trend.maxpoints-1;
	    var tim = row[0].getTime();
	    if(tbin == -1 || tim != times[tbin]) {
		// start new time-bin
		chart.trend.maxpoints++;
		times.push(tim);
		trend.push({});
		tbin++;
		chart.trailingNull = true;
	    }
	    scaleValues(row, scale);
	    for(var vv = 0; vv < units.length; vv++) {
		var key = units[vv];
		var val = row[1 + vv];
		// add key-value pair to time-bin - will this scramble the ordering?
		var topN = trend[tbin];
		topN[key] = val;
		if(val) chart.trailingNull = false;
	    }
	};
    }


    function tableQueryCallback(chart, cell, scale) {
	return function(row) {
	    scaleValues(row, scale);
	    chart.rows.push(row);
	};
    }

    function tableTailQueryCallback(chart, cell, scale) {
	return function(row) {
	    scaleValues(row, scale);
	    // just want the last N rows, newest first
	    // (events query sometimes gives more than N)
	    chart.rows.unshift(row);
	    if(chart.rows.length > chart.n) {
		chart.rows.pop();
	    }
	};
    }

    // build a multiquery job for each view, but only for
    // the cells that made it into the layout
    var cellsIncluded = {};
    for each(var row in spec.layout) {
	for each(var cellID in row) {
	    cellsIncluded[cellID] = true;
	}
    }

    var jobs = {};
    for (var cellID in spec.cells) {
	if(!cellsIncluded[cellID]) continue;
	var cell = spec.cells[cellID];
	var view = cell.view || spec.common.view;
	var job = jobs[view];
	if(job == null) {
	    job = jobs[view] = {};

	    job.q = new Query();
	    job.q.view = view;
	    job.q.multiquery = true;
	    job.q.select = [];
	    job.q.where = [];
	    job.q.interval = [];
	    job.q.group = [];
	    job.q.sort = [];
	    job.q.truncate = [];
	    job.q.includeother = [];
	    job.q.nullsinkeys = [];
	    job.q.globaltruncate = [];
	    job.q.threads = 1;

	    job.callbacks = [];
	    job.charts = {};
	}

	// construct query for this cell
	var stack = cell.stack==null ? (spec.common.stack==null ? true : spec.common.stack) : cell.stack;
	var table = cell.table==null ? (spec.common.table||false) : cell.table;
	var divclass = get_divclass(table);
	var select = [];

	// tables don't inherit the 'group' setting since it's so
	// unlikely that we want time-grouping there.
	var default_group = table ? null : spec.common.group;
	var group = cell.group==null ? default_group : cell.group;

	// time always comes first, if we are grouping
	if(group) select.push("time");

	// then keys
	var keys = cell.keys || spec.common.keys;
	if(keys == null) keys = [];
	select = select.concat(keys);

	// pick up keynames here too
	var keynames = cell.keynames || spec.common.keynames;
	if(keynames == null) keynames = keys;

	// now add values and their properties
	var values = cell.values || spec.common.values;
	var sort = null;
	var nvalues = 0;
	var units = [];
	var sortUnits = null;
	var scale = [];
	var colors = [];

	if(cell.values) {
	    for each (var valkey in values) {
		var valspec = spec.value[valkey];
		if(valspec) {
		    nvalues += valspec.select.length;
		    select = select.concat(valspec.select);
		    
		    if(valspec.units) units = units.concat(valspec.units);
		    else units = units.concat(valspec.select);  // fall back on select strings
		    
		    if(valspec.scale) scale = scale.concat(valspec.scale);
		    else for(var ss = 0; ss < valspec.select.length; ss++) scale.push(1); // no scale => all '1's

		    if(valspec.colors) colors = colors.concat(valspec.colors);
		    else for(var ss = 0; ss < valspec.select.length; ss++) colors.push(""); // "" => auto
		}
	    }
	    sort = select[select.length - 1]; // sort by last value
	    sortUnits = units[units.length - 1];

	}

	// add the cell's query to the multiquery for this view
	job.q.select.push(select.join(","));
	job.q.sort.push(sort);
	job.q.where.push(cell.filter==null ? spec.common.filter[view] : cell.filter);
	var minutes = 10; // update only, but allow for recent data rewrite (DNS, NetFlow etc.)
	if(update == "no") minutes = (cell.minutes || spec.common.minutes);
	job.q.interval.push("now - " + minutes + " min,last");
	job.q.group.push(group);
	var truncate = cell.n || spec.common.n;
	job.q.truncate.push(truncate);
	job.q.includeother.push(cell.includeother==null ? (spec.common.includeother||false) : cell.includeother);
	job.q.nullsinkeys.push(cell.nullsinkeys==null ? (spec.common.nullsinkeys||false) : cell.nullsinkeys);
	job.q.globaltruncate.push(cell.globaltruncate==null ? (spec.common.globaltruncate||false) : cell.globaltruncate);

	// add keynames, color, units etc. to the chart object so that
	// they are visible to the poller when charts are constructed
	var chart = job.charts[cellID] = {
	    "id": cellID,
	    "keynames": keynames,
	    "units": sortUnits,
	    "separator": separator,
	    "otherstr": otherstr,
	    "divclass": divclass,
	    "nvalues": nvalues,
	    "colors": colors,
	    "n": truncate,
	    "stack": stack
	};

	// now prepare the chart object to receive data row by row as the query runs
	if(divclass == "trend") {
	    // build the chart data structure to match what chart is looking for
	    chart.trend =  { 'maxpoints': 0, 'times': [], "trends": {}};
	    chart.trend.trends[cellID] = [];
	    // and add the query callback to populate it.  Note that we use a
	    // function-to-build-a-function so that 'chart' is captured in closure
	    if(cell.keys && cell.keys.length > 0) {
		job.callbacks.push(topNTrendCallback(chart, cell, scale));
	    }
	    else {
		job.callbacks.push(valueTrendCallback(chart, cell, scale, units));
	    }
	}
	else if(divclass == "topntable") {
	    // client side expects just an array of arrays in "chart.rows"
	    chart.rows = [];
	    if(nvalues) {
		// round out the keynames list to be the table column headings
		if(group) chart.keynames.unshift("time");
		chart.keynames = chart.keynames.concat(units);
	    }
	    if(view == "events") {
		// if "id" is in the keys, then this is a "tailing" query
		var tail = false;
		for each(var kk in keys) if(kk == "id") tail = true;
		chart.tail = tail;
	    }
	    job.callbacks.push(chart.tail ?
			       tableTailQueryCallback(chart, cell, scale) :
			       tableQueryCallback(chart, cell, scale));
	}
	else {
	    throw("unknown divclass: " + divclass);
	}
    }

    function cleanupChart(chart) {
	// remove trailing nulls - cause by "last" timebin not having data yet
	if(chart
	   && chart.trailingNull
	   && chart.trend
	   && chart.trend.times) {
	    chart.trend.times.pop();
	}
    }
	
    if(debug) {
	// just print the jobs
	println(JSON.stringify(jobs, null, 4));
    }
    else {
	// run each multiquery job, and collect results
	var result = {};
	result.ok = true;
	result.charts = {};
	try {
	    for each (var job in jobs) {
		job.q.run(job.callbacks);
		for (var ch in job.charts) {
		    var chart = job.charts[ch];
		    cleanupChart(chart);
		    result.charts[ch] = chart;
		}
	    }
	} catch(err) {
	    result.ok=false;
	    result.error = err;
	}
	println(JSON.stringify(result, null, 2));
    }
}
else {
    // action = "dash" -- emit HTML describing the dashboard

    function get_cell_title(cell) {
	var title = cell.title;
	if(title == null) {
	    // make up a title from the spec
	    var keynames = cell.keynames || cell.keys;
	    var view = cell.view || spec.common.view;
	    var filter = cell.filter==null ? spec.common.filter[view] : cell.filter;
	    if(keynames) {
		title = "Top " + keynames.join();
		if(cell.values) title += " By: " + cell.values.join();
		if(filter) title += " Where: " + filter;
	    }
	    else if(cell.values) {
		title = "Trend " + cell.values,join();
		if(filter) title += " Where: " + filter;
	    }
	}
	return title;
    }

    println("<html>");
    println("<head>");
    println('<link rel="icon" type="image/vnd.microsoft.icon" href="/inmsf/images/favicon.ico" />');
    println('<link rel="stylesheet" href="/inmsf/inc/dash/jquery-ui-1.11.0/jquery-ui.min.css" type="text/css"/>');
    println('<link rel="stylesheet" href="/inmsf/inc/dash/jquery-ui-1.11.0/jquery-ui.structure.min.css" type="text/css"/>');
    println('<link rel="stylesheet" href="/inmsf/inc/dash/jquery-ui-1.11.0/jquery-ui.theme.min.css" type="text/css"/>');
    println('<link rel="stylesheet" href="/inmsf/inc/dash/css/dash.css" type="text/css"/>');
    println('<link rel="stylesheet" href="/inmsf/inc/dash/css/stripchart.css" type="text/css"/>');
    println('<script type="text/javascript" src="/inmsf/inc/dash/jquery-2.1.1.js"></script>');
    println('<script type="text/javascript" src="/inmsf/inc/dash/jquery-ui-1.11.0/jquery-ui.min.js"></script>');
    println('<script type="text/javascript" src="/inmsf/inc/dash/jquery.stripchart.js"></script>');
    println('<script type="text/javascript" src="/inmsf/inc/dash/chart.js"></script>');

    // poll for data periodically
    println('<script type="text/javascript" src="/inmsf/inc/dash/poller2.js"></script>');

    println("</head>");
    println("<body>");
    println('<div id="dashboard">');
    println('<table>');
    println('<thead>');
    println('<tr><th colspan="2">' + spec.title + '</th></tr>');
    println('</thead>');
    println('<tbody>');

    for each(var row in spec.layout) {
	println('<tr>');
	for each(var cellID in row) {
	    var cell = spec.cells[cellID];
	    var table = cell.table==null ? (spec.common.table||false) : cell.table;
	    var divclass = get_divclass(table);
	    var title = get_cell_title(cell);

	    println('<td>');
	    println('<div>');
	    println('<h3>' + title + '</h3>');
	    println('<div id="' + cellID + '" class="' + divclass + '" </div>');
	    println('</div>');
	    println('</td>');
	}
	println('</tr>');
    }

    println('</tbody>');
    println('</table>');
    println("</div>");
    println("</body>");
    println("</html>");
}