Appearance
Cyber security log analysis
This example shows how to use NodeFilters and EdgeFilters to cleanup the graph and focus on a certain period of time, or on nodes with a certain property.
Note that the graph is not modified, only the view is filtered.
The timeline shown here is very simple, but we provide a much more complete timeline plugin here: ogma-timeline
js
import Ogma from '@linkurious/ogma';
import { TimeFilter } from './timefilter';
const ogma = new Ogma({
container: 'graph-container',
options: {
backgroundColor: null,
detect: { nodeTexts: false, edgeTexts: false }
}
});
// node types
const PORT = 'port';
const SERVER = 'server';
const IP = 'ip';
// edge types
const LISTENS_TO = 'listens_to';
const CONNECTS_TO = 'connects_to';
const ANIMATION_DURATION = 0;
const appState = {
edgeGrouping: true,
selectedPorts: {},
timeFilter: [-Infinity, Infinity]
};
// transformations
const transformations = {
edgeGrouping: null,
portFilter: null,
timeFilter: null
};
const initTransformations = () => {
// port filtering policy
transformations.portFilter = ogma.transformations.addNodeFilter({
criteria: node => {
if (node.getData('type') === PORT) {
return appState.selectedPorts[node.getData('port')];
} else {
return true;
}
},
enabled: true
});
// order matters, this has to go before the edge grouping
transformations.timeFilter = ogma.transformations.addEdgeFilter({
criteria: edge => {
if (edge.getData('type') === CONNECTS_TO) {
const date = edge.getData('date') || appState.timeFilter[0];
return appState.timeFilter[0] <= date && date <= appState.timeFilter[1];
}
return true;
}
});
// Edge grouping policy:
//
// group all edges between an “IP” and a “PORT” node by (protocol + service)
// Grouped edge data:
// - protocol
// - service
// - total query bytes
// - total response bytes
// - group size (number of edges in the group)
transformations.edgeGrouping = ogma.transformations.addEdgeGrouping({
selector: edge =>
!edge.isExcluded() && edge.getData('type') === CONNECTS_TO,
groupIdFunction: e => e.getData('protocol') + '/' + e.getData('service'),
generator: edges => {
const e = edges.get(0);
const protocol = e.getData('protocol');
const service = e.getData('service');
let query_bytes = 0,
response_bytes = 0;
edges.getData().forEach(edgeData => {
query_bytes += edgeData.query_bytes || 0;
response_bytes += edgeData.response_bytes || 0;
});
return {
data: {
type: CONNECTS_TO,
query_bytes: query_bytes,
response_bytes: response_bytes,
group_size: edges.size,
protocol: protocol,
service: service
}
};
},
enabled: true
});
};
const layout = () =>
ogma.layouts.force({
gpu: true,
charge: 7,
locate: true
});
// format traffic values
const formatPutThrough = bytes => {
bytes = +bytes;
return bytes > 1024 * 1024
? Math.round(bytes / (1024 * 1024)) + 'M'
: bytes > 1024
? Math.round(bytes / 1024) + 'KB'
: bytes + 'B';
};
// Style the graph based on the properties of nodes and edges
const addStyles = () => {
ogma.styles.addNodeRule(
node => {
return node.getData('type') === SERVER;
},
{
radius: 10,
color: '#fff',
icon: {
font: 'Font Awesome 5 Free',
content: '\uF233',
color: '#0094ff',
style: 'bold',
minVisibleSize: 0,
scale: 0.7
}
}
);
// node styles
ogma.styles.addNodeRule(
n => {
return n.getData('type') === IP;
},
{
innerStroke: '#0094ff',
color: '#fff',
// number of outgoing “CONNECTS_TO” edges (taking filters into account)
radius: node =>
10 +
Math.min(
node.getAdjacentEdges({ direction: 'out ' }).filter(edge => {
return edge.getData('type') === CONNECTS_TO;
}).size,
40
),
icon: {
font: 'Font Awesome 5 Free',
content: '\uF233',
style: 'bold',
color: '#0094ff',
minVisibleSize: 0,
scale: 0.4
},
text: {
content: node => node.getData('ip'),
secondary: {
content: (
node // number of outgoing “CONNECTS_TO” edges (taking filters into account)
) =>
node.getAdjacentEdges({ direction: 'out ' }).filter(edge => {
return edge.getData('type') === CONNECTS_TO;
}).size +
' connections\n' +
// received bytes (sum of all “response_bytes” from outgoing
// “CONNECTS_TO” edges, taking filters into account)
formatPutThrough(
node
.getAdjacentEdges({ direction: 'out' })
.filter(edge => {
return edge.getData('type') === CONNECTS_TO;
})
.reduce((total, edge) => {
return total + edge.getData('response_bytes');
}, 0)
) +
' received\n' +
// sent bytes (sum of all “query_bytes” from outgoing “CONNECTS_TO” edges,
// taking filters into account)
formatPutThrough(
node
.getAdjacentEdges({ direction: 'out' })
.filter(edge => {
return edge.getData('type') === CONNECTS_TO;
})
.reduce((total, edge) => {
return total + edge.getData('query_bytes');
}, 0)
) +
' sent\n',
color: '#0094ff',
backgroundColor: 'white',
minVisibleSize: 60,
margin: 1
},
color: 'white',
backgroundColor: '#0094ff'
}
}
);
ogma.styles.addNodeRule(
n => {
return n.getData('type') === PORT;
},
{
color: 'orange',
shape: 'square',
text: {
content: node => ':' + node.getData('port'),
position: 'center'
},
radius: 8,
// Highlight Port 1900 as dangerous
pulse: {
enabled: node => node.getData('port') === '1900',
endRatio: 5,
width: 1,
startColor: 'red',
endColor: 'red',
interval: 1000,
startRatio: 1.0
}
}
);
ogma.styles.addEdgeRule({
color: edge =>
edge.getData('type') === CONNECTS_TO ? '#0094ff' : 'orange',
width: edge => 1 + 1.3 * Math.sqrt(edge.getData('group_size')),
shape: edge => (edge.getData('type') === CONNECTS_TO ? null : 'arrow'),
text: {
content: edge => {
if (edge.getData('type') === CONNECTS_TO) {
const response_bytes = edge.getData('response_bytes') || 0;
const query_bytes = edge.getData('query_bytes') || 0;
return (
formatPutThrough(query_bytes) +
' in, ' +
formatPutThrough(response_bytes) +
' out'
);
} else {
return LISTENS_TO;
}
},
color: edge =>
edge.getData('type') === CONNECTS_TO ? '#0094ff' : 'orange'
}
});
ogma.styles.setHoveredNodeAttributes({
text: {
backgroundColor: '0094ff'
}
});
ogma.styles.setSelectedNodeAttributes({
text: {
style: 'bold',
backgroundColor: '0094ff'
}
});
ogma.styles.setHoveredEdgeAttributes({
text: {
backgroundColor: 'white'
}
});
ogma.styles.setSelectedEdgeAttributes({
text: {
style: 'bold',
backgroundColor: 'white'
}
});
};
const updateState = () => {
// port filters
const ports = document.querySelectorAll('#ports input');
Array.prototype.forEach.call(ports, portInput => {
const port = portInput.getAttribute('data-port-id');
appState.selectedPorts[port] = portInput.checked;
});
// edge grouping
appState.edgeGrouping = document.getElementById('group-edges').checked;
// time filter
if (appState.edgeGrouping !== transformations.edgeGrouping.isEnabled()) {
transformations.edgeGrouping.toggle(ANIMATION_DURATION);
}
transformations.portFilter.refresh(ANIMATION_DURATION);
return ogma.transformations.afterNextUpdate();
};
const initUI = graph => {
createPortSelector();
appState.timeFilter = ogma
.getEdges()
.filter(edge => {
return edge.getData('type') === CONNECTS_TO;
})
.getData('date')
.reduce(
(acc, date) => {
acc[0] = Math.min(acc[0], date);
acc[1] = Math.max(acc[1], date);
return acc;
},
[Infinity, -Infinity]
);
new TimeFilter(document.getElementById('timeline'), () => {
if (transformations.timeFilter) {
transformations.timeFilter.refresh();
}
}).update(
ogma
.getEdges('raw')
.filter(edge => {
return edge.getData('type') === CONNECTS_TO;
})
.getData()
.sort((e1, e2) => {
return e1.date - e2.date;
}),
appState
);
document.getElementById('filters').addEventListener('change', updateState);
};
const createPortSelector = graph => {
const portValues = {};
appState.selectedPorts = {};
ogma
.getNodes()
.filter(node => {
return node.getData('type') === PORT;
})
.forEach(node => {
const ref = node.getData();
const port = ref.port;
portValues[port] = portValues[port] || [];
portValues[port].push(node.getId());
appState.selectedPorts[port] = true;
});
const container = document.getElementById('ports');
appState.ports = portValues;
container.innerHTML = Object.keys(portValues)
.sort((a, b) => {
return portValues[b].length - portValues[a].length;
})
.map(port => {
return (
'<input type="checkbox" data-port-id="' +
port +
'" ' +
'checked id="port-' +
port +
'" name="port-' +
port +
'">' +
'<label for="port-' +
port +
'" title="' +
portValues[port].length +
' connections"> ' +
port +
'</label>'
);
})
.join('');
};
// load data
const loadAndParseCSV = url =>
new Promise((complete, error) => {
Papa.parse(url, {
download: true,
complete: complete,
error: error
});
});
loadAndParseCSV('./logs-merged_sample.csv')
.then(records => recordsToGraph(records.data))
.then(graph => ogma.setGraph(graph))
.then(() => layout())
.then(graph => initUI(graph))
.then(initTransformations)
.then(() => updateState())
.then(() => addStyles())
.then(initUI)
.then(() => ogma.view.locateGraph({ padding: 20 }));
const recordsToGraph = records => {
// deduplication lookup hashes
const portsMap = {};
const ipsMap = {};
const serversMap = {};
const listensMap = {};
const nodes = [],
edges = [];
records.forEach(record => {
const [
timestamp,
src_ip,
dest_ip,
dest_port,
protocol,
service,
duration,
out_bytes,
in_bytes,
status
] = record;
// Port node
const portId = dest_ip + ':' + dest_port;
let port = portsMap[portId];
if (!port) {
port = portsMap[portId] = {
id: portId,
data: {
ip: dest_ip,
port: dest_port,
type: PORT
}
};
nodes.push(port);
}
// Server node
const serverId = dest_ip;
let server = serversMap[serverId];
if (!server) {
server = serversMap[serverId] = {
id: serverId,
data: {
ip: dest_ip,
type: SERVER
}
};
nodes.push(server);
}
// "Listens to" edge
let listen = listensMap[portId];
if (!listen) {
listen = listensMap[portId] = {
id: portId,
source: serverId,
target: portId,
data: {
type: LISTENS_TO
}
};
edges.push(listen);
}
// IP node
const ipId = src_ip;
let ip = ipsMap[ipId];
if (!ip) {
ip = ipsMap[ipId] = {
id: ipId,
data: {
ip: src_ip,
type: IP
}
};
nodes.push(ip);
}
// Connection edge
const connectionId = `${src_ip}-${dest_ip}:${dest_port}@${Math.round(
timestamp * 1000
)}`;
const connection = {
id: connectionId,
source: ipId,
target: portId,
data: {
type: CONNECTS_TO,
date: Math.round(timestamp * 1000),
query_bytes: isNaN(out_bytes) ? 0 : parseInt(out_bytes),
response_bytes: isNaN(in_bytes) ? 0 : parseInt(in_bytes),
protocol: protocol,
service: service
}
};
edges.push(connection);
});
return { nodes: nodes, edges: edges };
};
html
<!DOCTYPE html>
<html>
<head>
<title>Ogma cyber security analysis</title>
<link
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css"
rel="stylesheet"
/>
<link type="text/css" rel="stylesheet" href="style.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.3.0/papaparse.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/crossfilter2/1.4.6/crossfilter.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>
</head>
<body>
<i class="fa fa-camera-retro fa-1x" style="color: rgba(0, 0, 0, 0)"></i>
<div id="graph-container"></div>
<div id="ui">
<form id="filters">
<h3>Ports</h3>
<div id="ports"></div>
<h3>Connections</h3>
<input type="checkbox" id="group-edges" checked name="group-edges" />
<label for="group-edges">Group</label>
</form>
<div id="timeline"></div>
</div>
<script src="index.js"></script>
</body>
</html>
css
body {
margin: 0;
padding: 0;
width: 100%;
height: 100%;
font:
14px/20px 'Open Sans',
Arial,
sans-serif;
}
*,
*:before,
*:after {
box-sizing: border-box;
}
[type='checkbox']:checked,
[type='checkbox']:not(:checked) {
position: absolute;
left: -9999px;
}
[type='checkbox']:checked + label,
[type='checkbox']:not(:checked) + label {
position: relative;
padding-left: 28px;
cursor: pointer;
line-height: 20px;
display: inline-block;
}
[type='checkbox']:checked + label:before,
[type='checkbox']:not(:checked) + label:before {
content: '';
position: absolute;
left: 0;
top: 0;
width: 18px;
height: 18px;
border: 1px solid #ddd;
background: #ddd;
}
[type='checkbox']:checked + label:after,
[type='checkbox']:not(:checked) + label:after {
content: '';
width: 8px;
height: 8px;
background: black;
position: absolute;
top: 5px;
left: 5px;
-webkit-transition: all 0.2s ease;
transition: all 0.2s ease;
}
[type='checkbox']:not(:checked) + label:after {
opacity: 0;
-webkit-transform: scale(0);
transform: scale(0);
}
[type='checkbox']:checked + label:after {
opacity: 1;
-webkit-transform: scale(1);
transform: scale(1);
}
#graph-container {
top: 0;
bottom: 0;
left: 0;
right: 0;
position: absolute;
margin: 0;
overflow: hidden;
}
#filters {
position: absolute;
background: white;
top: 10px;
left: 10px;
border-radius: 5px;
font-family: Georgia, 'Times New Roman', Times, serif;
padding: 10px 20px;
box-shadow: 0 0 5px rgba(0, 0, 0, 0.3);
}
#ui h3 {
font-size: 1em;
}
#ports {
width: 200px;
display: block;
padding-bottom: 10px;
}
#ports label {
display: inline-block;
width: 40%;
}
#timeline {
position: absolute;
bottom: 0;
width: 100%;
}
#time-filter {
position: absolute;
bottom: 10px;
right: 10px;
width: 100%;
}
#time-filter svg {
float: right;
right: 0;
}
#time-filter .bar {
fill: rgba(70, 130, 180, 0.75);
}
#time-filter .selected .bar {
fill: rgba(255, 0, 0, 0.75);
}
.handle.handle--e,
.handle.handle--w {
fill: #ffffff;
stroke: #000000;
stroke-width: 0.3;
}
js
// d3-based simple timline UI
const MS_IN_HOUR = 1000 * 1000 * 60 * 60;
const getSize = container => {
const width = container.offsetWidth - 20;
const height = container.offsetHeight;
return { width: width, height: height };
};
export class TimeFilter {
constructor(container, update) {
const id = 'time-filter';
this._container =
document.getElementById(id) || document.createElement('div');
this._container.id = id;
this._update = update;
window.addEventListener('resize', () => {
return this._updateSize();
});
container.appendChild(this._container);
}
_updateSize() {
const ref = getSize(this._container);
const width = ref.width;
const height = ref.height;
d3.select('svg').attr('width', width).attr('height', height);
}
init(appState) {
this._container.innerHTML = '<svg></svg>';
const ref = getSize(this._container);
const width = ref.width;
const height = ref.height;
const buckets = this._buckets;
const svg = d3.select('svg');
const margin = { top: 0, right: 0, bottom: 0, left: 0 };
const g = svg
.append('g')
.attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
svg.attr('width', width).attr('height', height + 30);
const min = buckets[0].start;
const max = buckets[buckets.length - 1].end;
const minValue = 0;
const maxValue = buckets.reduce((a, b) => {
return Math.max(a, b.value);
}, 0);
const x = d3.scaleLinear().domain([min, max]).range([0, width]);
const y = d3.scaleLinear().domain([0, maxValue]).range([height, 0]);
const brushed = extent => {
extent = extent || d3.event.selection.map(x.invert, x);
const ids = [];
let start = Infinity,
end = -Infinity;
bar.classed('selected', d => {
const inside = extent[0] <= d.start && d.end <= extent[1];
if (inside) {
start = Math.min(start, d.start);
end = Math.max(end, d.end);
}
return inside;
});
appState.timeFilter = [start, end];
this._update();
};
const brush = d3
.brushX()
.extent([
[0, 0],
[width, height]
])
.on('start brush', brushed);
const barWidth = width / buckets.length;
const bar = g
.selectAll('g')
.data(buckets)
.enter()
.append('g')
.attr('transform', (d, i) => {
return 'translate(' + i * barWidth + ',0)';
});
bar
.append('rect')
.attr('y', d => {
return y(d.value);
})
.attr('height', d => {
return height - y(d.value);
})
.attr('width', barWidth)
.attr('class', 'bar');
const beforebrushed = () => {
d3.event.stopImmediatePropagation();
d3.select(this.parentNode).transition().call(brush.move, x.range());
};
g.append('g')
.call(brush)
.call(brush.move, [min + MS_IN_HOUR, max - MS_IN_HOUR].map(x))
.selectAll('.overlay')
.on('mousedown touchstart', beforebrushed, true);
g.append('g')
.attr('transform', 'translate(0,' + height + ')')
.call(
d3.axisBottom(x).tickFormat(d => {
const date = new Date(d);
return date.getHours() + ':' + date.getMinutes();
})
);
brushed([buckets[0].end, buckets[buckets.length - 1].start]);
}
update(edges, appState) {
let start = edges[0].date;
let next = start + MS_IN_HOUR;
let value = 0;
const buckets = [
{
start: edges[0].date - MS_IN_HOUR,
end: edges[0].date,
next: next,
value: value
}
];
edges.forEach((e, i) => {
while (e.date > next) {
// store
buckets.push({ start: start, end: next, value: value });
// reset
start = next;
next = start + MS_IN_HOUR;
value = 0;
}
value += e.query_bytes || 0;
});
if (value !== 0) {
buckets.push({ start: start, end: next, value: value });
}
buckets.push({ start: next, end: next + MS_IN_HOUR, next: 0, value: 0 });
this._buckets = buckets;
this.init(appState);
}
}