Cyber security log analysis

cyber-securitytimelinefiltering

This example shows how to use NodeFilters and EdgeFilters to cleanup the graph and focus on a certain period of time, or on nodes with a certain property.
Note that the graph is not modified, only the view is filtered.
The timeline shown here is very simple, but we provide a much more complete timeline plugin here: ogma-timeline

index.js

import Ogma from '@linkurious/ogma';
import { TimeFilter } from './timefilter';

const ogma = new Ogma({
  container: 'graph-container',
  options: {
    backgroundColor: null,
    detect: { nodeTexts: false, edgeTexts: false }
  }
});

// node types
const PORT = 'port';
const SERVER = 'server';
const IP = 'ip';

// edge types
const LISTENS_TO = 'listens_to';
const CONNECTS_TO = 'connects_to';

const ANIMATION_DURATION = 0;

const appState = {
  edgeGrouping: true,
  selectedPorts: {},
  timeFilter: [-Infinity, Infinity]
};

// transformations
const transformations = {
  edgeGrouping: null,
  portFilter: null,
  timeFilter: null
};

const initTransformations = () => {
  // port filtering policy
  transformations.portFilter = ogma.transformations.addNodeFilter({
    criteria: node => {
      if (node.getData('type') === PORT) {
        return appState.selectedPorts[node.getData('port')];
      } else {
        return true;
      }
    },
    enabled: true
  });

  // order matters, this has to go before the edge grouping
  transformations.timeFilter = ogma.transformations.addEdgeFilter({
    criteria: edge => {
      if (edge.getData('type') === CONNECTS_TO) {
        const date = edge.getData('date') || appState.timeFilter[0];
        return appState.timeFilter[0] <= date && date <= appState.timeFilter[1];
      }
      return true;
    }
  });

  // Edge grouping policy:
  //
  // group all edges between an “IP” and a “PORT” node by (protocol + service)
  // Grouped edge data:
  //   - protocol
  //   - service
  //   - total query bytes
  //   - total response bytes
  //   - group size (number of edges in the group)
  transformations.edgeGrouping = ogma.transformations.addEdgeGrouping({
    selector: edge =>
      !edge.isExcluded() && edge.getData('type') === CONNECTS_TO,
    groupIdFunction: e => e.getData('protocol') + '/' + e.getData('service'),
    generator: edges => {
      const e = edges.get(0);
      const protocol = e.getData('protocol');
      const service = e.getData('service');
      let query_bytes = 0,
        response_bytes = 0;
      edges.getData().forEach(edgeData => {
        query_bytes += edgeData.query_bytes || 0;
        response_bytes += edgeData.response_bytes || 0;
      });

      return {
        data: {
          type: CONNECTS_TO,
          query_bytes: query_bytes,
          response_bytes: response_bytes,
          group_size: edges.size,
          protocol: protocol,
          service: service
        }
      };
    },
    enabled: true
  });
};

const layout = () =>
  ogma.layouts.force({
    gpu: true,
    charge: 7,
    locate: true
  });

// format traffic values
const formatPutThrough = bytes => {
  bytes = +bytes;
  return bytes > 1024 * 1024
    ? Math.round(bytes / (1024 * 1024)) + 'M'
    : bytes > 1024
      ? Math.round(bytes / 1024) + 'KB'
      : bytes + 'B';
};

// Style the graph based on the properties of nodes and edges
const addStyles = () => {
  ogma.styles.addNodeRule(
    node => {
      return node.getData('type') === SERVER;
    },
    {
      radius: 10,
      color: '#fff',
      icon: {
        font: 'Font Awesome 5 Free',
        content: '\uF233',
        color: '#0094ff',
        style: 'bold',
        minVisibleSize: 0,
        scale: 0.7
      }
    }
  );

  // node styles
  ogma.styles.addNodeRule(
    n => {
      return n.getData('type') === IP;
    },
    {
      innerStroke: '#0094ff',
      color: '#fff',
      // number of outgoing “CONNECTS_TO” edges (taking filters into account)
      radius: node =>
        10 +
        Math.min(
          node.getAdjacentEdges({ direction: 'out ' }).filter(edge => {
            return edge.getData('type') === CONNECTS_TO;
          }).size,
          40
        ),
      icon: {
        font: 'Font Awesome 5 Free',
        content: '\uF233',
        style: 'bold',
        color: '#0094ff',
        minVisibleSize: 0,
        scale: 0.4
      },
      text: {
        content: node => node.getData('ip'),
        secondary: {
          content: (
            node // number of outgoing “CONNECTS_TO” edges (taking filters into account)
          ) =>
            node.getAdjacentEdges({ direction: 'out ' }).filter(edge => {
              return edge.getData('type') === CONNECTS_TO;
            }).size +
            ' connections\n' +
            // received bytes (sum of all “response_bytes” from outgoing
            // “CONNECTS_TO” edges, taking filters into account)
            formatPutThrough(
              node
                .getAdjacentEdges({ direction: 'out' })
                .filter(edge => {
                  return edge.getData('type') === CONNECTS_TO;
                })
                .reduce((total, edge) => {
                  return total + edge.getData('response_bytes');
                }, 0)
            ) +
            ' received\n' +
            // sent bytes (sum of all “query_bytes” from outgoing “CONNECTS_TO” edges,
            // taking filters into account)
            formatPutThrough(
              node
                .getAdjacentEdges({ direction: 'out' })
                .filter(edge => {
                  return edge.getData('type') === CONNECTS_TO;
                })
                .reduce((total, edge) => {
                  return total + edge.getData('query_bytes');
                }, 0)
            ) +
            ' sent\n',
          color: '#0094ff',
          backgroundColor: 'white',
          minVisibleSize: 60,
          margin: 1
        },
        color: 'white',
        backgroundColor: '#0094ff'
      }
    }
  );

  ogma.styles.addNodeRule(
    n => {
      return n.getData('type') === PORT;
    },
    {
      color: 'orange',
      shape: 'square',
      text: {
        content: node => ':' + node.getData('port'),
        position: 'center'
      },
      radius: 8,
      // Highlight Port 1900 as dangerous
      pulse: {
        enabled: node => node.getData('port') === '1900',
        endRatio: 5,
        width: 1,
        startColor: 'red',
        endColor: 'red',
        interval: 1000,
        startRatio: 1.0
      }
    }
  );

  ogma.styles.addEdgeRule({
    color: edge =>
      edge.getData('type') === CONNECTS_TO ? '#0094ff' : 'orange',
    width: edge => 1 + 1.3 * Math.sqrt(edge.getData('group_size')),
    shape: edge => (edge.getData('type') === CONNECTS_TO ? null : 'arrow'),
    text: {
      content: edge => {
        if (edge.getData('type') === CONNECTS_TO) {
          const response_bytes = edge.getData('response_bytes') || 0;
          const query_bytes = edge.getData('query_bytes') || 0;
          return (
            formatPutThrough(query_bytes) +
            ' in, ' +
            formatPutThrough(response_bytes) +
            ' out'
          );
        } else {
          return LISTENS_TO;
        }
      },
      color: edge =>
        edge.getData('type') === CONNECTS_TO ? '#0094ff' : 'orange'
    }
  });

  ogma.styles.setHoveredNodeAttributes({
    text: {
      backgroundColor: '0094ff'
    }
  });
  ogma.styles.setSelectedNodeAttributes({
    text: {
      style: 'bold',
      backgroundColor: '0094ff'
    }
  });

  ogma.styles.setHoveredEdgeAttributes({
    text: {
      backgroundColor: 'white'
    }
  });
  ogma.styles.setSelectedEdgeAttributes({
    text: {
      style: 'bold',
      backgroundColor: 'white'
    }
  });
};

const updateState = () => {
  // port filters
  const ports = document.querySelectorAll('#ports input');
  Array.prototype.forEach.call(ports, portInput => {
    const port = portInput.getAttribute('data-port-id');
    appState.selectedPorts[port] = portInput.checked;
  });

  // edge grouping
  appState.edgeGrouping = document.getElementById('group-edges').checked;
  // time filter
  if (appState.edgeGrouping !== transformations.edgeGrouping.isEnabled()) {
    transformations.edgeGrouping.toggle(ANIMATION_DURATION);
  }
  transformations.portFilter.refresh(ANIMATION_DURATION);
  return ogma.transformations.afterNextUpdate();
};

const initUI = graph => {
  createPortSelector();
  appState.timeFilter = ogma
    .getEdges()
    .filter(edge => {
      return edge.getData('type') === CONNECTS_TO;
    })
    .getData('date')
    .reduce(
      (acc, date) => {
        acc[0] = Math.min(acc[0], date);
        acc[1] = Math.max(acc[1], date);
        return acc;
      },
      [Infinity, -Infinity]
    );
  new TimeFilter(document.getElementById('timeline'), () => {
    if (transformations.timeFilter) {
      transformations.timeFilter.refresh();
    }
  }).update(
    ogma
      .getEdges('raw')
      .filter(edge => {
        return edge.getData('type') === CONNECTS_TO;
      })
      .getData()
      .sort((e1, e2) => {
        return e1.date - e2.date;
      }),
    appState
  );
  document.getElementById('filters').addEventListener('change', updateState);
};

const createPortSelector = graph => {
  const portValues = {};
  appState.selectedPorts = {};
  ogma
    .getNodes()
    .filter(node => {
      return node.getData('type') === PORT;
    })
    .forEach(node => {
      const ref = node.getData();
      const port = ref.port;
      portValues[port] = portValues[port] || [];
      portValues[port].push(node.getId());
      appState.selectedPorts[port] = true;
    });

  const container = document.getElementById('ports');
  appState.ports = portValues;
  container.innerHTML = Object.keys(portValues)
    .sort((a, b) => {
      return portValues[b].length - portValues[a].length;
    })
    .map(port => {
      return (
        '<input type="checkbox" data-port-id="' +
        port +
        '" ' +
        'checked id="port-' +
        port +
        '" name="port-' +
        port +
        '">' +
        '<label for="port-' +
        port +
        '" title="' +
        portValues[port].length +
        ' connections"> ' +
        port +
        '</label>'
      );
    })
    .join('');
};

// load data
const loadAndParseCSV = url =>
  new Promise((complete, error) => {
    Papa.parse(url, {
      download: true,
      complete: complete,
      error: error
    });
  });

loadAndParseCSV('./logs-merged_sample.csv')
  .then(records => recordsToGraph(records.data))
  .then(graph => ogma.setGraph(graph))
  .then(() => layout())
  .then(graph => initUI(graph))
  .then(initTransformations)
  .then(() => updateState())
  .then(() => addStyles())
  .then(initUI)

  .then(() => ogma.view.locateGraph({ padding: 20 }));

const recordsToGraph = records => {
  // deduplication lookup hashes
  const portsMap = {};
  const ipsMap = {};
  const serversMap = {};
  const listensMap = {};

  const nodes = [],
    edges = [];

  records.forEach(record => {
    const [
      timestamp,
      src_ip,
      dest_ip,
      dest_port,
      protocol,
      service,
      duration,
      out_bytes,
      in_bytes,
      status
    ] = record;

    // Port node
    const portId = dest_ip + ':' + dest_port;
    let port = portsMap[portId];
    if (!port) {
      port = portsMap[portId] = {
        id: portId,
        data: {
          ip: dest_ip,
          port: dest_port,
          type: PORT
        }
      };
      nodes.push(port);
    }

    // Server node
    const serverId = dest_ip;
    let server = serversMap[serverId];
    if (!server) {
      server = serversMap[serverId] = {
        id: serverId,
        data: {
          ip: dest_ip,
          type: SERVER
        }
      };
      nodes.push(server);
    }

    // "Listens to" edge
    let listen = listensMap[portId];
    if (!listen) {
      listen = listensMap[portId] = {
        id: portId,
        source: serverId,
        target: portId,
        data: {
          type: LISTENS_TO
        }
      };
      edges.push(listen);
    }

    // IP node
    const ipId = src_ip;
    let ip = ipsMap[ipId];
    if (!ip) {
      ip = ipsMap[ipId] = {
        id: ipId,
        data: {
          ip: src_ip,
          type: IP
        }
      };
      nodes.push(ip);
    }

    // Connection edge
    const connectionId = `${src_ip}-${dest_ip}:${dest_port}@${Math.round(
      timestamp * 1000
    )}`;
    const connection = {
      id: connectionId,
      source: ipId,
      target: portId,
      data: {
        type: CONNECTS_TO,
        date: Math.round(timestamp * 1000),
        query_bytes: isNaN(out_bytes) ? 0 : parseInt(out_bytes),
        response_bytes: isNaN(in_bytes) ? 0 : parseInt(in_bytes),
        protocol: protocol,
        service: service
      }
    };
    edges.push(connection);
  });

  return { nodes: nodes, edges: edges };
};

index.html

<!DOCTYPE html>
<html>
  <head>
    <title>Ogma cyber security analysis</title>

    <link
      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css"
      rel="stylesheet"
    />
    <link type="text/css" rel="stylesheet" href="style.css">
    <script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.3.0/papaparse.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/crossfilter2/1.4.6/crossfilter.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>
  </head>

  <body>
    <i class="fa fa-camera-retro fa-1x" style="color: rgba(0, 0, 0, 0)"></i>
    <div id="graph-container"></div>
    <div id="ui">
      <form id="filters">
        <h3>Ports</h3>
        <div id="ports"></div>
        <h3>Connections</h3>
        <input type="checkbox" id="group-edges" checked name="group-edges" />
        <label for="group-edges">Group</label>
      </form>
      <div id="timeline"></div>
    </div>
    <script src="index.js"></script>
  </body>
</html>

style.css

body {
  margin: 0;
  padding: 0;
  width: 100%;
  height: 100%;
  font:
    14px/20px 'Open Sans',
    Arial,
    sans-serif;
}

*,
*:before,
*:after {
  box-sizing: border-box;
}

[type='checkbox']:checked,
[type='checkbox']:not(:checked) {
  position: absolute;
  left: -9999px;
}
[type='checkbox']:checked + label,
[type='checkbox']:not(:checked) + label {
  position: relative;
  padding-left: 28px;
  cursor: pointer;
  line-height: 20px;
  display: inline-block;
}

[type='checkbox']:checked + label:before,
[type='checkbox']:not(:checked) + label:before {
  content: '';
  position: absolute;
  left: 0;
  top: 0;
  width: 18px;
  height: 18px;
  border: 1px solid #ddd;
  background: #ddd;
}
[type='checkbox']:checked + label:after,
[type='checkbox']:not(:checked) + label:after {
  content: '';
  width: 8px;
  height: 8px;
  background: black;
  position: absolute;
  top: 5px;
  left: 5px;
  -webkit-transition: all 0.2s ease;
  transition: all 0.2s ease;
}
[type='checkbox']:not(:checked) + label:after {
  opacity: 0;
  -webkit-transform: scale(0);
  transform: scale(0);
}
[type='checkbox']:checked + label:after {
  opacity: 1;
  -webkit-transform: scale(1);
  transform: scale(1);
}

#graph-container {
  top: 0;
  bottom: 0;
  left: 0;
  right: 0;
  position: absolute;
  margin: 0;
  overflow: hidden;
}

#filters {
  position: absolute;
  background: white;
  top: 10px;
  left: 10px;
  border-radius: 5px;
  font-family: Georgia, 'Times New Roman', Times, serif;
  padding: 10px 20px;
  box-shadow: 0 0 5px rgba(0, 0, 0, 0.3);
}

#ui h3 {
  font-size: 1em;
}

#ports {
  width: 200px;
  display: block;
  padding-bottom: 10px;
}

#ports label {
  display: inline-block;
  width: 40%;
}

#timeline {
  position: absolute;
  bottom: 0;
  width: 100%;
}

#time-filter {
  position: absolute;
  bottom: 10px;
  right: 10px;
  width: 100%;
}

#time-filter svg {
  float: right;
  right: 0;
}

#time-filter .bar {
  fill: rgba(70, 130, 180, 0.75);
}

#time-filter .selected .bar {
  fill: rgba(255, 0, 0, 0.75);
}

.handle.handle--e,
.handle.handle--w {
  fill: #ffffff;
  stroke: #000000;
  stroke-width: 0.3;
}

timefilter.js

// d3-based simple timline UI
const MS_IN_HOUR = 1000 * 1000 * 60 * 60;

const getSize = container => {
  const width = container.offsetWidth - 20;
  const height = container.offsetHeight;
  return { width: width, height: height };
};

export class TimeFilter {
  constructor(container, update) {
    const id = 'time-filter';
    this._container =
      document.getElementById(id) || document.createElement('div');
    this._container.id = id;

    this._update = update;

    window.addEventListener('resize', () => {
      return this._updateSize();
    });
    container.appendChild(this._container);
  }

  _updateSize() {
    const ref = getSize(this._container);
    const width = ref.width;
    const height = ref.height;
    d3.select('svg').attr('width', width).attr('height', height);
  }

  init(appState) {
    this._container.innerHTML = '<svg></svg>';
    const ref = getSize(this._container);
    const width = ref.width;
    const height = ref.height;
    const buckets = this._buckets;

    const svg = d3.select('svg');
    const margin = { top: 0, right: 0, bottom: 0, left: 0 };
    const g = svg
      .append('g')
      .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
    svg.attr('width', width).attr('height', height + 30);

    const min = buckets[0].start;
    const max = buckets[buckets.length - 1].end;

    const minValue = 0;
    const maxValue = buckets.reduce((a, b) => {
      return Math.max(a, b.value);
    }, 0);

    const x = d3.scaleLinear().domain([min, max]).range([0, width]);

    const y = d3.scaleLinear().domain([0, maxValue]).range([height, 0]);

    const brushed = extent => {
      extent = extent || d3.event.selection.map(x.invert, x);
      const ids = [];
      let start = Infinity,
        end = -Infinity;
      bar.classed('selected', d => {
        const inside = extent[0] <= d.start && d.end <= extent[1];
        if (inside) {
          start = Math.min(start, d.start);
          end = Math.max(end, d.end);
        }
        return inside;
      });
      appState.timeFilter = [start, end];
      this._update();
    };

    const brush = d3
      .brushX()
      .extent([
        [0, 0],
        [width, height]
      ])
      .on('start brush', brushed);

    const barWidth = width / buckets.length;

    const bar = g
      .selectAll('g')
      .data(buckets)
      .enter()
      .append('g')
      .attr('transform', (d, i) => {
        return 'translate(' + i * barWidth + ',0)';
      });

    bar
      .append('rect')
      .attr('y', d => {
        return y(d.value);
      })
      .attr('height', d => {
        return height - y(d.value);
      })
      .attr('width', barWidth)
      .attr('class', 'bar');

    const beforebrushed = () => {
      d3.event.stopImmediatePropagation();
      d3.select(this.parentNode).transition().call(brush.move, x.range());
    };

    g.append('g')
      .call(brush)
      .call(brush.move, [min + MS_IN_HOUR, max - MS_IN_HOUR].map(x))
      .selectAll('.overlay')
      .on('mousedown touchstart', beforebrushed, true);

    g.append('g')
      .attr('transform', 'translate(0,' + height + ')')
      .call(
        d3.axisBottom(x).tickFormat(d => {
          const date = new Date(d);
          return date.getHours() + ':' + date.getMinutes();
        })
      );

    brushed([buckets[0].end, buckets[buckets.length - 1].start]);
  }

  update(edges, appState) {
    let start = edges[0].date;
    let next = start + MS_IN_HOUR;
    let value = 0;

    const buckets = [
      {
        start: edges[0].date - MS_IN_HOUR,
        end: edges[0].date,
        next: next,
        value: value
      }
    ];
    edges.forEach((e, i) => {
      while (e.date > next) {
        // store
        buckets.push({ start: start, end: next, value: value });
        // reset
        start = next;
        next = start + MS_IN_HOUR;
        value = 0;
      }
      value += e.query_bytes || 0;
    });
    if (value !== 0) {
      buckets.push({ start: start, end: next, value: value });
    }

    buckets.push({ start: next, end: next + MS_IN_HOUR, next: 0, value: 0 });

    this._buckets = buckets;
    this.init(appState);
  }
}