Data Lineage

analysisstyles

This example shows how to untangle huge data processing infrastructure with a lean data lineage visualization. Select a Report to track which dataset has been produced from. The dataset is inspired by the Data Lineage blog post of Rik Van Bruggen, with some tweak of the data model.

index.ts

import Ogma, { NodeId, Node, NodeList, Point } from '@linkurious/ogma';

type NodeType =
  | 'Log'
  | 'Process'
  | 'User'
  | 'System'
  | 'Dataset'
  | 'Database'
  | 'Report';
interface NodeData {
  type: NodeType;
  name: string;
}
const ogma = new Ogma<NodeData>({
  container: 'graph-container'
});

// Menu UI
const reportInput = document.querySelector<HTMLInputElement>('#report-target')!;
const reportsList = document.querySelector<HTMLInputElement>('#reports-list')!;
const resetButton = document.querySelector<HTMLInputElement>('#reset')!;

const ICONS: Record<NodeType, string> = {
  Log: '\uf036',
  Process: '\uf7d9',
  User: '\uf007',
  System: '\uf233',
  Dataset: '\uf0ce',
  Database: '\uf1c0',
  Report: '\uf1ea'
};

// Cache here coords after the first layout
const coords: Record<NodeId, Point> = {};

const saveCoords = () => {
  ogma.getNodes().forEach(node => {
    coords[node.getId()] = node.getPosition();
  });
};

const restoreCoords = () =>
  ogma.getNodes().setAttributes(
    {
      x: node => coords[node.getId()].x,
      y: node => coords[node.getId()].y
    },
    {
      duration: 500
    }
  );

const fillReports = () => {
  const reports = getNodesByType('Report').toJSON({
    attributes: ['text']
  });
  reports.forEach(report => {
    const option = document.createElement('option');
    option.value = report.data!.name;
    reportsList.appendChild(option);
  });
  // Force a DOM refresh
  reportsList.focus();
};

// Load data from a json file.
const graph = await Ogma.parse.jsonFromUrl<NodeData>('files/datalineage.json');
await ogma.setGraph(graph);
await ogma.view.locateGraph();
await saveCoords();
await fillReports();

// Define the Node style rules
ogma.styles.addNodeRule({
  text: node => node.getData('name'),

  // we use Ogma rules here to reflects the same colors in the legend tool
  color: ogma.rules.map({
    field: 'type',
    values: {
      Log: '#1b9e77',
      Process: '#d95f02',
      User: '#7570b3',
      System: '#e7298a',
      Dataset: '#66a61e',
      Database: '#e6ab02',
      Report: '#a6761d'
    }
  }),

  radius: node => 10 + Math.log(node.getDegree()),

  // assign icons based on the node category
  icon: {
    content: node => ICONS[node.getData('type')],
    font: 'Font Awesome 5 Free',
    style: 'bold',
    color: 'white'
  }
});

// Define the Edge style rules
ogma.styles.addEdgeRule({
  shape: 'arrow'
});

function getNodesByType(type: NodeType) {
  return ogma.getNodes().filter(node => node.getData('type') === type);
}

const enrichPathWith = (nodeList: NodeList<NodeData>, type: NodeType) => {
  const attachedType = nodeList
    .getAdjacentNodes()
    .filter(node => node.getData('type') === type);
  // now get the links between these users and the nodes in the list
  const adjacentEdges = attachedType
    .getAdjacentEdges()
    .filter(
      edge =>
        nodeList.includes(edge.getSource()) ||
        nodeList.includes(edge.getTarget())
    );
  return {
    nodes: attachedType,
    edges: adjacentEdges
  };
};

const getChainFromTo = (report: Node<NodeData>) => {
  // Here's the main idea:
  // * For a given report all shortest paths between it and the various datasets are computed
  // * Once collected all paths, merge them together
  // * Enrich the data lineage with the User and Database nodes
  // Note: this particular algorithm depends on the particular data modelling of the data lineage dataset
  const datasets = getNodesByType('Dataset').toList();

  return Promise.all(
    datasets.map(d => {
      return ogma.algorithms.shortestPath({
        source: d.getId(),
        target: report.getId(),
        directed: true
      });
    })
  ).then(paths => {
    const allPaths = {
      nodes: ogma.createNodeList(),
      edges: ogma.createEdgeList()
    };
    paths.forEach(p => {
      if (p != null) {
        const usersData = enrichPathWith(p.nodes, 'User');
        const databaseData = enrichPathWith(p.nodes, 'Database');
        allPaths.nodes = allPaths.nodes
          .concat(p.nodes)
          .concat(usersData.nodes)
          .concat(databaseData.nodes);
        allPaths.edges = allPaths.edges
          .concat(p.edges)
          .concat(usersData.edges)
          .concat(databaseData.edges);
      }
    });

    return allPaths;
  });
};

const showDataLineage = async (node: Node<NodeData>) => {
  const paths = await getChainFromTo(node);
  const t1 = ogma.transformations.addNodeFilter({
    criteria: (
      node // check if node is in chain
    ) => paths.nodes.includes(node),
    duration: 500
  });
  const t2 = ogma.transformations.addEdgeFilter({
    criteria: edge => paths.edges.includes(edge),
    duration: 500
  });
  // Wait both transformations to run, then apply the layout
  await Promise.all([t1.whenApplied(), t2.whenApplied()]);

  await ogma.layouts.hierarchical({
    roots: [node.getId()], // from the report backwards
    direction: 'RL', // Rotate the layout right (report) to left (datasets)
    locate: {
      padding: 100 // give some space to avoid the Report node go under the floating menu
    },
    duration: 600,
    levelDistance: 100 // Make some space between nodes as horizontal
  });
};

const resetFilter = async () => {
  // Destroy the filters, then restore all the original positions
  await Promise.all(ogma.transformations.getList().map(t => t.destroy(600)));
  await restoreCoords();
  await ogma.view.locateGraph({ duration: 800 });
};

// Legend
await ogma.tools.legend.enable({
  position: 'bottom',
  titleTextAlign: 'center',
  shapeColor: 'black',
  circleStrokeWidth: 1,
  fontSize: 14,
  borderColor: '#ddd',
  fontFamily: '"Inter", Helvetica, sans-serif',
  titleFunction: () => 'Node Entities'
});

reportInput.addEventListener('change', evt => {
  if (evt.target) {
    const target = ogma
      .getNodes()
      .filter(
        node => node.getData('name') === (evt.target as HTMLInputElement).value
      )
      .get(0);
    if (target) showDataLineage(target);
  }
});

resetButton.addEventListener('click', () => {
  if (ogma.transformations.getList().length) {
    resetFilter();
    reportInput.value = '';
  }
});

index.html

<!doctype html>
<html>
  <head>
    <meta charset="utf-8" />
    <link
      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.11.2/css/solid.min.css"
      rel="stylesheet"
    />

    <link type="text/css" rel="stylesheet" href="styles.css" />
  </head>

  <body>
    <div id="graph-container"></div>
    <div class="panel" id="ui">
      <div class="section layout">
        <h2>Data Lineage Options</h2>
        <p>
          <label for="report-target">Report to track:</label>
          <input
            list="reports-list"
            name="report-target"
            id="report-target"
            class="clearable-input"
            placeholder="Select"
          />
        </p>
      </div>
      <div class="controls">
        <button id="reset">Reset</button>
      </div>
    </div>
    <datalist id="reports-list"> </datalist>
    <script type="module" src="index.ts"></script>
  </body>
</html>

styles.css

.toolbar {
  display: block;
  position: absolute;
  top: 20px;
  right: 20px;
  padding: 10px;
  box-shadow: 0 1px 5px rgba(0, 0, 0, 0.65);
  border-radius: 4px;
  background: #ffffff;
  color: #222222;
  font-weight: 300;
  z-index: 9999;
}

.toolbar .section {
  position: relative;
  display: block;
}

.toolbar .section h3 {
  display: block;
  font-weight: 300;
  border-bottom: 1px solid #ddd;
  color: #606060;
  font-size: 1rem;
}

.toolbar .section .clearable-input {
  border-radius: 4px;
  padding: 5px;
  border: 1px solid #dddddd;
}

.toolbar .controls {
  text-align: center;
  margin-top: 10px;
}

html,
body {
  font-family: 'Inter', sans-serif;
}

:root {
  --base-color: #4999f7;
  --active-color: var(--base-color);
  --gray: #d9d9d9;
  --white: #ffffff;
  --lighter-gray: #f4f4f4;
  --light-gray: #e6e6e6;
  --inactive-color: #cee5ff;
  --group-color: #525fe1;
  --group-inactive-color: #c2c8ff;
  --selection-color: #04ddcb;
  --darker-gray: #b6b6b6;
  --dark-gray: #555;
  --dark-color: #3a3535;
  --edge-color: var(--dark-color);
  --border-radius: 5px;
  --button-border-radius: var(--border-radius);
  --edge-inactive-color: var(--light-gray);
  --button-background-color: #ffffff;
  --shadow-color: rgba(0, 0, 0, 0.25);
  --shadow-hover-color: rgba(0, 0, 0, 0.5);
  --button-shadow: 0 0 4px var(--shadow-color);
  --button-shadow-hover: 0 0 4px var(--shadow-hover-color);
  --button-icon-color: #000000;
  --button-icon-hover-color: var(--active-color);
}

#graph-container {
  top: 0;
  bottom: 0;
  left: 0;
  right: 0;
  position: absolute;
  margin: 0;
  overflow: hidden;
}

.ui {
  position: absolute;
  display: flex;
  flex-direction: column;
  gap: 0.5em;
}

#custom-group-btn {
  top: 40px;
}

.panel {
  background: var(--button-background-color);
  border-radius: var(--button-border-radius);
  box-shadow: var(--button-shadow);
  padding: 10px;
}

.panel {
  position: absolute;
  top: 20px;
  left: 20px;
}

.panel h2 {
  text-transform: uppercase;
  font-weight: 400;
  font-size: 14px;
  margin: 0;
}

.panel {
  margin-top: 1px;
  padding: 5px 10px;
  text-align: center;
}

.panel button {
  background: var(--button-background-color);
  border: none;
  border-radius: var(--button-border-radius);
  border-color: var(--shadow-color);
  padding: 5px 10px;
  cursor: pointer;
  width: 100%;
  color: var(--dark-gray);
  border: 1px solid var(--light-gray);
}

.panel button:hover {
  background: var(--lighter-gray);
  border: 1px solid var(--darker-gray);
}

.panel button[disabled] {
  color: var(--light-gray);
  border: 1px solid var(--light-gray);
  background-color: var(--lighter-gray);
}