Web: Updates for Table and Column Lineage (#2725)

* Initial commit with new rendering engine.

* Adding prototype for column lineage view.

* Updates for zoom controls.

* Removing extra imports.

* Extra meta information.

* Adjusting application styles.

* Improving code quality, splitting components.

* Moving code around.

* Saving partial progress.

* Some progress on lineage view.

* Checkpoint for tooltips.

* Adding in side navigation.

* Fixing links.

* Fixing up lineage state.

* Fixing column lineage.

* Fixing up the back button.

* Fixing tests.

* Filtering out the non-null assertions.

* More test fixes.

* Code review comments and running formatter.

* Adding full mode to graph.

* Changing some language.

* Minor update to fix search issue.

* Removing Some text.

* Improving fonts and layout in the action bar.

* Table level node encoding.

* Column Level changes.

* Fixing reload on column lineage and adjusting colors.

---------

Co-authored-by: phix <[email protected]>

web/.gitignore

@@ -12,6 +12,7 @@
 /coverage
 /dist
 /node_modules
+/libs/graph/node_modules
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*

web/libs/graph/README.md

@@ -0,0 +1,111 @@
+# graph
+
+`graph` is a library for generating and rendering graphs. It uses elkjs to generate layouts and renders them using a custom renderer.
+
+## Installing/Using
+
+To use the library, add the following to your package.json:
+
+```json
+{
+ "dependencies": {
+ "elkjs": "^0.8.2"
+ }
+}
+```
+
+You also need to use the webpack CopyPlugin to copy the elk-worker file. Add the following to your webpack config:
+
+```js
+const CopyPlugin = require('copy-webpack-plugin');
+const path = require('path');
+
+// look for elkjs package folder
+const elkjsRoot = path.dirname(require.resolve('elkjs/package.json'));
+
+// add the CopyPlugin to the webpack config
+ plugins: [
+ ...
+ new CopyPlugin({
+ patterns: [
+ { from: path.join(elkjsRoot, 'lib/elk-worker.min.js'), to: 'elk-worker.min.js' },
+ ],
+ }),
+ ]
+```
+
+## useLayout
+
+`useLayout` provides a common interface for creating layouts with [ElkJs](https://github.com/kieler/elkjs).
+
+```ts
+import { useLayout } from 'graph'
+
+const { nodes: positionedNodes, edges: positionedEdges } = useLayout<'myKind', MyNodeDataType>({
+ id,
+ nodes,
+ edges,
+ direction: 'right',
+ webWorkerUrl,
+ getLayoutOptions
+});
+```
+
+The layout calculations are asynchronous. Once the layout is complete, the returned `nodes` will each include
+a `bottomLeftCorner: {x: number: y: number }` property, along with all the original properties.
+
+## ZoomPanSvg Component
+
+// To Add
+
+## Graph Component
+
+The Graph component is used to render a Graph. A `TaskNode` with run status is included, but custom ones are supported.
+
+```tsx
+import { Graph, Edge, Node } from 'graph';
+
+import { MyNodeComponent, MyNodeDataType, NodeRendererMap } from './';
+
+const myNodesRenderers: NodeRendererMap<'myNode', MyNodeDataType> = new Map().set(
+ 'myNode',
+ MyNodeComponent,
+);
+
+// declare the nodes and edges
+const nodes: Node<'myNode', MyNodeDataType>[] = [
+ {
+ id: '1',
+ label: 'Task 1',
+ type: 'myNode',
+ data: {
+ // any additional data you want to store
+ },
+ },
+ {
+ id: '2',
+ label: 'Task 2',
+ type: 'myNode',
+ data: {
+ // any additional data you want to store
+ },
+ },
+];
+
+const edges: Edge[] = [
+ {
+ id: '1',
+ source: '1',
+ target: '2',
+ type: 'elbow',
+ },
+];
+
+// create a graph
+<Graph<'myNode', MyNodeDataType>
+ nodes={nodes}
+ edges={edges}
+ direction="right"
+ nodeRenderers={myNodesRenderers}
+/>;
+```

web/libs/graph/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+ ...config,
+ setupFilesAfterEnv: ['./jest.setup.js'],
+}

web/libs/graph/jest.setup.js

@@ -0,0 +1,19 @@
+/* eslint-disable no-undef */
+import '@testing-library/jest-dom'
+
+// Establish API mocking before all tests.
+beforeAll(() => {
+ Object.defineProperty(window, 'matchMedia', {
+ writable: true,
+ value: jest.fn().mockImplementation((query) => ({
+ matches: false,
+ media: query,
+ onchange: null,
+ addListener: jest.fn(), // deprecated
+ removeListener: jest.fn(), // deprecated
+ addEventListener: jest.fn(),
+ removeEventListener: jest.fn(),
+ dispatchEvent: jest.fn(),
+ })),
+ })
+})

web/libs/graph/package.json

@@ -0,0 +1,46 @@
+{
+ "name": "graph",
+ "version": "0.0.1",
+ "private": true,
+ "main": "src/index.ts",
+ "scripts": {
+ "lint": "cd .. && yarn lint",
+ "test": "jest",
+ "test:silent": "yarn test --silent",
+ "test:ci": "yarn test:silent --coverage --runInBand",
+ "test:watch": "yarn test --watch",
+ "tsc:validate": "tsc --noEmit"
+ },
+ "dependencies": {
+ "@react-hook/size": "^2.1.2",
+ "lodash": "^4.17.21"
+ },
+ "peerDependencies": {
+ "@chakra-ui/react": "^2",
+ "elkjs": "^0.8.2",
+ "react": "^18",
+ "react-dom": "^18",
+ "react-router-dom": "^6"
+ },
+ "devDependencies": {
+ "@testing-library/jest-dom": "^5.16.5",
+ "@testing-library/react": "^14.0.0",
+ "@testing-library/react-hooks": "^8.0.1",
+ "@testing-library/user-event": "^14.4.3",
+ "@types/jest": "^27.5.2",
+ "@types/react": "^18.0.15",
+ "@types/react-dom": "^18.0.6",
+ "d3-interpolate": "^2.0.1",
+ "d3-selection": "^2.0.0",
+ "d3-transition": "^2.0.0",
+ "d3-zoom": "^2.0.0",
+ "elkjs": "^0.8.2",
+ "jest": "^27.5.1",
+ "prettier": "^2.8.8",
+ "react": "^18.2.0",
+ "react-dom": "^18.2.0",
+ "react-router-dom": "^6.3.0",
+ "typescript": "^5.1.3",
+ "web-worker": "^1.2.0"
+ }
+}

web/libs/graph/src/components/Edge/Edge.tsx

@@ -0,0 +1,20 @@
+import React from 'react'
+
+import { ElbowEdge } from './ElbowEdge'
+import { StraightEdge } from './StraightEdge'
+import type { PositionedEdge } from '../../types'
+
+export interface EdgeProps {
+ edge: PositionedEdge
+ isMiniMap?: boolean
+}
+
+export const Edge = (props: EdgeProps) => {
+ const { edge } = props
+ switch (edge.type) {
+ case 'straight':
+ return <StraightEdge {...props} />
+ default:
+ return <ElbowEdge {...props} />
+ }
+}

web/libs/graph/src/components/Edge/EdgeLabel.tsx

@@ -0,0 +1,26 @@
+import React from 'react'
+
+import { grey } from '@mui/material/colors'
+import type { ElkLabel } from 'elkjs'
+
+interface Props {
+ label?: ElkLabel
+ endPointY?: number
+}
+
+export const EdgeLabel = ({ label, endPointY }: Props) => {
+ const labelColor = grey['400']
+
+ if (!label || !label.y || !label.x) return null
+
+ let { y } = label
+ // The edge and label are rendering a little differently,
+ // so we need some extra magic numbers to work right
+ if (endPointY) y = label.y - 5 >= endPointY ? endPointY + 25 : endPointY - 15
+
+ return (
+ <text fill={labelColor} x={label.x} y={y}>
+ {label.text}
+ </text>
+ )
+}

web/libs/graph/src/components/Edge/ElbowEdge.tsx

@@ -0,0 +1,63 @@
+import React, { useMemo } from 'react'
+
+import { chakra, keyframes, usePrefersReducedMotion } from '@chakra-ui/react'
+
+import { EdgeLabel } from './EdgeLabel'
+import { grey } from '@mui/material/colors'
+import type { EdgeProps } from './Edge'
+
+const ChakraPolyline = chakra('polyline') // need to use animation prop
+const marchingAnts = keyframes({ from: { strokeDashoffset: 60 }, to: { strokeDashoffset: 0 } })
+
+export const ElbowEdge = ({ edge, isMiniMap }: EdgeProps) => {
+ const reducedMotion = usePrefersReducedMotion() || isMiniMap // do not animate the minimap
+
+ const points = useMemo(() => {
+ const { startPoint, bendPoints, endPoint } = edge
+
+ return [
+ // source
+ { x: startPoint.x, y: startPoint.y },
+ ...(bendPoints?.map((bendPoint) => ({ x: bendPoint.x, y: bendPoint.y })) ?? []),
+ // target
+ { x: endPoint.x, y: endPoint.y },
+ ]
+ }, [edge])
+
+ // Find the longest edge that the label would be near
+ let longestEdge: { y: number; length: number } | undefined
+ if (edge.label) {
+ points.forEach((p, i) => {
+ if (i > 0) {
+ const length = p.x - points[i - 1].x
+ if (!longestEdge || longestEdge.length < length) longestEdge = { y: p.y, length }
+ }
+ })
+ }
+ return (
+ <>
+ <polyline
+ id={`${edge.sourceNodeId}-${edge.targetNodeId}`}
+ fill='none'
+ stroke={edge.color || grey['600']}
+ strokeWidth={edge.strokeWidth || 2}
+ strokeLinejoin='round'
+ points={points.map(({ x, y }) => `${x},${y}`).join(' ')}
+ />
+ <EdgeLabel label={edge.label} endPointY={longestEdge?.y} />
+ {!reducedMotion && edge.isAnimated && (
+ <ChakraPolyline
+ id={`${edge.sourceNodeId}-${edge.targetNodeId}-animated`}
+ fill='none'
+ strokeLinecap='round'
+ stroke={edge.color || grey['600']}
+ strokeWidth={edge.strokeWidth || 5}
+ strokeLinejoin='round'
+ strokeDasharray='0px 60px'
+ animation={`${marchingAnts} infinite 2s linear`}
+ points={points.map(({ x, y }) => `${x},${y}`).join(' ')}
+ />
+ )}
+ </>
+ )
+}

web/libs/graph/src/components/Edge/StraightEdge.tsx

@@ -0,0 +1,48 @@
+import React from 'react'
+
+import { chakra, keyframes, usePrefersReducedMotion } from '@chakra-ui/react'
+
+import { EdgeLabel } from './EdgeLabel'
+import { grey } from '@mui/material/colors'
+import type { EdgeProps } from './Edge'
+
+const ChakraLine = chakra('line') // need to use animation prop
+const marchingAnts = keyframes({ from: { strokeDashoffset: 60 }, to: { strokeDashoffset: 0 } })
+
+export const StraightEdge = ({ edge, isMiniMap }: EdgeProps) => {
+ const reducedMotion = usePrefersReducedMotion() || isMiniMap // do not animate the minimap
+ const color = grey['600']
+
+ return (
+ <>
+ <line
+ id={`${edge.sourceNodeId}-${edge.targetNodeId}`}
+ fill='none'
+ stroke={edge.color || color}
+ strokeWidth={edge.strokeWidth || 2}
+ strokeLinejoin='round'
+ x1={edge.startPoint.x}
+ y1={edge.startPoint.y}
+ x2={edge.endPoint.x}
+ y2={edge.endPoint.y}
+ />
+ <EdgeLabel label={edge.label} endPointY={edge.endPoint.y} />
+ {!reducedMotion && edge.isAnimated && (
+ <ChakraLine
+ id={`${edge.sourceNodeId}-${edge.targetNodeId}-animated`}
+ fill='none'
+ strokeLinecap='round'
+ stroke={edge.color || color}
+ strokeWidth={edge.strokeWidth || 5}
+ strokeLinejoin='round'
+ strokeDasharray='0px 60px'
+ animation={`${marchingAnts} infinite 2s linear`}
+ x1={edge.startPoint.x}
+ y1={edge.startPoint.y}
+ x2={edge.endPoint.x}
+ y2={edge.endPoint.y}
+ />
+ )}
+ </>
+ )
+}

web/libs/graph/src/components/Edge/index.ts

@@ -0,0 +1 @@
+export * from './Edge'

web/libs/graph/src/components/Graph.test.tsx

@@ -0,0 +1,32 @@
+import React from 'react'
+
+import '@testing-library/jest-dom'
+
+import { PositionedNode } from '../index'
+
+/* CUSTOM NODES */
+export interface SimpleNodeData {
+ displayName?: string
+}
+
+interface SimpleNodeProps {
+ node: PositionedNode<'simple', SimpleNodeData>
+}
+
+// a task node has a name and an operator that get displayed
+const SimpleNode = ({ node }: SimpleNodeProps) => (
+ <rect
+ x={10}
+ y={20}
+ width={node.width}
+ height={node.height}
+ data-testid={node.id}
+ data-custom='true'
+ />
+)
+
+SimpleNode.getLayoutOptions = (node: SimpleNodeProps['node']) => ({
+ ...node,
+ width: node.width ?? 100,
+ height: node.height ?? 100,
+})