From 2e6c7606c80b7fddadc9db4c28a02dfc3a018eea Mon Sep 17 00:00:00 2001
From: EnixCoda <enixcoda@gmail.com>
Date: Wed, 21 Oct 2020 21:08:13 +0800
Subject: [PATCH] feat: async mode

---
 src/components/FileExplorer.tsx    |  11 +-
 src/components/Icon.tsx            |   6 +
 src/components/Node.tsx            |   8 +-
 src/components/SideBar.tsx         |   3 +-
 src/driver/core/FileExplorer.ts    |  68 ++----
 src/driver/core/SideBar.ts         |   9 +-
 src/global.d.ts                    |   5 +-
 src/platforms/GitHub/API.ts        |   6 +-
 src/platforms/GitHub/index.ts      |  47 +++-
 src/platforms/Gitee/index.ts       |   6 +-
 src/platforms/platform.d.ts        |   7 +-
 src/utils/EventHub.ts              |  54 +++++
 src/utils/VisibleNodesGenerator.ts | 365 ++++++++++++++++-------------
 src/utils/general.ts               |  56 +++--
 src/utils/gitSubmodule.ts          |  15 +-
 15 files changed, 417 insertions(+), 249 deletions(-)
 create mode 100644 src/utils/EventHub.ts

diff --git a/src/components/FileExplorer.tsx b/src/components/FileExplorer.tsx
index 565f86e..adaf560 100644
--- a/src/components/FileExplorer.tsx
+++ b/src/components/FileExplorer.tsx
@@ -30,14 +30,12 @@ const RawFileExplorer: React.FC<Props & ConnectorState> = function RawFileExplor
     setUpTree,
     treeRoot,
   } = props
-  const {
-    val: { compressSingletonFolder },
-  } = useConfigs()
+  const { val: config } = useConfigs()
 
   React.useEffect(() => {
     const { setUpTree, treeRoot, metaData } = props
-    setUpTree({ treeRoot, metaData, compressSingletonFolder })
-  }, [setUpTree, treeRoot, compressSingletonFolder])
+    setUpTree({ treeRoot, metaData, config })
+  }, [setUpTree, treeRoot, config.compressSingletonFolder, config.access_token])
 
   React.useEffect(() => {
     const { execAfterRender } = props
@@ -141,7 +139,7 @@ const VirtualNode = React.memo(function VirtualNode({
     searchKey && isValidRegexpSource(searchKey) ? new RegExp(searchKey, 'gi') : undefined
   if (!visibleNodes) return null
 
-  const { nodes, depths, focusedNode, expandedNodes } = visibleNodes
+  const { nodes, focusedNode, expandedNodes, loading, depths } = visibleNodes as VisibleNodes
   const node = nodes[index]
   return (
     <Node
@@ -150,6 +148,7 @@ const VirtualNode = React.memo(function VirtualNode({
       node={node}
       depth={depths.get(node) || 0}
       focused={focusedNode === node}
+      loading={loading.has(node.path)}
       expanded={expandedNodes.has(node.path)}
       onClick={onNodeClick}
       renderActions={renderActions}
diff --git a/src/components/Icon.tsx b/src/components/Icon.tsx
index e0943df..b628315 100644
--- a/src/components/Icon.tsx
+++ b/src/components/Icon.tsx
@@ -1,6 +1,7 @@
 import {
   ChevronDownIcon as ChevronDown,
   ChevronRightIcon as ChevronRight,
+  ClockIcon as Clock,
   FileCodeIcon as FileCode,
   FileIcon as File,
   FileMediaIcon as FileMedia,
@@ -26,6 +27,11 @@ function getSVGIconComponent(
   name: string
 } {
   switch (type) {
+    case 'loading':
+      return {
+        IconComponent: Clock,
+        name: 'Clock',
+      }
     case 'hourglass':
       return {
         IconComponent: Hourglass,
diff --git a/src/components/Node.tsx b/src/components/Node.tsx
index b2a2355..7f4624e 100644
--- a/src/components/Node.tsx
+++ b/src/components/Node.tsx
@@ -23,6 +23,7 @@ type Props = {
   depth: number
   expanded: boolean
   focused: boolean
+  loading: boolean
   renderActions?(node: TreeNode): React.ReactNode
   style?: React.CSSProperties
   regex?: RegExp
@@ -32,6 +33,7 @@ export function Node({
   depth,
   expanded,
   focused,
+  loading,
   renderActions,
   style,
   onClick,
@@ -57,7 +59,7 @@ export function Node({
       title={path}
     >
       <div className={'node-item-label'}>
-        <NodeItemIcon node={node} open={expanded} />
+        <NodeItemIcon node={node} open={expanded} loading={loading} />
         {name.includes('/') ? (
           name.split('/').map((chunk, index) => (
             <React.Fragment key={chunk}>
@@ -77,9 +79,11 @@ export function Node({
 const NodeItemIcon = React.memo(function NodeItemIcon({
   node,
   open = false,
+  loading,
 }: {
   node: TreeNode
   open?: boolean
+  loading?: boolean
 }) {
   const {
     val: { icons },
@@ -96,7 +100,7 @@ const NodeItemIcon = React.memo(function NodeItemIcon({
       <Icon
         className={'node-item-type-icon'}
         placeholder={node.type !== 'tree'}
-        type={getIconType(node)}
+        type={loading ? 'loading' : getIconType(node)}
       />
       {node.type === 'commit' ? (
         <Icon type={getIconType(node)} />
diff --git a/src/components/SideBar.tsx b/src/components/SideBar.tsx
index 75259e5..135a5fb 100644
--- a/src/components/SideBar.tsx
+++ b/src/components/SideBar.tsx
@@ -100,6 +100,7 @@ const RawGitako: React.FC<Props & ConnectorState> = function RawGitako(props) {
     errorDueToAuth,
     metaData,
     treeData: treeRoot,
+    defer,
     error,
     shouldShow,
     showSettings,
@@ -119,7 +120,7 @@ const RawGitako: React.FC<Props & ConnectorState> = function RawGitako(props) {
           <div className={'gitako-side-bar-content'}>
             <div className={'header'}>
               {metaData ? <MetaBar metaData={metaData} /> : <div />}
-
+              {defer && <p>In Async Mode</p>}
               <div className={'close-side-bar-button-position'}>
                 <button className={'close-side-bar-button'} onClick={toggleShowSideBar}>
                   <Icon className={'action-icon'} type={'x'} />
diff --git a/src/driver/core/FileExplorer.ts b/src/driver/core/FileExplorer.ts
index 9a9405d..bb6aa21 100644
--- a/src/driver/core/FileExplorer.ts
+++ b/src/driver/core/FileExplorer.ts
@@ -31,24 +31,14 @@ export type ConnectorState = {
   expandTo: GetCreatedMethod<typeof expandTo>
 }
 
-function getVisibleParentNode(
-  nodes: TreeNode[],
-  focusedNode: TreeNode,
-  depths: Map<TreeNode, number>,
-) {
-  const focusedNodeIndex = nodes.indexOf(focusedNode)
-  const focusedNodeDepth = depths.get(focusedNode)
-  let indexOfParentNode = focusedNodeIndex - 1
-  let depth: number | undefined
-  while (indexOfParentNode !== -1) {
-    depth = depths.get(nodes[indexOfParentNode])
-    if (depth === undefined || focusedNodeDepth === undefined || !(depth >= focusedNodeDepth)) {
-      break
+function getVisibleParentNode(nodes: TreeNode[], focusedNode: TreeNode) {
+  let index = nodes.indexOf(focusedNode) - 1
+  while (index >= 0) {
+    if (nodes[index].contents?.includes(focusedNode)) {
+      return nodes[index]
     }
-    --indexOfParentNode
+    --index
   }
-  const parentNode = nodes[indexOfParentNode]
-  return parentNode
 }
 
 type Task = () => void
@@ -58,14 +48,22 @@ let visibleNodesGenerator: VisibleNodesGenerator
 type BoundMethodCreator<Args extends any[] = []> = MethodCreator<Props, ConnectorState, Args>
 
 export const setUpTree: BoundMethodCreator<[
-  Pick<Props, 'treeRoot' | 'metaData'> & Pick<Config, 'compressSingletonFolder'>,
-]> = dispatch => async ({ treeRoot, metaData, compressSingletonFolder }) => {
+  Pick<Props, 'treeRoot' | 'metaData'> & { config: Config },
+]> = dispatch => async ({ treeRoot, metaData, config }) => {
   if (!treeRoot) return
   dispatch.set({ state: 'rendering' })
 
-  visibleNodesGenerator = new VisibleNodesGenerator(treeRoot, {
+  const { compressSingletonFolder } = config
+
+  visibleNodesGenerator = new VisibleNodesGenerator({
+    root: treeRoot,
     compress: compressSingletonFolder,
+    async getTreeData(path) {
+      const { root } = await platform.getTreeData(metaData, path, false, config.access_token)
+      return root
+    },
   })
+  visibleNodesGenerator.hub.addEventListener('emit', visibleNodes => dispatch.set({ visibleNodes }))
 
   tasksAfterRender.push(DOMHelper.focusSearchInput)
   dispatch.set({ state: 'done' })
@@ -74,7 +72,6 @@ export const setUpTree: BoundMethodCreator<[
     visibleNodesGenerator.visibleNodes.nodes.forEach(node =>
       dispatch.call(toggleNodeExpansion, node, { skipScrollToNode: true, recursive: true }),
     )
-    dispatch.call(updateVisibleNodes)
   } else {
     const targetPath = platform.getCurrentPath(metaData.branchName)
     if (targetPath) dispatch.call(goTo, targetPath)
@@ -91,7 +88,7 @@ export const execAfterRender: BoundMethodCreator = dispatch => () => {
 export const handleKeyDown: BoundMethodCreator<[React.KeyboardEvent]> = dispatch => event => {
   const [{ searched, visibleNodes }, { loadWithPJAX }] = dispatch.get()
   if (!visibleNodes) return
-  const { nodes, focusedNode, expandedNodes, depths } = visibleNodes
+  const { nodes, focusedNode, expandedNodes } = visibleNodes
   function handleVerticalMove(index: number) {
     if (0 <= index && index < nodes.length) {
       DOMHelper.focusFileExplorer()
@@ -123,7 +120,7 @@ export const handleKeyDown: BoundMethodCreator<[React.KeyboardEvent]> = dispatch
           dispatch.call(setExpand, focusedNode, false)
         } else {
           // go forward to the start of the list, find the closest node with lower depth
-          const parentNode = getVisibleParentNode(nodes, focusedNode, depths)
+          const parentNode = getVisibleParentNode(nodes, focusedNode)
           if (parentNode) {
             dispatch.call(focusNode, parentNode, false)
           }
@@ -136,9 +133,7 @@ export const handleKeyDown: BoundMethodCreator<[React.KeyboardEvent]> = dispatch
         if (focusedNode.type === 'tree') {
           if (expandedNodes.has(focusedNode.path)) {
             const nextNode = nodes[focusedNodeIndex + 1]
-            const d1 = depths.get(nextNode)
-            const d2 = depths.get(focusedNode)
-            if (d1 !== undefined && d2 !== undefined && d1 > d2) {
+            if (focusedNode.contents?.includes(nextNode)) {
               dispatch.call(focusNode, nextNode, false)
             }
           } else {
@@ -200,23 +195,21 @@ export const search: BoundMethodCreator<[string]> = dispatch => searchKey => {
   dispatch.set({ searchKey, searched: searchKey !== '' })
   const regexps = searchKeyToRegexps(searchKey)
   visibleNodesGenerator.search(regexps)
-  dispatch.call(updateVisibleNodes)
 }
 
 export const goTo: BoundMethodCreator<[string[]]> = dispatch => async currentPath => {
   dispatch.set({ searchKey: '', searched: false })
   visibleNodesGenerator.search(null)
-  dispatch.call(updateVisibleNodes)
   tasksAfterRender.push(() => {
     dispatch.call(expandTo, currentPath)
   })
 }
 
-export const setExpand: BoundMethodCreator<[TreeNode, boolean]> = dispatch => (
+export const setExpand: BoundMethodCreator<[TreeNode, boolean]> = dispatch => async (
   node,
   expand = false,
 ) => {
-  visibleNodesGenerator.setExpand(node, expand)
+  await visibleNodesGenerator.setExpand(node, expand)
   dispatch.call(focusNode, node, false)
 }
 
@@ -226,8 +219,8 @@ export const toggleNodeExpansion: BoundMethodCreator<[
     skipScrollToNode?: boolean
     recursive?: boolean
   },
-]> = dispatch => (node, { skipScrollToNode = false, recursive = false }) => {
-  visibleNodesGenerator.toggleExpand(node, recursive)
+]> = dispatch => async (node, { skipScrollToNode = false, recursive = false }) => {
+  await visibleNodesGenerator.toggleExpand(node, recursive)
   dispatch.call(focusNode, node, skipScrollToNode)
   tasksAfterRender.push(DOMHelper.focusFileExplorer)
 }
@@ -235,10 +228,7 @@ export const toggleNodeExpansion: BoundMethodCreator<[
 export const focusNode: BoundMethodCreator<[TreeNode | null, boolean]> = dispatch => (
   node: TreeNode | null,
 ) => {
-  const [{ visibleNodes }] = dispatch.get()
-  if (!visibleNodes) return
   visibleNodesGenerator.focusNode(node)
-  dispatch.call(updateVisibleNodes)
 }
 
 export const onNodeClick: BoundMethodCreator<[
@@ -275,15 +265,9 @@ export const onNodeClick: BoundMethodCreator<[
   }
 }
 
-export const expandTo: BoundMethodCreator<[string[]]> = dispatch => currentPath => {
-  const nodeExpandedTo = visibleNodesGenerator.expandTo(currentPath.join('/'))
+export const expandTo: BoundMethodCreator<[string[]]> = dispatch => async currentPath => {
+  const nodeExpandedTo = await visibleNodesGenerator.expandTo(currentPath.join('/'))
   if (nodeExpandedTo) {
     visibleNodesGenerator.focusNode(nodeExpandedTo)
   }
-  dispatch.call(updateVisibleNodes)
-}
-
-export const updateVisibleNodes: BoundMethodCreator = dispatch => () => {
-  const { visibleNodes } = visibleNodesGenerator
-  dispatch.set({ visibleNodes })
 }
diff --git a/src/driver/core/SideBar.ts b/src/driver/core/SideBar.ts
index ab9cb61..b3e18ae 100644
--- a/src/driver/core/SideBar.ts
+++ b/src/driver/core/SideBar.ts
@@ -21,6 +21,7 @@ export type ConnectorState = {
   // file tree data
   treeData?: TreeNode
   logoContainerElement: Element | null
+  defer?: boolean
   disabled: boolean
   initializingPromise: Promise<void> | null
 } & {
@@ -69,6 +70,8 @@ export const init: BoundMethodCreator = dispatch => async () => {
         userName: metaData.userName,
         repoName: metaData.repoName,
       },
+      '/',
+      true,
       accessToken,
     )
     const caughtAggressiveError = getTreeDataAggressively?.catch(error => {
@@ -103,6 +106,8 @@ export const init: BoundMethodCreator = dispatch => async () => {
           userName: metaData.userName,
           repoName: metaData.repoName,
         },
+        '/',
+        true,
         accessToken,
       )
     } else {
@@ -114,9 +119,9 @@ export const init: BoundMethodCreator = dispatch => async () => {
       })
     }
     getTreeData
-      .then(async treeData => {
+      .then(async ({ root: treeData, defer }) => {
         if (treeData) {
-          dispatch.set({ treeData })
+          dispatch.set({ treeData, defer })
         }
       })
       .catch(err => dispatch.call(handleError, err))
diff --git a/src/global.d.ts b/src/global.d.ts
index 9eb1812..3b22a28 100644
--- a/src/global.d.ts
+++ b/src/global.d.ts
@@ -23,5 +23,8 @@ type IO<T> = {
   onChange(value: T): void
 }
 
-// do not use with generics
 type Override<Original, Incoming> = Omit<Original, keyof Incoming> & Incoming
+
+type VoidFN<T> = (payload: T) => void
+
+type Async<T> = T | Promise<T>
diff --git a/src/platforms/GitHub/API.ts b/src/platforms/GitHub/API.ts
index d28db7a..01e3fd4 100644
--- a/src/platforms/GitHub/API.ts
+++ b/src/platforms/GitHub/API.ts
@@ -74,9 +74,13 @@ export async function getTreeData(
   userName: string,
   repoName: string,
   branchName: string,
+  recursive?: boolean,
   accessToken?: string,
 ): Promise<GitHubAPI.TreeData> {
-  const url = `https://${API_ENDPOINT}/repos/${userName}/${repoName}/git/trees/${branchName}?recursive=1`
+  const search = new URLSearchParams()
+  if (recursive) search.set('recursive', '1')
+  const url =
+    `https://${API_ENDPOINT}/repos/${userName}/${repoName}/git/trees/${branchName}?` + search
   return await request(url, { accessToken })
 }
 
diff --git a/src/platforms/GitHub/index.ts b/src/platforms/GitHub/index.ts
index d0b325d..73aad0b 100644
--- a/src/platforms/GitHub/index.ts
+++ b/src/platforms/GitHub/index.ts
@@ -26,6 +26,15 @@ function processTree(tree: TreeNode[]): TreeNode {
       const item = pathToItem.get(path)
       if (item) {
         itemsToCreateTreeNode.push(item)
+      } else {
+        const $item: TreeNode = {
+          name: path.split('/').pop() || '',
+          path,
+          type: 'tree',
+          contents: [],
+        }
+        pathToItem.set(path, $item)
+        itemsToCreateTreeNode.push($item)
       }
       // 'a/b' -> 'a'
       // 'a' -> ''
@@ -66,6 +75,8 @@ export function isEnterprise() {
   return !window.location.host.endsWith('github.com')
 }
 
+const pathSHAMap = new Map<string, string>()
+
 export const GitHub: Platform = {
   isEnterprise,
   resolveMeta() {
@@ -97,7 +108,7 @@ export const GitHub: Platform = {
       defaultBranchName: data.default_branch,
     }
   },
-  async getTreeData(metaData, accessToken) {
+  async getTreeData(metaData, path = '/', recursive, accessToken) {
     const { userName, repoName, branchName } = metaData
 
     const pullId = URLHelper.isInPullPage()
@@ -157,11 +168,35 @@ export const GitHub: Platform = {
         ),
       )
 
-      const tree = processTree(nodes)
-      return tree
+      const root = processTree(nodes)
+      return { root }
+    }
+
+    const sha = path === '/' ? branchName : pathSHAMap.get(path)
+    if (!sha) throw new Error(`No sha for path "${path}"`)
+    const treeData = await API.getTreeData(userName, repoName, sha, recursive, accessToken)
+
+    // remove deep items
+    if (treeData.truncated) {
+      if (treeData.tree.some(item => item.path.includes('/')))
+        treeData.tree = treeData.tree.filter(item => !item.path.includes('/'))
+    }
+
+    // update map
+    if (path !== '/' || treeData.truncated) {
+      if (path !== '/') {
+        function sanitizePath(path: string) {
+          return path.replace(/\/\/+/g, '/').replace(/^\/|\/$/g, '') || '/'
+        }
+        treeData.tree.forEach(item => {
+          item.path = sanitizePath(`${path}/${item.path}`)
+        })
+      }
+      treeData.tree.forEach(item => {
+        pathSHAMap.set(item.path, item.sha)
+      })
     }
 
-    const treeData = await API.getTreeData(userName, repoName, branchName, accessToken)
     const root = processTree(
       treeData.tree.map(item => ({
         path: item.path || '',
@@ -193,12 +228,12 @@ export const GitHub: Platform = {
         )
 
         if (blobData && blobData.encoding === 'base64' && blobData.content) {
-          resolveGitModules(root, Base64.decode(blobData.content))
+          await resolveGitModules(root, Base64.decode(blobData.content))
         }
       }
     }
 
-    return root
+    return { root, defer: treeData.truncated }
   },
   shouldShow() {
     return Boolean(DOMHelper.isInCodePage() || URLHelper.isInPullPage())
diff --git a/src/platforms/Gitee/index.ts b/src/platforms/Gitee/index.ts
index a5f9e87..7a16edf 100644
--- a/src/platforms/Gitee/index.ts
+++ b/src/platforms/Gitee/index.ts
@@ -108,7 +108,7 @@ export const Gitee: Platform = {
       defaultBranchName: data.default_branch,
     }
   },
-  async getTreeData(metaData, accessToken) {
+  async getTreeData(metaData, path, recursive, accessToken) {
     const { userName, repoName, branchName } = metaData
     const treeData = await API.getTreeData(userName, repoName, branchName)
     const root = parseTreeData(treeData, metaData)
@@ -124,12 +124,12 @@ export const Gitee: Platform = {
         )
 
         if (blobData && blobData.encoding === 'base64' && blobData.content) {
-          resolveGitModules(root, Base64.decode(blobData.content))
+          await resolveGitModules(root, Base64.decode(blobData.content))
         }
       }
     }
 
-    return root
+    return { root }
   },
   shouldShow() {
     return DOMHelper.isInCodePage()
diff --git a/src/platforms/platform.d.ts b/src/platforms/platform.d.ts
index 0dee955..fb5de6e 100644
--- a/src/platforms/platform.d.ts
+++ b/src/platforms/platform.d.ts
@@ -5,7 +5,12 @@ type Platform = {
     metaData: Pick<MetaData, 'userName' | 'repoName'>,
     accessToken?: string,
   ): Promise<Pick<MetaData, 'userUrl' | 'repoUrl' | 'defaultBranchName'>>
-  getTreeData(metaData: MetaData, accessToken?: string): Promise<TreeNode>
+  getTreeData(
+    metaData: MetaData,
+    path?: string,
+    recursive?: boolean,
+    accessToken?: string,
+  ): Promise<{ root: TreeNode; defer?: boolean }>
   shouldShow(): boolean
   shouldExpandAll?(): boolean
   getCurrentPath(branchName: string): string[] | null
diff --git a/src/utils/EventHub.ts b/src/utils/EventHub.ts
new file mode 100644
index 0000000..1fc02c5
--- /dev/null
+++ b/src/utils/EventHub.ts
@@ -0,0 +1,54 @@
+export class EventSubscription<Data, Listener extends VoidFN<Data> = VoidFN<Data>> {
+  listeners: Listener[] = []
+
+  emit(data: Data) {
+    for (const listener of this.listeners) listener(data)
+  }
+
+  addEventListener(listener: Listener) {
+    this.listeners.push(listener)
+    return () => this.removeEventListener(listener)
+  }
+
+  removeEventListener(listener: Listener) {
+    const index = this.listeners.indexOf(listener)
+    if (index !== -1) this.listeners.splice(index, 1)
+  }
+}
+
+export class EventHub<
+  Shape extends {
+    [event: string]: any
+  }
+> {
+  ports: {
+    [key in keyof Shape]: EventSubscription<Shape[key]>
+  } = {} as EventHub<Shape>['ports']
+
+  getPort<Event extends keyof Shape>(event: Event) {
+    if (!this.ports[event]) this.ports[event] = new EventSubscription<Shape[typeof event]>()
+
+    return this.ports[event]
+  }
+
+  emit<Event extends keyof Shape>(event: Event, data: Shape[Event]) {
+    const port = this.getPort(event)
+    return port.emit(data)
+  }
+
+  addEventListener<Event extends keyof Shape>(
+    event: Event,
+    listener: (data: Shape[Event]) => void,
+  ) {
+    const port = this.getPort(event)
+    return port.addEventListener(listener)
+  }
+
+  removeEventListener<Event extends keyof Shape>(
+    event: Event,
+    listener: (data: Shape[Event]) => void,
+  ) {
+    const port = this.getPort(event)
+    return port.removeEventListener(listener)
+  }
+}
diff --git a/src/utils/VisibleNodesGenerator.ts b/src/utils/VisibleNodesGenerator.ts
index dc4a7b0..b5ae2c0 100644
--- a/src/utils/VisibleNodesGenerator.ts
+++ b/src/utils/VisibleNodesGenerator.ts
@@ -1,25 +1,5 @@
-import { findNode } from './general'
-
-/**
- * This is the stack for generating an array of nodes for rendering
- *
- * when lower layer changes, higher layers would reset
- * when higher layer changes, lower layers would not notice
- *
- *  render stack                 | when will change         | on change callback
- *
- *  ^  changes frequently
- *  |
- *  |4 focus                     | when hover/focus move    | onFocusChange
- *  |                            |                          |   expandedNodes + focusNode -> visibleNodes
- *  |3 expansion                 | when fold/unfold         | onExpansionChange
- *  |                            |                          |   searchedNodes + toggleNode -> expandedNodes
- *  |2 filters                   | when filter(search)      | onSearch
- *  |                            |                          |   treeNodes + searchKey -> searchedNodes
- *  |1 tree: { root <-> nodes }  | when tree init           | treeHelper.parse
- *  |                                                       |   tree data from api -> { root, nodes }
- *  v  stable
- */
+import { EventHub } from './EventHub'
+import { findNode, traverse, withEffect } from './general'
 
 function search(
   root: TreeNode,
@@ -31,18 +11,18 @@ function search(
 
   if (root.type === 'tree' && root.contents) {
     let childMatch = false
-    for (const item of root.contents) {
-      if (match(item)) {
+    for (const node of root.contents) {
+      if (match(node)) {
         childMatch = true
         break
       }
     }
 
-    for (const item of root.contents) {
-      const $item = search(item, match, onChildMatch)
-      if ($item) {
-        if ($item !== item) childMatch = true
-        contents.push($item)
+    for (const node of root.contents) {
+      const $node = search(node, match, onChildMatch)
+      if ($node) {
+        if ($node !== node) childMatch = true
+        contents.push($node)
       }
     }
 
@@ -75,10 +55,10 @@ function compressTree(root: TreeNode, prefix: string[] = []): TreeNode {
 
     let compressed = false
     const contents = []
-    for (const item of root.contents) {
-      const $item = compressTree(item)
-      if ($item !== item) compressed = true
-      contents.push($item)
+    for (const node of root.contents) {
+      const $node = compressTree(node)
+      if ($node !== node) compressed = true
+      contents.push($node)
     }
     if (compressed)
       return {
@@ -95,177 +75,242 @@ function compressTree(root: TreeNode, prefix: string[] = []): TreeNode {
     : root
 }
 
-class L1 {
-  root: TreeNode
-
-  constructor(root: TreeNode) {
-    this.root = root
+function mergeNodes(target: TreeNode, source: TreeNode) {
+  for (const node of source.contents || []) {
+    const dup = target.contents?.find($node => $node.path === node.path)
+    if (dup) {
+      mergeNodes(dup, node)
+    } else {
+      if (!target.contents) target.contents = []
+      target.contents.push(node)
+    }
   }
 }
 
-class L2 {
-  private l1: L1
-  compress: boolean
-  root: TreeNode | null = null
+class BaseLayer {
+  baseRoot: TreeNode
+  getTreeData: (path: string) => Async<TreeNode>
+  loading: Set<TreeNode['path']> = new Set()
 
-  constructor(l1: L1, options: Options) {
-    this.l1 = l1
+  baseHub = new EventHub<{
+    emit: BaseLayer['baseRoot']
+    loadingChange: BaseLayer['loading']
+  }>()
+
+  constructor({ root, getTreeData }: Options) {
+    this.baseRoot = root
+    this.getTreeData = getTreeData
+  }
+
+  loadTreeData = async (path: string) => {
+    const node = await findNode(this.baseRoot, path)
+    if (node && node.type !== 'tree') return node
+    if (node?.contents?.length) return node // check in memory
+    if (this.loading.has(path)) return
+
+    this.loading.add(path)
+    this.baseHub.emit('loadingChange', this.loading)
+    mergeNodes(this.baseRoot, await this.getTreeData(path))
+    this.loading.delete(path)
+    this.baseHub.emit('loadingChange', this.loading)
+    this.baseHub.emit('emit', this.baseRoot)
+
+    return await findNode(this.baseRoot, path)
+  }
+}
+
+class ShakeLayer extends BaseLayer {
+  shackedRoot: TreeNode | null = null
+  lastMatch: Parameters<ShakeLayer['shake']>[0] = undefined
+  shakeHub = new EventHub<{ emit: TreeNode | null }>()
+
+  constructor(options: Options) {
+    super(options)
+
+    this.baseHub.addEventListener('emit', () => this.shake(this.lastMatch))
+  }
+
+  shake = withEffect(
+    (p?: { match: (node: TreeNode) => boolean; onChildMatch: (node: TreeNode) => void }) => {
+      this.lastMatch = p
+      if (p) {
+        const { match, onChildMatch } = p
+        this.shackedRoot = search(this.baseRoot, match, onChildMatch)
+      } else this.shackedRoot = this.baseRoot
+    },
+    () => this.shakeHub.emit('emit', this.shackedRoot),
+  )
+}
+
+class CompressLayer extends ShakeLayer {
+  private compress: boolean
+  depths = new Map<TreeNode, number>()
+  compressedRoot: TreeNode | null = null
+  compressHub = new EventHub<{ emit: TreeNode | null }>()
+
+  constructor(options: Options) {
+    super(options)
     this.compress = Boolean(options.compress)
+
+    this.shakeHub.addEventListener('emit', () => this.compressTree())
   }
 
-  search = (
-    match: null | ((node: TreeNode) => boolean),
-    onChildMatch: (node: TreeNode) => void,
-  ) => {
-    const rootNode = match ? search(this.l1.root, match, onChildMatch) : this.l1.root
+  compressTree = withEffect(
+    () => {
+      this.depths.clear()
+      this.compressedRoot =
+        this.shackedRoot && this.compress
+          ? {
+              ...this.shackedRoot,
+              contents: this.shackedRoot.contents?.map(node => compressTree(node)),
+            }
+          : this.shackedRoot
 
-    this.root =
-      rootNode && this.compress
-        ? { ...rootNode, contents: rootNode.contents?.map(node => compressTree(node)) }
-        : rootNode
-  }
+      const recordDepth = (node: TreeNode, depth = 0) => {
+        this.depths.set(node, depth)
+        for (const $node of node.contents || []) {
+          recordDepth($node, depth + 1)
+        }
+      }
+      if (this.compressedRoot) recordDepth(this.compressedRoot, -1)
+    },
+    () => this.compressHub.emit('emit', this.compressedRoot),
+  )
 }
 
-class L3 {
-  private l1: L1
-  private l2: L2
-
+class FlattenLayer extends CompressLayer {
+  focusedNode: TreeNode | null = null
   nodes: TreeNode[] = []
   expandedNodes: Set<TreeNode['path']> = new Set()
-  depths: Map<TreeNode, number> = new Map()
+  flattenHub = new EventHub<{ emit: null }>()
 
-  constructor(l1: L1, l2: L2) {
-    this.l1 = l1
-    this.l2 = l2
+  constructor(options: Options) {
+    super(options)
+
+    this.compressHub.addEventListener('emit', () => this.generateVisibleNodes())
   }
 
-  toggleExpand = (node: TreeNode, recursive?: boolean) => {
-    const expand = !this.expandedNodes.has(node.path)
-    if (recursive) {
-      const recursiveSetExpand = (node: TreeNode, expand: boolean) => {
-        this.barelySetExpand(node, expand)
-        node.contents?.forEach($node => recursiveSetExpand($node, expand))
-      }
-      recursiveSetExpand(node, expand)
-      this.generateVisibleNodes()
-    } else {
-      this.setExpand(node, expand)
-    }
+  generateVisibleNodes = withEffect(
+    async () => {
+      const nodes: TreeNode[] = []
+      await traverse(
+        this.compressedRoot?.contents,
+        node => {
+          nodes.push(node)
+          return node.type === 'tree' && this.expandedNodes.has(node.path)
+        },
+        node => {
+          return node.contents || []
+        },
+      )
+      this.nodes = nodes
+    },
+    () => this.flattenHub.emit('emit', null),
+  )
+
+  focusNode = (node: TreeNode | null) => {
+    this.focusedNode = node
   }
 
   barelySetExpand = (node: TreeNode, expand: boolean) => {
-    if (expand && node.contents) {
-      // only node with contents is expandable
+    if (expand) {
       this.expandedNodes.add(node.path)
     } else {
       this.expandedNodes.delete(node.path)
     }
   }
 
-  setExpand = (node: TreeNode, expand: boolean) => {
+  $setExpand = (node: TreeNode, expand: boolean) => {
     this.barelySetExpand(node, expand)
-    this.generateVisibleNodes()
+    if (expand && node.type === 'tree') return this.loadTreeData(node.path)
   }
+  setExpand = withEffect(this.$setExpand, this.generateVisibleNodes)
 
-  expandTo = (path: string, expandAlongTheWay?: boolean) => {
-    const rootNode = this.l2.root
-    if (expandAlongTheWay && path.includes('/')) {
-      this.expandTo(path.slice(0, path.lastIndexOf('/')), true)
-    }
-    const node = rootNode && findNode(rootNode, path.split('/'), node => this.setExpand(node, true))
-    if (node) this.setExpand(node, true)
-    return node
-  }
-
-  search = (regexp: RegExp | null) => {
-    this.expandedNodes.clear()
-    this.l2.search(regexp && (node => regexp.test(node.name)), node =>
-      this.expandedNodes.add(node.path),
+  toggleExpand = withEffect(async (node: TreeNode, recursive = false) => {
+    const expand = !this.expandedNodes.has(node.path)
+    await traverse(
+      [node],
+      async node => {
+        await this.$setExpand(node, expand)
+        return recursive
+      },
+      node => node.contents || [],
     )
-    this.generateVisibleNodes()
-  }
+  }, this.generateVisibleNodes)
 
-  generateVisibleNodes = () => {
-    this.depths.clear()
-    const nodes: TreeNode[] = []
-    if (this.l2.root?.contents) {
-      const traverse = (root: TreeNode, depth = 0) => {
-        nodes.push(root)
-        this.depths.set(root, depth)
-        if (this.expandedNodes.has(root.path) && root.type === 'tree' && root.contents?.length) {
-          for (const item of root.contents) {
-            traverse(item, depth + 1)
+  expandTo = withEffect(async (path: string) => {
+    const rootNode = this.compressedRoot
+    if (rootNode) {
+      await traverse(
+        [rootNode],
+        async node => {
+          const match = path.startsWith(node.path)
+          if (node.path && match) {
+            // rootNode.path === ''
+            await this.$setExpand(node, true)
           }
-        }
-      }
-      for (const item of this.l2.root.contents) {
-        traverse(item)
-      }
+          return match
+        },
+        node => node?.contents || [],
+      )
+
+      const node = await findNode(rootNode, path)
+      return node
     }
-    this.nodes = nodes
-  }
-}
+  }, this.generateVisibleNodes)
 
-export type VisibleNodes = {
-  nodes: L3['nodes']
-  depths: L3['depths']
-  expandedNodes: L3['expandedNodes']
-  focusedNode: L4['focusedNode']
-}
-
-class L4 {
-  private l1: L1
-  private l2: L2
-  private l3: L3
-
-  focusedNode: TreeNode | null
-
-  constructor(l1: L1, l2: L2, l3: L3) {
-    this.l1 = l1
-    this.l2 = l2
-    this.l3 = l3
-    this.focusedNode = null
-  }
-
-  focusNode = (node: TreeNode | null) => {
-    this.focusedNode = node
-  }
+  search = withEffect((regexp: RegExp | null) => {
+    this.focusNode(null)
+    this.shake(
+      regexp
+        ? {
+            match: node => regexp.test(node.name),
+            onChildMatch: node => this.$setExpand(node, true),
+          }
+        : undefined,
+    )
+  }, this.generateVisibleNodes)
 }
 
 type Options = {
-  compress?: boolean
+  root: BaseLayer['baseRoot']
+  getTreeData: BaseLayer['getTreeData']
+  compress: CompressLayer['compress']
 }
 
-export class VisibleNodesGenerator {
-  private l1: L1
-  private l2: L2
-  private l3: L3
-  private l4: L4
+export type VisibleNodes = {
+  loading: BaseLayer['loading']
+  depths: CompressLayer['depths']
+  nodes: FlattenLayer['nodes']
+  expandedNodes: FlattenLayer['expandedNodes']
+  focusedNode: FlattenLayer['focusedNode']
+}
 
-  constructor(root: TreeNode, options: Options) {
-    this.l1 = new L1(root)
-    this.l2 = new L2(this.l1, options)
-    this.l3 = new L3(this.l1, this.l2)
-    this.l4 = new L4(this.l1, this.l2, this.l3)
+export class VisibleNodesGenerator extends FlattenLayer {
+  hub = new EventHub<{
+    emit: VisibleNodes
+  }>()
+  constructor(options: Options) {
+    super(options)
+
+    this.focusNode = withEffect(this.focusNode.bind(this), this.update.bind(this))
 
     this.search(null)
+    this.flattenHub.addEventListener('emit', () => this.update())
+    this.baseHub.addEventListener('loadingChange', () => this.update())
   }
 
-  search: L3['search'] = regexps => {
-      this.l3.search(regexps)
-      this.l4.focusNode(null)
-    }
-  setExpand: L3['setExpand'] = (...args) => this.l3.setExpand(...args)
-  toggleExpand: L3['toggleExpand'] = (...args) => this.l3.toggleExpand(...args)
-  expandTo: L3['expandTo'] = (...args) => this.l3.expandTo(...args)
-  focusNode: L4['focusNode'] = (...args) => this.l4.focusNode(...args)
+  update() {
+    this.hub.emit('emit', this.visibleNodes)
+  }
 
-  get visibleNodes() {
+  get visibleNodes(): VisibleNodes {
     return {
-      nodes: this.l3.nodes,
-      depths: this.l3.depths,
-      expandedNodes: this.l3.expandedNodes,
-      focusedNode: this.l4.focusedNode,
+      nodes: this.nodes,
+      depths: this.depths,
+      expandedNodes: this.expandedNodes,
+      focusedNode: this.focusedNode,
+      loading: this.loading,
     }
   }
 }
diff --git a/src/utils/general.ts b/src/utils/general.ts
index b7bd562..50dcbf3 100644
--- a/src/utils/general.ts
+++ b/src/utils/general.ts
@@ -47,38 +47,49 @@ export function friendlyFormatShortcut(shortcut?: string) {
 }
 
 /**
- * if item's name matches path, return self-depth of the item
+ * if item's name matches path, return the depth of the item
  * else return 0
  */
 function measureDistance(item: TreeNode, path: TreeNode['name'][]): number {
   const pathString = path.join('/')
-  if (item.name.indexOf(pathString + '/') === 0) {
+  if (item.name.startsWith(pathString + '/')) {
     // If accessing a leading item of compressed node, path will be shorter than item.name
     return path.length
-  } else if (pathString === item.name || pathString.indexOf(item.name + '/') === 0) {
+  } else if (pathString === item.name || pathString.startsWith(item.name + '/')) {
     return item.name.split('/').length
   }
   return 0
 }
 
+export async function traverse<T>(
+  range: T[] = [],
+  conditionAndEffect: (node: T) => Async<boolean>,
+  getChildren: (node: T) => T[],
+) {
+  for (const item of range) {
+    if (await conditionAndEffect(item)) {
+      await traverse(getChildren(item), conditionAndEffect, getChildren)
+    }
+  }
+}
+
 /**
  * look for the first item matches given path under root.content
  */
-export function findNode(
-  root: TreeNode,
-  path: TreeNode['name'][],
-  callback?: (node: TreeNode) => void,
-): TreeNode | undefined {
-  if (Array.isArray(root.contents)) {
-    for (const item of root.contents) {
-      const distance = measureDistance(item, path)
-      if (distance > 0) {
-        if (callback) callback(item)
-        if (path.length === distance) return item
-        return findNode(item, path.slice(distance), callback)
+export async function findNode(root: TreeNode, path: TreeNode['path']) {
+  let node: TreeNode | undefined
+  await traverse(
+    [root],
+    $node => {
+      if (path === $node.path) {
+        node = $node
+        return false
       }
-    }
-  }
+      return path.startsWith($node.path)
+    },
+    node => node.contents || [],
+  )
+  return node
 }
 
 export function createStyleSheet(content: string) {
@@ -159,3 +170,14 @@ export function isValidRegexpSource(source: string) {
     return false
   }
 }
+
+export function withEffect<Method extends (...args: any[]) => any>(
+  method: Method,
+  effect: (payload: ReturnType<Method>) => void,
+): (...args: Parameters<Method>) => ReturnType<Method> {
+  return (...args) => {
+    const returnValue = method.apply(null, args)
+    Promise.resolve(returnValue).then(effect)
+    return returnValue
+  }
+}
diff --git a/src/utils/gitSubmodule.ts b/src/utils/gitSubmodule.ts
index be78173..87f81b1 100644
--- a/src/utils/gitSubmodule.ts
+++ b/src/utils/gitSubmodule.ts
@@ -38,13 +38,14 @@ type ParsedModule = {
   [key: string]: string | undefined
 }
 
-function handleParsed(root: TreeNode, parsed: ParsedINI) {
-  Object.values(parsed).forEach(value => {
+// TODO: merge into getTreeData callback
+async function handleParsed(root: TreeNode, parsed: ParsedINI) {
+  for (const value of Object.values(parsed)) {
     if (typeof value === 'string') return
     const url = value?.url
     const path = value?.path
     if (typeof url === 'string' && typeof path === 'string') {
-      const node = findNode(root, path.split('/'))
+      const node = await findNode(root, path)
       if (node) {
         if (subModuleURLRegex.HTTPGit.test(url)) {
           node.url = transformModuleHTTPDotGitURL(node, url)
@@ -61,16 +62,16 @@ function handleParsed(root: TreeNode, parsed: ParsedINI) {
         // raiseError(new Error(`Submodule node not found`), { path })
       }
     } else {
-      handleParsed(root, value as ParsedINI)
+      await handleParsed(root, value as ParsedINI)
     }
-  })
+  }
 }
 
-export function resolveGitModules(root: TreeNode, content: string) {
+export async function resolveGitModules(root: TreeNode, content: string) {
   try {
     if (Array.isArray(root.contents)) {
       const parsed: ParsedINI = ini.parse(content)
-      handleParsed(root, parsed)
+      await handleParsed(root, parsed)
     }
   } catch (err) {
     throw new Error(`Error resolving git modules`)