From 3cacad133ced8bfef061415c8e1ad46c753e92b3 Mon Sep 17 00:00:00 2001 From: EnixCoda Date: Tue, 3 May 2022 16:41:45 +0800 Subject: [PATCH] refactor: enhance network requests --- src/platforms/GitHub/API.ts | 178 ++++++++-------- src/platforms/GitHub/DOMHelper.ts | 4 + src/platforms/GitHub/getCommitTreeData.ts | 44 ++++ .../GitHub/getPullRequestTreeData.ts | 128 ++++++++++++ src/platforms/GitHub/index.ts | 192 +----------------- src/platforms/GitHub/utils.ts | 51 +++++ tsconfig.json | 2 +- 7 files changed, 325 insertions(+), 274 deletions(-) create mode 100644 src/platforms/GitHub/getCommitTreeData.ts create mode 100644 src/platforms/GitHub/getPullRequestTreeData.ts diff --git a/src/platforms/GitHub/API.ts b/src/platforms/GitHub/API.ts index 3ab9af3..c1b4d98 100644 --- a/src/platforms/GitHub/API.ts +++ b/src/platforms/GitHub/API.ts @@ -1,7 +1,8 @@ import { errors } from 'platforms' import { isEnterprise } from '.' +import { continuousLoadPages, getDOM, resolveHeaderLink } from './utils' -function apiRateLimitExceeded(content: any /* safe any */) { +function isAPIRateLimitExceeded(content: any /* safe any */) { return content?.['documentation_url'] === 'https://developer.github.com/v3/#rate-limiting' } @@ -13,15 +14,23 @@ function isBlockedProject(content: any /* safe any */) { return content?.['message'] === 'Repository access blocked' } -async function request( +export const responseBodyResolvers = { + asIs: (response: Response) => response, + json(response: Response) { + const contentType = response.headers.get('Content-Type') || response.headers.get('content-type') + if (contentType?.includes('application/json')) return response.json() + throw new Error(`Response content type is "${contentType}"`) + }, +} + +async function request( url: string, { accessToken, - resolveMode = 'body-json', }: { - resolveMode?: 'body-json' | 'response' accessToken?: string } = {}, + bodyResolver: (response: Response) => Async = responseBodyResolvers.json, ) { const headers = {} as HeadersInit & { Authorization?: string @@ -37,35 +46,25 @@ async function request( throw new Error(errors.CONNECTION_BLOCKED) } - const contentType = res.headers.get('Content-Type') || res.headers.get('content-type') - const isJson = contentType?.includes('application/json') // About res.ok: // True if res.status between 200~299 // Ref: https://developer.mozilla.org/en-US/docs/Web/API/Response/ok - if (res.ok) { - switch (resolveMode) { - case 'body-json': { - if (isJson) return res.json() - throw new Error(`Response content type is "${contentType}"`) - } - case 'response': - return res - } - throw new Error(`Unknown resolve mode: ${resolveMode}`) - } + if (res.ok) return bodyResolver(res) if (res.status === 404 || res.status === 401) throw new Error(errors.NOT_FOUND) - else if (res.status === 403) throw new Error(errors.API_RATE_LIMIT) - else if (res.status === 500) throw new Error(errors.SERVER_FAULT) - else if (resolveMode && isJson) { - const content = await res.json() - if (apiRateLimitExceeded(content)) throw new Error(errors.API_RATE_LIMIT) - if (isEmptyProject(content)) throw new Error(errors.EMPTY_PROJECT) - if (isBlockedProject(content)) throw new Error(errors.BLOCKED_PROJECT) - throw new Error(`Unknown message content "${content?.message}"`) - } else { - throw new Error(`Response content type is "${contentType}"`) - } + if (res.status === 403) throw new Error(errors.API_RATE_LIMIT) + if (res.status === 500) throw new Error(errors.SERVER_FAULT) + + const content = await responseBodyResolvers.json(res) + if (isAPIRateLimitExceeded(content)) throw new Error(errors.API_RATE_LIMIT) + if (isEmptyProject(content)) throw new Error(errors.EMPTY_PROJECT) + if (isBlockedProject(content)) throw new Error(errors.BLOCKED_PROJECT) + + const message = + typeof content === 'object' + ? (content as Record | null)?.message + : `${content}` + throw new Error(`Unknown message content "${message}"`) } const API_ENDPOINT = isEnterprise() ? `${window.location.host}/api/v3` : 'api.github.com' @@ -103,16 +102,28 @@ export async function getPullData( return await request(url, { accessToken }) } +export async function requestPullTreeData( + userName: string, + repoName: string, + pullId: string, + page: number, + pageSize?: number, + accessToken?: string, +) { + const search = new URLSearchParams({ page: page.toString(), per_page: `${pageSize}` }) + const url = `https://${API_ENDPOINT}/repos/${userName}/${repoName}/pulls/${pullId}/files?${search}` + return await request(url, { accessToken }, responseBodyResolvers.asIs) +} + export async function getPullTreeData( userName: string, repoName: string, pullId: string, page: number, + pageSize?: number, accessToken?: string, ): Promise { - const search = new URLSearchParams({ page: page.toString() }) - const url = `https://${API_ENDPOINT}/repos/${userName}/${repoName}/pulls/${pullId}/files?${search}` - return await request(url, { accessToken }) + return (await requestPullTreeData(userName, repoName, pullId, page, pageSize, accessToken)).json() } export async function getPullComments( @@ -129,39 +140,15 @@ export async function getPullPageDocuments( userName: string, repoName: string, pullId: string, + document?: Document, ): Promise { // Response of this API contains view of few files but is not complete. - const filesDOM = await getDOM( - `https://${window.location.host}/${userName}/${repoName}/pull/${pullId}/files?_pjax=%23js-repo-pjax-container`, + return continuousLoadPages( + document || + (await getDOM( + `https://${window.location.host}/${userName}/${repoName}/pull/${pullId}/files?_pjax=%23js-repo-pjax-container`, + )), ) - const hookElement: HTMLDivElement | null = filesDOM.querySelector('div.js-pull-refresh-on-pjax') - const hookSearchParams = new URLSearchParams(hookElement?.dataset.url) - const [baseSHA, headSHA] = [ - hookSearchParams.get('start_commit_oid'), - hookSearchParams.get('end_commit_oid'), - ] - if (!baseSHA || !headSHA) throw new Error(`Cannot fetch SHA for comparison`) - - // The SHA used to be retrieved from DOM of the pull page, but they can be unreliable if the PR has conflicts - const search = new URLSearchParams(window.location.search) - search.set('sha1', baseSHA) - search.set('sha2', headSHA) - let lines = 0 - const diffsDOMs: Document[] = [] - while (true) { - search.set('lines', lines.toString()) - const diffsDOM = await getDOM( - `https://${window.location.host}/${userName}/${repoName}/diffs?${search}`, - ) - diffsDOMs.push(diffsDOM) - - if (diffsDOM.querySelector('.js-diff-progressive-container')) { - lines += 3000 - } else { - break - } - } - return diffsDOMs } export async function getCommitPageDocuments( @@ -169,33 +156,8 @@ export async function getCommitPageDocuments( repoName: string, commitId: string, ): Promise { - /** - * - */ - const fragmentSelector = 'include-fragment[data-targets="diff-file-filter.progressiveLoaders"]' - - let doc = document - const documents: Document[] = [doc] - while (true) { - const fragment = doc.querySelector(fragmentSelector) as HTMLElement - if (!fragment) break - const src = fragment.getAttribute('src') - if (!src) break - const nextDoc = await getDOM(src) - documents.push(nextDoc) - doc = nextDoc - } - - return documents -} - -async function getDOM(url: string) { - return new DOMParser().parseFromString(await (await fetch(url)).text(), 'text/html') + // arguments are not used because info are collected from DOM directly + return continuousLoadPages(document) } export async function getBlobData( @@ -225,7 +187,7 @@ export async function OAuth(code: string): Promise { } } -export async function getCommitTreeData( +export async function requestCommitTreeData( userName: string, repoName: string, sha: string, @@ -237,5 +199,41 @@ export async function getCommitTreeData( page: `${page}`, }) const url = `https://${API_ENDPOINT}/repos/${userName}/${repoName}/commits/${sha}?` + search - return await request(url, { accessToken, resolveMode: 'response' }) + return await request(url, { accessToken }, responseBodyResolvers.asIs) +} + +export async function getPaginatedData(sendRequest: (page: number) => Promise) { + const responses: Response[] = [] + let page = 1 + while (true) { + const response = await sendRequest(page) + responses.push(response) + + const headerLink = response.headers.get('link') + if (headerLink) { + const rels = resolveHeaderLink(headerLink) + if (rels) { + if (rels.position === 'first') { + page++ + } else if (rels.position === 'middle') { + const searchOfLast = new URL(rels.last).searchParams + if (`${page}` === searchOfLast.get('page')) { + // this should not actually happen because GitHub responds `prev` and `first` for the first page + break + } + page++ + } else { + // i.e. rels.position === 'last' + break + } + } else { + // unexpected link header content + break + } + } else { + // no link headers if there is <100 files + break + } + } + return Promise.all(responses.map(responseBodyResolvers.json)) as Promise } diff --git a/src/platforms/GitHub/DOMHelper.ts b/src/platforms/GitHub/DOMHelper.ts index 56a2685..e61e012 100644 --- a/src/platforms/GitHub/DOMHelper.ts +++ b/src/platforms/GitHub/DOMHelper.ts @@ -31,6 +31,10 @@ export function isInCodePage() { return Boolean($(branchListSelector, e => e.offsetWidth > 0 && e.offsetHeight > 0)) } +export function isInPullFilesPage() { + return $('#files_tab_counter') +} + export function getIssueTitle() { const title = $('.gh-header-title')?.textContent return title?.trim().replace(/\n/g, '') diff --git a/src/platforms/GitHub/getCommitTreeData.ts b/src/platforms/GitHub/getCommitTreeData.ts new file mode 100644 index 0000000..def1892 --- /dev/null +++ b/src/platforms/GitHub/getCommitTreeData.ts @@ -0,0 +1,44 @@ +import { formatID } from 'utils/DOMHelper' +import * as API from './API' +import { processTree } from './index' + +export async function getCommitTreeData( + { userName, repoName }: Pick, + commitSHA: string, + accessToken: string | undefined, +) { + const treeData = ( + await API.getPaginatedData(page => + API.requestCommitTreeData(userName, repoName, commitSHA, page, accessToken), + ) + ) + .map(({ files }) => files) + .flat() + + const documents = await API.getCommitPageDocuments(userName, repoName, commitSHA) + + const getItemURL = (path: string) => { + for (const doc of documents) { + const id = doc.querySelector(`[data-path="${path}"]`)?.parentElement?.id + if (id) return formatID(id) + } + } + + const root = processTree( + treeData.map(item => ({ + type: 'blob', + path: item.filename, + name: item.filename.split('/').pop() || '', + url: getItemURL(item.filename) || item.blob_url, + sha: item.patch, + diff: { + status: item.status, + additions: item.additions, + deletions: item.deletions, + changes: item.changes, + }, + })), + ) + + return { root } +} diff --git a/src/platforms/GitHub/getPullRequestTreeData.ts b/src/platforms/GitHub/getPullRequestTreeData.ts new file mode 100644 index 0000000..5c3fe47 --- /dev/null +++ b/src/platforms/GitHub/getPullRequestTreeData.ts @@ -0,0 +1,128 @@ +import { formatHash } from 'utils/general' +import * as API from './API' +import { isInPullFilesPage } from './DOMHelper' +import { processTree } from './index' +import { getCommentsMap } from './utils' + +export async function getPullRequestTreeData( + metaData: Pick, + pullId: string, + accessToken?: string, + useSafeRequest = true, +) { + const { userName, repoName } = metaData + const [treeData, commentData] = await Promise.all([ + useSafeRequest + ? safeGetPullRequestTreeData(metaData, pullId, accessToken) + : fastGetPullRequestTreeData(metaData, pullId, accessToken), + API.getPullComments(userName, repoName, pullId, accessToken), + ]) + + const docs = await API.getPullPageDocuments(userName, repoName, pullId, isInPullFilesPage() ? document : undefined) + // query all elements at once to make getFileElementHash run faster + const elementsHavePath = docs.map(doc => doc.querySelectorAll(`[data-path]`)) + const getFileElementHash = (path: string) => { + let e + for (const group of elementsHavePath) { + for (let i = 0; i < group.length; i++) { + const element = group[i] + if (element.getAttribute('data-path')?.startsWith(path)) { + e = element + break + } + } + if (e) break + } + return e?.parentElement?.id + } + + const urlMainPart = `https://${window.location.host}/${userName}/${repoName}/pull/${pullId}/files${window.location.search}` + const commentsMap = getCommentsMap(commentData) + const nodes: TreeNode[] = treeData.map( + ({ filename, sha, additions, deletions, changes, status }) => ({ + path: filename || '', + type: 'blob', + name: filename?.split('/').pop() || '', + url: `${urlMainPart}${formatHash(getFileElementHash(filename))}`, + sha: sha, + comments: commentsMap.get(filename), + diff: { + status, + additions, + deletions, + changes, + }, + }), + ) + + const root = processTree(nodes) + return { root } +} + +const GITHUB_API_RESPONSE_LENGTH_LIMIT = 3000 +const GITHUB_API_RESPONSE_MAX_SIZE_PER_PAGE = 100 +const MAX_PAGE = Math.ceil(GITHUB_API_RESPONSE_LENGTH_LIMIT / GITHUB_API_RESPONSE_MAX_SIZE_PER_PAGE) + +async function safeGetPullRequestTreeData( + { userName, repoName }: Pick, + pullId: string, + accessToken?: string, +) { + return ( + await API.getPaginatedData(page => + API.requestPullTreeData( + userName, + repoName, + pullId, + page, + GITHUB_API_RESPONSE_MAX_SIZE_PER_PAGE, + accessToken, + ), + ) + ).flat() +} + +async function fastGetPullRequestTreeData( + { userName, repoName }: Pick, + pullId: string, + accessToken?: string, +) { + const [pullData, treeData] = await Promise.all([ + API.getPullData(userName, repoName, pullId, accessToken), + API.getPullTreeData( + userName, + repoName, + pullId, + 1, + GITHUB_API_RESPONSE_MAX_SIZE_PER_PAGE, + accessToken, + ), + ]) + + const count = pullData.changed_files + if (treeData.length < count) { + let page = 1 + const restPages = [] + while (page * GITHUB_API_RESPONSE_MAX_SIZE_PER_PAGE < count) { + restPages.push(++page) + } + if (page > MAX_PAGE) { + // TODO: hint + } + const moreFiles = await Promise.all( + restPages.map(page => + API.getPullTreeData( + userName, + repoName, + pullId, + page, + GITHUB_API_RESPONSE_MAX_SIZE_PER_PAGE, + accessToken, + ), + ), + ) + treeData.push(...moreFiles.flat()) + } + + return treeData +} diff --git a/src/platforms/GitHub/index.ts b/src/platforms/GitHub/index.ts index 7b8ac61..99a05a5 100644 --- a/src/platforms/GitHub/index.ts +++ b/src/platforms/GitHub/index.ts @@ -1,21 +1,19 @@ import { useConfigs } from 'containers/ConfigsContext' import { GITHUB_OAUTH } from 'env' import { Base64 } from 'js-base64' -import { formatID } from 'utils/DOMHelper' -import { formatHash, run } from 'utils/general' import { resolveGitModules } from 'utils/gitSubmodule' import { sortFoldersToFront } from 'utils/treeParser' import * as API from './API' import * as DOMHelper from './DOMHelper' +import { getCommitTreeData } from './getCommitTreeData' +import { getPullRequestTreeData } from './getPullRequestTreeData' import { useEnterpriseStatBarStyleFix } from './hooks/useEnterpriseStatBarStyleFix' import { useGitHubAttachCopyFileButton } from './hooks/useGitHubAttachCopyFileButton' import { useGitHubAttachCopySnippetButton } from './hooks/useGitHubAttachCopySnippetButton' import { useGitHubCodeFold } from './hooks/useGitHubCodeFold' import * as URLHelper from './URLHelper' -import { resolveHeaderLink } from './utils' -export { useGitHubCodeFold } from './hooks/useGitHubCodeFold' -function processTree(tree: TreeNode[]): TreeNode { +export function processTree(tree: TreeNode[]): TreeNode { // nodes are created from items and put onto tree const pathToItem = new Map() tree.forEach(item => pathToItem.set(item.path, item)) @@ -86,27 +84,6 @@ export function isEnterprise() { return !window.location.host.endsWith('github.com') } -function resolvePageScope(defaultBranchName?: string) { - const parsed = URLHelper.parse() - switch (parsed.type) { - case 'blob': - case 'tree': { - // handle URLs like {user}/{repo}/{'tree'|'blob'}/{sha|branch}, issue #131 - const branchName = DOMHelper.getCurrentBranch() - if (branchName && branchName !== defaultBranchName) return `branch-${branchName}` - break - } - case 'tags': - return 'tags' - case 'releases': - return 'releases' - case 'pull': - const pullId = URLHelper.isInPullPage() - if (pullId) return `pull-${pullId}` - } - return 'general' -} - const pathSHAMap = new Map() // Try lookup PJAX containers, #js-repo-pjax-container could exist while #repo-content-pjax-container does not. @@ -169,17 +146,14 @@ export const GitHub: Platform = { branchUrl, } }, - async getTreeData(metaData, path = '/', recursive, accessToken) { + getTreeData(metaData, path = '/', recursive, accessToken) { const pullId = URLHelper.isInPullPage() - if (pullId) { - return await getPullRequestTreeData(metaData, pullId, accessToken) - } - const commitId = URLHelper.isInCommitPage() - if (commitId) { - return await getCommitTreeData(metaData, commitId, accessToken) - } + if (pullId) return getPullRequestTreeData(metaData, pullId, accessToken) - return await getRepositoryTreeData(metaData, path, recursive, accessToken) + const commitId = URLHelper.isInCommitPage() + if (commitId) return getCommitTreeData(metaData, commitId, accessToken) + + return getRepositoryTreeData(metaData, path, recursive, accessToken) }, shouldShow() { return Boolean( @@ -295,151 +269,3 @@ async function getRepositoryTreeData( return { root, defer: treeData.truncated } } - -async function getCommitTreeData( - { userName, repoName }: Pick, - commitSHA: string, - accessToken: string | undefined, -) { - const treeData: GitHubAPI.CommitTreeItem[] = [] - let page = 1 - while (true) { - const response = await API.getCommitTreeData(userName, repoName, commitSHA, page, accessToken) - const { files } = (await response.json()) as GitHubAPI.CommitResponseData - treeData.push(...files) - - const headerLink = response.headers.get('link') - if (headerLink) { - const rels = resolveHeaderLink(headerLink) - if (rels) { - if (rels.position === 'first') { - page++ - } else if (rels.position === 'middle') { - const searchOfLast = new URL(rels.last).searchParams - if (`${page}` === searchOfLast.get('page')) { - // this should not actually happen because GitHub responds `prev` and `first` for the first page - break - } - page++ - } else { - // i.e. rels.position === 'last' - break - } - } else { - // unexpected link header content - break - } - } else { - // no link headers if there is <100 files - break - } - } - - const documents = await API.getCommitPageDocuments(userName, repoName, commitSHA) - - const getItemURL = (path: string) => { - for (const doc of documents) { - const id = doc.querySelector(`[data-path="${path}"]`)?.parentElement?.id - if (id) return formatID(id) - } - } - - const root = processTree( - treeData.map(item => ({ - type: 'blob', - path: item.filename, - name: item.filename.split('/').pop() || '', - url: getItemURL(item.filename) || item.blob_url, - sha: item.patch, - diff: { - status: item.status, - additions: item.additions, - deletions: item.deletions, - changes: item.changes, - }, - })), - ) - - return { root } -} - -async function getPullRequestTreeData( - { userName, repoName }: Pick, - pullId: string, - accessToken?: string, -) { - // https://developer.github.com/v3/pulls/#list-pull-requests-files - const GITHUB_API_RESPONSE_LENGTH_LIMIT = 3000 - const GITHUB_API_PAGED_RESPONSE_LENGTH_LIMIT = 30 - const MAX_PAGE = Math.ceil( - GITHUB_API_RESPONSE_LENGTH_LIMIT / GITHUB_API_PAGED_RESPONSE_LENGTH_LIMIT, - ) - let page = 1 - const [pullData, treeData, commentData] = await Promise.all([ - API.getPullData(userName, repoName, pullId, accessToken), - API.getPullTreeData(userName, repoName, pullId, page, accessToken), - API.getPullComments(userName, repoName, pullId, accessToken), - ]) - - const count = pullData.changed_files - if (treeData.length < count) { - const restPages = [] - while (page * GITHUB_API_PAGED_RESPONSE_LENGTH_LIMIT < count) { - restPages.push(++page) - } - if (page > MAX_PAGE) { - // TODO: hint - } - const moreFiles = await Promise.all( - restPages.map(page => API.getPullTreeData(userName, repoName, pullId, page, accessToken)), - ) - treeData.push(...([] as GitHubAPI.PullTreeData).concat(...moreFiles)) - } - - const docs = await API.getPullPageDocuments(userName, repoName, pullId) - // query all elements at once to make getFileElementHash run faster - const elementsHavePath = docs.map(doc => doc.querySelectorAll(`[data-path]`)) - const getFileElementHash = (path: string) => { - let e - for (const group of elementsHavePath) { - for (let i = 0; i < group.length; i++) { - const element = group[i] - if (element.getAttribute('data-path')?.startsWith(path)) { - e = element - break - } - } - if (e) break - } - return e?.parentElement?.id - } - - const urlMainPart = `https://${window.location.host}/${userName}/${repoName}/pull/${pullId}/files${window.location.search}` - const nodes: TreeNode[] = treeData.map( - ({ filename, sha, additions, deletions, changes, status }) => ({ - path: filename || '', - type: 'blob', - name: filename?.split('/').pop() || '', - url: `${urlMainPart}${formatHash(getFileElementHash(filename))}`, - sha: sha, - comments: run(() => { - const comments = commentData?.filter(comment => filename === comment.path) - if (comments?.length) - return { - active: comments.filter(comment => comment.position !== null).length, - resolved: comments.filter(comment => comment.position === null).length, - } - }), - diff: { - status, - additions, - deletions, - changes, - }, - }), - ) - - const root = processTree(nodes) - return { root } -} - diff --git a/src/platforms/GitHub/utils.ts b/src/platforms/GitHub/utils.ts index d112f82..bac1be7 100644 --- a/src/platforms/GitHub/utils.ts +++ b/src/platforms/GitHub/utils.ts @@ -73,3 +73,54 @@ export function resolveHeaderLink(raw: string) { return } } + +export async function getDOM(url: string) { + return new DOMParser().parseFromString(await (await fetch(url)).text(), 'text/html') +} + +export async function continuousLoadPages(doc: Document, onReceivePage?: (doc: Document) => void) { + /** + * + */ + const fragmentSelector = 'include-fragment[data-targets="diff-file-filter.progressiveLoaders"]' + const documents: Document[] = [doc] + while (true) { + const fragment = doc.querySelector(fragmentSelector) as HTMLElement + if (!fragment) break + const src = fragment.getAttribute('src') + if (!src) break + doc = await getDOM(src) + documents.push(doc) + onReceivePage?.(doc) + } + return documents +} + +export function getCommentsMap(commentData: GitHubAPI.PullComments) { + const commentsMap = new Map< + string, + { + active: number + resolved: number + } + >() + commentData.forEach(comment => { + let stat = commentsMap.get(comment.path) + if (!stat) { + stat = { + active: 0, + resolved: 0, + } + commentsMap.set(comment.path, stat) + } + + if (comment.position === null) stat.active++ + else stat.resolved++ + }) + return commentsMap +} diff --git a/tsconfig.json b/tsconfig.json index 58cdfac..7f2ad7b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,7 @@ "target": "ESNext", "jsx": "react", "strict": true, - "lib": ["dom", "es2017.object", "es2016", "ES2020.String"], + "lib": ["dom", "es2017.object", "es2016", "ES2019.Array", "ES2020.String"], "baseUrl": "src", "resolveJsonModule": true, "allowSyntheticDefaultImports": true,