build: automated scripts for updating icons

Makefile

@@ -4,6 +4,11 @@ FULL_VERSION=v$(RAW_VERSION)
 pull-icons:
 	git clone git@github.com:vscode-icons/vscode-icons.git vscode-icons --depth=1
 
+update-icons:
+	cd vscode-icons && git pull
+	node scripts/resolve-languages-map
+	node scripts/generate-icon-index
+
 build:
 	rm -rf dist
 	yarn build

scripts/check-emit-dir.js

@@ -0,0 +1,14 @@
+const path = require('path')
+const fs = require('fs').promises
+
+const emitDirPath = path.resolve(__dirname, 'tmp')
+exports.emitDirPath = emitDirPath
+
+async function checkEmitDir() {
+  try {
+    await fs.mkdir(emitDirPath)
+  } catch (err) {
+    await fs.stat(emitDirPath)
+  }
+}
+exports.checkEmitDir = checkEmitDir

scripts/generate-file-icon-index.js

@@ -1,18 +1,21 @@
-const languageIds = require('./language-id-ext.json')
+const { languages } = require('./tmp/languages')
+const fileName = 'file-icons-index'
 
-function generateCSV() {
+const link = 'https://github.com/vscode-icons/vscode-icons/wiki/ListOfFiles'
+
+function parsePageContent() {
   const records = []
-
   document.body
     .querySelector('table')
     .querySelectorAll('tbody tr')
     .forEach(tr => {
       const [name, id, dark, light] = Array.from(tr.querySelectorAll('td'))
       const exts = []
+      const ids = []
       const names = []
-      id.innerHTML
-        .replace(/<sub>|<\/sub>/g, '')
-        .split(', ')
+
+      id.querySelector('sub')
+        .innerHTML.split(', ')
         .map(part => {
           const tags = part.match(/<(\w+)>(.*?)<\/\1>/g)
           if (tags) {
@@ -21,51 +24,50 @@ function generateCSV() {
               if (match) {
                 const [, tag, content] = match
                 if (tag === 'strong') {
-                  // filenames in bold
-                  names.push(content)
+                  // filenames
+                  names.push(content.toLowerCase())
                 } else if (tag === 'code') {
-                  // language ids in code block
-                  const map = Object.values(languageIds).find(({ ids }) =>
-                    (Array.isArray(ids) ? ids : [ids]).includes(content),
-                  )
-                  if (map && map.exts) exts.push(map.exts)
+                  // language ids
+                  ids.push(content)
                 } else {
                   console.warn(`Found unrecognized format`, subPart, tag) // unknown
                 }
               }
             })
           } else if (part) {
-            // extensions are in regular fonts
-            exts.push(part.replace(/^\./, ''))
+            // extensions
+            exts.push(part.toLowerCase())
           }
         })
       records.push({
         name: name.innerText,
-        exts,
         names,
+        exts,
+        ids,
         icon: getSrc(dark.querySelector('img')) || getSrc(light.querySelector('img')),
       })
     })
+  return records
 
   function getSrc(img) {
     return img && img.src
   }
-
-  const prepend = 'https://github.com/vscode-icons/vscode-icons/raw/master/icons/file_type_'
-  const append = '.svg?sanitize=true'
-  const separator = ':'
-  const csv = records
-    .map(({ name, names, exts, icon }) =>
-      [
-        name,
-        names.join(separator),
-        exts.join(separator),
-        // icon.replace(prepend, '').replace(append, ''), // assumption: name is equal to this
-      ].join(','),
-    )
-    .join('\n')
-
-  return csv
 }
 
-console.log(generateCSV())
+function prepareCSV({ name, names, exts, ids, icon }) {
+  ids.forEach(content => {
+    const defaultExtension = Object.values(languages)
+      .find(({ ids }) => (Array.isArray(ids) ? ids : [ids]).includes(content))
+      .defaultExtension.toLowerCase()
+    if (!exts.includes(defaultExtension)) exts.push(defaultExtension)
+  })
+  const iconFile = icon.replace(/^.*?file_type_(.*?)\..*$/, '$1')
+  const cols = [name, names.join(':'), exts.join(':')]
+  if (!['file'].includes(name) && name !== iconFile) cols.push(iconFile)
+  return cols
+}
+
+exports.fileName = fileName
+exports.link = link
+exports.parsePageContent = parsePageContent
+exports.prepareCSV = prepareCSV

scripts/generate-folder-icon-index.js

@@ -1,3 +1,7 @@
+const fileName = 'folder-icons-index'
+
+const link = 'https://github.com/vscode-icons/vscode-icons/wiki/ListOfFolders'
+
 function parsePageContent() {
   const records = []
   document.body
@@ -16,26 +20,27 @@ function parsePageContent() {
       })
     })
   return records
+
+  function getSrc(img) {
+    return img && img.src
+  }
 }
 
-function getSrc(img) {
-  return img && img.src
+function prepareCSV({ name, names, icon }) {
+  const [iconFileOpen, iconFileClosed] = [
+    icon.open.replace(/^.*?folder_type_(.*?)_opened\..*$/, '$1'),
+    icon.closed.replace(/^.*?folder_type_(.*?)\..*$/, '$1'),
+  ]
+  const cols = [name, names.join(':')]
+  if (
+    !['folder', 'root_folder'].includes(name) &&
+    (name !== iconFileOpen || iconFileOpen !== iconFileClosed)
+  )
+    cols.push(iconFileOpen, iconFileClosed)
+  return cols
 }
 
-function generateCSV(records) {
-  const prepend = 'https://github.com/vscode-icons/vscode-icons/raw/master/icons/folder_type_'
-  const append = '.svg?sanitize=true'
-  const separator = ':'
-  const csv = records
-    .map(({ name, names, icon }) =>
-      [
-        name,
-        names.join(separator),
-        // icon.replace(prepend, '').replace(append, ''), // assumption: name is equal to this
-      ].join(','),
-    )
-    .join('\n')
-  return csv
-}
-
-console.log(generateCSV(parsePageContent()))
+exports.fileName = fileName
+exports.link = link
+exports.parsePageContent = parsePageContent
+exports.prepareCSV = prepareCSV

scripts/generate-icon-index.js

@@ -0,0 +1,49 @@
+const path = require('path')
+const { promises: fs, existsSync } = require('fs')
+const puppeteer = require('puppeteer')
+const generateFileIconIndex = require('./generate-file-icon-index')
+const generateFolderIconIndex = require('./generate-folder-icon-index')
+const { emitDirPath, checkEmitDir } = require('./check-emit-dir')
+
+let browser
+async function getPage() {
+  const headless = process.env.HEADLESS !== 'false'
+  browser = browser || (await puppeteer.launch({ headless }))
+  return await browser.newPage()
+}
+
+async function generateCSV() {
+  await checkEmitDir()
+
+  await Promise.all(
+    [generateFileIconIndex, generateFolderIconIndex].map(
+      async ({ fileName, link, parsePageContent, prepareCSV }) => {
+        let records
+        const emitJSONPath = path.resolve(emitDirPath, fileName + '.json')
+        if (!existsSync(emitJSONPath)) {
+          const page = await getPage()
+          await page.goto(link)
+          records = await page.evaluate(parsePageContent)
+          await page.close()
+          await fs.writeFile(emitJSONPath, JSON.stringify(records))
+        } else {
+          records = require(emitJSONPath)
+        }
+
+        const rowSeparator = '\n'
+        const columnSeparator = ','
+        const csv = records.map(prepareCSV)
+
+        const emitPath = path.resolve(__dirname, '..', 'src/assets/icons')
+        await fs.writeFile(
+          path.resolve(emitPath, fileName + '.csv'),
+          csv.map(cols => cols.join(columnSeparator)).join(rowSeparator),
+        )
+      },
+    ),
+  )
+
+  if (browser) await browser.close()
+}
+
+generateCSV()

scripts/resolve-languages-map.js

@@ -0,0 +1,32 @@
+const path = require('path')
+const fs = require('fs').promises
+const typescript = require('typescript')
+const { emitDirPath, checkEmitDir } = require('./check-emit-dir')
+
+const files = [path.resolve(__dirname, '..', 'vscode-icons/src/iconsManifest/languages.ts')]
+
+const options = {
+  module: typescript.ModuleKind.CommonJS,
+  target: typescript.ScriptTarget.ES2015,
+  strict: true,
+  suppressOutputPathCheck: false,
+}
+
+async function main() {
+  await checkEmitDir()
+
+  const compilerHost = typescript.createCompilerHost(options)
+  compilerHost.writeFile = async (fileName, data, writeByteOrderMark, onError, sourceFiles) => {
+    if (sourceFiles.some(file => files.includes(file.fileName))) {
+      await fs.writeFile(path.resolve(emitDirPath, path.basename(fileName)), data)
+      console.log(`Emitted`, fileName)
+    } else {
+      console.log(`Skipped`, fileName)
+    }
+  }
+
+  const program = typescript.createProgram(files, options, compilerHost)
+  program.emit()
+}
+
+main()