fix(site): Upload template files on template version editor (#6222)

site/jest.setup.ts

@@ -3,6 +3,14 @@ import { cleanup } from "@testing-library/react"
 import crypto from "crypto"
 import { server } from "./src/testHelpers/server"
 import "jest-location-mock"
+import { TextEncoder, TextDecoder } from "util"
+import { Blob } from "buffer"
+
+global.TextEncoder = TextEncoder
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Polyfill for jsdom
+global.TextDecoder = TextDecoder as any
+// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Polyfill for jsdom
+global.Blob = Blob as any
 
 // Polyfill the getRandomValues that is used on utils/random.ts
 Object.defineProperty(global.self, "crypto", {

site/js-untar.d.ts

@@ -1,14 +1,21 @@
 declare module "js-untar" {
-  interface File {
+  export interface File {
     name: string
+    mode: string
     blob: Blob
+    gid: number
+    uid: number
+    mtime: number
+    gname: string
+    uname: string
+    type: "0" | "1" | "2" | "3" | "4" | "5" //https://en.wikipedia.org/wiki/Tar_(computing) on Type flag field
   }
 
   const Untar: (buffer: ArrayBuffer) => {
     then: (
-      resolve?: () => Promise<void>,
+      resolve?: (files: File[]) => void,
       reject?: () => Promise<void>,
-      progress: (file: File) => Promise<void>,
+      progress?: (file: File) => Promise<void>,
     ) => Promise<void>
   }
 

site/package.json

@@ -76,7 +76,6 @@
     "remark-gfm": "3.0.1",
     "rollup-plugin-visualizer": "5.9.0",
     "sourcemapped-stacktrace": "1.1.11",
-    "tar-js": "^0.3.0",
     "ts-prune": "0.10.3",
     "tzdata": "1.0.30",
     "ua-parser-js": "1.0.33",

site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/TemplateVersionEditorPage.tsx

@@ -16,14 +16,21 @@ type Params = {
 export const TemplateVersionEditorPage: FC = () => {
   const { version: versionName, template: templateName } = useParams() as Params
   const orgId = useOrganizationId()
-  const { isSuccess, data } = useTemplateVersionData(
-    orgId,
-    templateName,
-    versionName,
-  )
   const [editorState, sendEvent] = useMachine(templateVersionEditorMachine, {
     context: { orgId },
   })
+  const { isSuccess, data } = useTemplateVersionData(
+    {
+      orgId,
+      templateName,
+      versionName,
+    },
+    {
+      onSuccess(data) {
+        sendEvent({ type: "INITIALIZE", untarFiles: data.untarFiles })
+      },
+    },
+  )
 
   return (
     <>
@@ -34,7 +41,7 @@ export const TemplateVersionEditorPage: FC = () => {
       {isSuccess && (
         <TemplateVersionEditor
           template={data.template}
-          templateVersion={editorState.context.version || data.currentVersion}
+          templateVersion={editorState.context.version || data.version}
           defaultFileTree={data.fileTree}
           onPreview={(fileTree) => {
             sendEvent({

site/src/pages/TemplateVersionPage/TemplateVersionEditorPage/data.ts

@@ -1,32 +1,49 @@
-import { useQuery } from "@tanstack/react-query"
-import { getTemplateByName, getTemplateVersionByName } from "api/api"
+import { useQuery, UseQueryOptions } from "@tanstack/react-query"
+import { getFile, getTemplateByName, getTemplateVersionByName } from "api/api"
 import { createTemplateVersionFileTree } from "util/templateVersion"
+import untar, { File as UntarFile } from "js-untar"
 
 const getTemplateVersionData = async (
   orgId: string,
   templateName: string,
   versionName: string,
 ) => {
-  const [template, currentVersion] = await Promise.all([
+  const [template, version] = await Promise.all([
     getTemplateByName(orgId, templateName),
     getTemplateVersionByName(orgId, templateName, versionName),
   ])
-  const fileTree = await createTemplateVersionFileTree(currentVersion)
+  const tarFile = await getFile(version.job.file_id)
+  let untarFiles: UntarFile[] = []
+  await untar(tarFile).then((files) => {
+    untarFiles = files
+  })
+  const fileTree = await createTemplateVersionFileTree(untarFiles)
 
   return {
     template,
-    currentVersion,
+    version,
     fileTree,
+    untarFiles,
   }
 }
 
+type GetTemplateVersionResponse = Awaited<
+  ReturnType<typeof getTemplateVersionData>
+>
+
+type UseTemplateVersionDataParams = {
+  orgId: string
+  templateName: string
+  versionName: string
+}
+
 export const useTemplateVersionData = (
-  orgId: string,
-  templateName: string,
-  versionName: string,
+  { templateName, versionName, orgId }: UseTemplateVersionDataParams,
+  options?: UseQueryOptions<GetTemplateVersionResponse>,
 ) => {
   return useQuery({
     queryKey: ["templateVersion", templateName, versionName],
     queryFn: () => getTemplateVersionData(orgId, templateName, versionName),
+    ...options,
   })
 }

site/src/util/tar.test.ts

@@ -0,0 +1,52 @@
+import { TarReader, TarWriter, ITarFileInfo, TarFileType } from "./tar"
+
+const mtime = 1666666666666
+
+test("tar", async () => {
+  // Write
+  const writer = new TarWriter()
+  writer.addFile("a.txt", "hello", { mtime })
+  writer.addFile("b.txt", new Blob(["world"]), { mtime })
+  writer.addFile("c.txt", "", { mtime })
+  writer.addFolder("etc", { mtime })
+  writer.addFile("etc/d.txt", "", { mtime })
+  const blob = await writer.write()
+
+  // Read
+  const reader = new TarReader()
+  const fileInfos = await reader.readFile(blob)
+  verifyFile(fileInfos[0], reader.getTextFile(fileInfos[0].name) as string, {
+    name: "a.txt",
+    content: "hello",
+  })
+  verifyFile(fileInfos[1], reader.getTextFile(fileInfos[1].name) as string, {
+    name: "b.txt",
+    content: "world",
+  })
+  verifyFile(fileInfos[2], reader.getTextFile(fileInfos[2].name) as string, {
+    name: "c.txt",
+    content: "",
+  })
+  verifyFolder(fileInfos[3], {
+    name: "etc",
+  })
+  verifyFile(fileInfos[4], reader.getTextFile(fileInfos[4].name) as string, {
+    name: "etc/d.txt",
+    content: "",
+  })
+})
+
+function verifyFile(
+  info: ITarFileInfo,
+  content: string,
+  expected: { name: string; content: string },
+) {
+  expect(info.name).toEqual(expected.name)
+  expect(info.size).toEqual(expected.content.length)
+  expect(content).toEqual(expected.content)
+}
+
+function verifyFolder(info: ITarFileInfo, expected: { name: string }) {
+  expect(info.name).toEqual(expected.name)
+  expect(info.type).toEqual(TarFileType.Dir)
+}

site/src/util/tar.ts

@@ -0,0 +1,321 @@
+// Based on https://github.com/gera2ld/tarjs
+// and https://github.com/ankitrohatgi/tarballjs/blob/master/tarball.js
+export enum TarFileType {
+  File = "0",
+  Dir = "5",
+}
+const encoder = new TextEncoder()
+const utf8Encode = (input: string) => encoder.encode(input)
+const decoder = new TextDecoder()
+const utf8Decode = (input: Uint8Array) => decoder.decode(input)
+
+export interface ITarFileInfo {
+  name: string
+  type: TarFileType
+  size: number
+  headerOffset: number
+}
+
+export interface ITarWriteItem {
+  name: string
+  type: TarFileType
+  data: ArrayBuffer | Promise<ArrayBuffer> | null
+  size: number
+  opts?: Partial<ITarWriteOptions>
+}
+
+export interface ITarWriteOptions {
+  uid: number
+  gid: number
+  mode: number
+  mtime: number
+  user: string
+  group: string
+}
+
+export class TarReader {
+  private fileInfo: ITarFileInfo[] = []
+  private _buffer: ArrayBuffer | null = null
+
+  constructor() {
+    this.reset()
+  }
+
+  get buffer() {
+    if (!this._buffer) {
+      throw new Error("Buffer is not set")
+    }
+
+    return this._buffer
+  }
+
+  reset() {
+    this.fileInfo = []
+    this._buffer = null
+  }
+
+  async readFile(file: ArrayBuffer | Uint8Array | Blob) {
+    this.reset()
+    this._buffer = await getArrayBuffer(file)
+    this.readFileInfo()
+    return this.fileInfo
+  }
+
+  private readFileInfo() {
+    this.fileInfo = []
+    let offset = 0
+    let fileSize = 0
+    let fileName = ""
+    let fileType: TarFileType
+    while (offset < this.buffer.byteLength - 512) {
+      fileName = this.readFileName(offset)
+      if (!fileName) {
+        break
+      }
+      fileType = this.readFileType(offset)
+      fileSize = this.readFileSize(offset)
+
+      this.fileInfo.push({
+        name: fileName,
+        type: fileType,
+        size: fileSize,
+        headerOffset: offset,
+      })
+
+      offset += 512 + 512 * Math.floor((fileSize + 511) / 512)
+    }
+  }
+
+  private readString(offset: number, maxSize: number) {
+    let size = 0
+    let view = new Uint8Array(this.buffer, offset, maxSize)
+    while (size < maxSize && view[size]) {
+      size += 1
+    }
+    view = new Uint8Array(this.buffer, offset, size)
+    return utf8Decode(view)
+  }
+
+  private readFileName(offset: number) {
+    return this.readString(offset, 100)
+  }
+
+  private readFileType(offset: number) {
+    const typeView = new Uint8Array(this.buffer, offset + 156, 1)
+    const typeStr = String.fromCharCode(typeView[0])
+    if (typeStr === "0") {
+      return TarFileType.File
+    } else if (typeStr === "5") {
+      return TarFileType.Dir
+    } else {
+      throw new Error("No supported file type")
+    }
+  }
+
+  private readFileSize(offset: number) {
+    // offset = 124, length = 12
+    const view = new Uint8Array(this.buffer, offset + 124, 12)
+    const sizeStr = utf8Decode(view)
+    return parseInt(sizeStr, 8)
+  }
+
+  private readFileBlob(offset: number, size: number, mimetype: string) {
+    const view = new Uint8Array(this.buffer, offset, size)
+    return new Blob([view], { type: mimetype })
+  }
+
+  private readTextFile(offset: number, size: number) {
+    const view = new Uint8Array(this.buffer, offset, size)
+    return utf8Decode(view)
+  }
+
+  getTextFile(filename: string) {
+    const item = this.fileInfo.find((info) => info.name === filename)
+    if (item) {
+      return this.readTextFile(item.headerOffset + 512, item.size)
+    }
+  }
+
+  getFileBlob(filename: string, mimetype = "") {
+    const item = this.fileInfo.find((info) => info.name === filename)
+    if (item) {
+      return this.readFileBlob(item.headerOffset + 512, item.size, mimetype)
+    }
+  }
+}
+
+export class TarWriter {
+  private fileData: ITarWriteItem[] = []
+  private _buffer: ArrayBuffer | null = null
+
+  get buffer() {
+    if (!this._buffer) {
+      throw new Error("Buffer is not set")
+    }
+    return this._buffer
+  }
+
+  addFile(
+    name: string,
+    file: string | ArrayBuffer | Uint8Array | Blob,
+    opts?: Partial<ITarWriteOptions>,
+  ) {
+    const data = getArrayBuffer(file)
+    const size = (data as ArrayBuffer).byteLength ?? (file as Blob).size
+    const item: ITarWriteItem = {
+      name,
+      type: TarFileType.File,
+      data,
+      size,
+      opts,
+    }
+    this.fileData.push(item)
+  }
+
+  addFolder(name: string, opts?: Partial<ITarWriteOptions>) {
+    this.fileData.push({
+      name,
+      type: TarFileType.Dir,
+      data: null,
+      size: 0,
+      opts,
+    })
+  }
+
+  private createBuffer() {
+    const dataSize = this.fileData.reduce(
+      (prev, item) => prev + 512 + 512 * Math.floor((item.size + 511) / 512),
+      0,
+    )
+    const bufSize = 10240 * Math.floor((dataSize + 10240 - 1) / 10240)
+    this._buffer = new ArrayBuffer(bufSize)
+  }
+
+  async write() {
+    this.createBuffer()
+    const view = new Uint8Array(this.buffer)
+    let offset = 0
+    for (const item of this.fileData) {
+      // write header
+      this.writeFileName(item.name, offset)
+      this.writeFileType(item.type, offset)
+      this.writeFileSize(item.size, offset)
+      this.fillHeader(offset, item.opts as Partial<ITarWriteOptions>, item.type)
+      this.writeChecksum(offset)
+
+      // write data
+      const data = new Uint8Array((await item.data) as ArrayBuffer)
+      view.set(data, offset + 512)
+      offset += 512 + 512 * Math.floor((item.size + 511) / 512)
+    }
+    return new Blob([this.buffer], { type: "application/x-tar" })
+  }
+
+  private writeString(str: string, offset: number, size: number) {
+    const strView = utf8Encode(str)
+    const view = new Uint8Array(this.buffer, offset, size)
+    for (let i = 0; i < size; i += 1) {
+      view[i] = i < strView.length ? strView[i] : 0
+    }
+  }
+
+  private writeFileName(name: string, offset: number) {
+    // offset: 0
+    this.writeString(name, offset, 100)
+  }
+
+  private writeFileType(type: TarFileType, offset: number) {
+    // offset: 156
+    const typeView = new Uint8Array(this.buffer, offset + 156, 1)
+    typeView[0] = type.charCodeAt(0)
+  }
+
+  private writeFileSize(size: number, offset: number) {
+    // offset: 124
+    const sizeStr = size.toString(8).padStart(11, "0")
+    this.writeString(sizeStr, offset + 124, 12)
+  }
+
+  private writeFileMode(mode: number, offset: number) {
+    // offset: 100
+    this.writeString(mode.toString(8).padStart(7, "0"), offset + 100, 8)
+  }
+
+  private writeFileUid(uid: number, offset: number) {
+    // offset: 108
+    this.writeString(uid.toString(8).padStart(7, "0"), offset + 108, 8)
+  }
+
+  private writeFileGid(gid: number, offset: number) {
+    // offset: 116
+    this.writeString(gid.toString(8).padStart(7, "0"), offset + 116, 8)
+  }
+
+  private writeFileMtime(mtime: number, offset: number) {
+    // offset: 136
+    this.writeString(mtime.toString(8).padStart(11, "0"), offset + 136, 12)
+  }
+
+  private writeFileUser(user: string, offset: number) {
+    // offset: 265
+    this.writeString(user, offset + 265, 32)
+  }
+
+  private writeFileGroup(group: string, offset: number) {
+    // offset: 297
+    this.writeString(group, offset + 297, 32)
+  }
+
+  private writeChecksum(offset: number) {
+    // offset: 148
+    this.writeString("        ", offset + 148, 8) // first fill with spaces
+
+    // add up header bytes
+    const header = new Uint8Array(this.buffer, offset, 512)
+    let chksum = 0
+    for (let i = 0; i < 512; i += 1) {
+      chksum += header[i]
+    }
+    this.writeString(chksum.toString(8), offset + 148, 8)
+  }
+
+  private fillHeader(
+    offset: number,
+    opts: Partial<ITarWriteOptions>,
+    fileType: TarFileType,
+  ) {
+    const { uid, gid, mode, mtime, user, group } = {
+      uid: 1000,
+      gid: 1000,
+      mode: fileType === TarFileType.File ? 0o664 : 0o775,
+      mtime: ~~(Date.now() / 1000),
+      user: "tarballjs",
+      group: "tarballjs",
+      ...opts,
+    }
+
+    this.writeFileMode(mode, offset)
+    this.writeFileUid(uid, offset)
+    this.writeFileGid(gid, offset)
+    this.writeFileMtime(mtime, offset)
+
+    this.writeString("ustar", offset + 257, 6) // magic string
+    this.writeString("00", offset + 263, 2) // magic version
+
+    this.writeFileUser(user, offset)
+    this.writeFileGroup(group, offset)
+  }
+}
+
+function getArrayBuffer(file: string | ArrayBuffer | Uint8Array | Blob) {
+  if (typeof file === "string") {
+    return utf8Encode(file).buffer
+  }
+  if (file instanceof ArrayBuffer) {
+    return file
+  }
+  if (ArrayBuffer.isView(file)) {
+    return new Uint8Array(file).buffer
+  }
+  return file.arrayBuffer()
+}

site/src/util/templateVersion.ts

@@ -1,6 +1,6 @@
 import * as API from "api/api"
 import { TemplateVersion } from "api/typesGenerated"
-import untar from "js-untar"
+import untar, { File as UntarFile } from "js-untar"
 import { FileTree, setFile } from "./filetree"
 
 /**
@@ -41,21 +41,22 @@ export const getTemplateVersionFiles = async (
 
 const allowedExtensions = ["tf", "md", "Dockerfile"]
 
+export const isAllowedFile = (name: string) => {
+  return allowedExtensions.some((ext) => name.endsWith(ext))
+}
+
 export const createTemplateVersionFileTree = async (
-  version: TemplateVersion,
+  untarFiles: UntarFile[],
 ): Promise<FileTree> => {
   let fileTree: FileTree = {}
-  const tarFile = await API.getFile(version.job.file_id)
   const blobs: Record<string, Blob> = {}
 
-  await untar(tarFile).then(undefined, undefined, async (file) => {
-    if (allowedExtensions.some((ext) => file.name.endsWith(ext))) {
-      blobs[file.name] = file.blob
+  for (const untarFile of untarFiles) {
+    if (isAllowedFile(untarFile.name)) {
+      blobs[untarFile.name] = untarFile.blob
     }
-  })
+  }
 
-  // We don't want to get the blob text during untar to not block the main thread.
-  // Also, by doing it here, we can make all the loading in parallel.
   await Promise.all(
     Object.entries(blobs).map(async ([fullPath, blob]) => {
       const content = await blob.text()

site/src/xServices/templateVersionEditor/templateVersionEditorXService.ts

@@ -7,8 +7,10 @@ import {
 } from "api/typesGenerated"
 import { assign, createMachine } from "xstate"
 import * as API from "api/api"
-import Tar from "tar-js"
+import { File as UntarFile } from "js-untar"
 import { FileTree, traverse } from "util/filetree"
+import { isAllowedFile } from "util/templateVersion"
+import { TarWriter } from "util/tar"
 
 export interface CreateVersionData {
   file: File
@@ -22,6 +24,7 @@ export interface TemplateVersionEditorMachineContext {
   version?: TemplateVersion
   resources?: WorkspaceResource[]
   buildLogs?: ProvisionerJobLog[]
+  untarFiles?: UntarFile[]
 }
 
 export const templateVersionEditorMachine = createMachine(
@@ -31,6 +34,7 @@ export const templateVersionEditorMachine = createMachine(
     schema: {
       context: {} as TemplateVersionEditorMachineContext,
       events: {} as
+        | { type: "INITIALIZE"; untarFiles: UntarFile[] }
         | {
             type: "CREATE_VERSION"
             fileTree: FileTree
@@ -61,8 +65,16 @@ export const templateVersionEditorMachine = createMachine(
       },
     },
     tsTypes: {} as import("./templateVersionEditorXService.typegen").Typegen0,
-    initial: "idle",
+    initial: "initializing",
     states: {
+      initializing: {
+        on: {
+          INITIALIZE: {
+            actions: ["assignUntarFiles"],
+            target: "idle",
+          },
+        },
+      },
       idle: {
         on: {
           CREATE_VERSION: {
@@ -201,20 +213,51 @@ export const templateVersionEditorMachine = createMachine(
           }
         },
       }),
+      assignUntarFiles: assign({
+        untarFiles: (_, { untarFiles }) => untarFiles,
+      }),
     },
     services: {
-      uploadTar: (ctx) => {
-        if (!ctx.fileTree) {
-          throw new Error("files must be set")
+      uploadTar: async ({ fileTree, untarFiles }) => {
+        if (!fileTree) {
+          throw new Error("file tree must to be set")
         }
-        const tar = new Tar()
-        let out: Uint8Array = new Uint8Array()
-        traverse(ctx.fileTree, (content, _filename, fullPath) => {
+        if (!untarFiles) {
+          throw new Error("untar files must to be set")
+        }
+        const tar = new TarWriter()
+
+        // Add previous non editable files
+        for (const untarFile of untarFiles) {
+          if (!isAllowedFile(untarFile.name)) {
+            if (untarFile.type === "5") {
+              tar.addFolder(untarFile.name, {
+                mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42
+                mtime: untarFile.mtime,
+                user: untarFile.uname,
+                group: untarFile.gname,
+              })
+            } else {
+              const buffer = await untarFile.blob.arrayBuffer()
+              tar.addFile(untarFile.name, new Uint8Array(buffer), {
+                mode: parseInt(untarFile.mode, 8) & 0xfff, // https://github.com/beatgammit/tar-js/blob/master/lib/tar.js#L42
+                mtime: untarFile.mtime,
+                user: untarFile.uname,
+                group: untarFile.gname,
+              })
+            }
+          }
+        }
+        // Add the editable files
+        traverse(fileTree, (content, _filename, fullPath) => {
           if (typeof content === "string") {
-            out = tar.append(fullPath, content)
+            tar.addFile(fullPath, content)
+          } else {
+            tar.addFolder(fullPath)
           }
         })
-        return API.uploadTemplateFile(new File([out], "template.tar"))
+        const blob = await tar.write()
+        return API.uploadTemplateFile(new File([blob], "template.tar"))
       },
       createBuild: (ctx) => {
         if (!ctx.uploadResponse) {

site/yarn.lock

@@ -13678,11 +13678,6 @@ tapable@^2.1.1, tapable@^2.2.0:
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0"
   integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==
 
-tar-js@^0.3.0:
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/tar-js/-/tar-js-0.3.0.tgz#6949aabfb0ba18bb1562ae51a439fd0f30183a17"
-  integrity sha512-9uqP2hJUZNKRkwPDe5nXxXdzo6w+BFBPq9x/tyi5/U/DneuSesO/HMb0y5TeWpfcv49YDJTs7SrrZeeu8ZHWDA==
-
 tar-stream@^2.0.1:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"