feat: Import improvements (#3064)

* feat: Split and simplify import/export pages in prep for more options

* minor fixes

* File operations for imports

* test

* icons

app/components/DocumentViews.tsx

@@ -61,6 +61,7 @@ function DocumentViews({ document, isOpen }: Props) {
                 subtitle={subtitle}
                 image={<Avatar key={item.id} src={item.avatarUrl} size={32} />}
                 border={false}
+                compact
                 small
               />
             );

app/components/List/Item.tsx

@@ -10,6 +10,7 @@ type Props = {
   title: React.ReactNode;
   subtitle?: React.ReactNode;
   actions?: React.ReactNode;
+  compact?: boolean;
   border?: boolean;
   small?: boolean;
 };
@@ -49,6 +50,7 @@ const ListItem = (
       <Wrapper
         ref={ref}
         $border={border}
+        $compact={compact}
         activeStyle={{
           background: theme.primary,
         }}
@@ -62,16 +64,16 @@ const ListItem = (
   }
 
   return (
-    <Wrapper $border={border} {...rest}>
+    <Wrapper $compact={compact} $border={border} {...rest}>
       {content(false)}
     </Wrapper>
   );
 };
 
-const Wrapper = styled.div<{ $border?: boolean }>`
+const Wrapper = styled.div<{ $compact?: boolean; $border?: boolean }>`
   display: flex;
-  padding: ${(props) => (props.$border === false ? 0 : "8px 0")};
-  margin: ${(props) => (props.$border === false ? "8px 0" : 0)};
+  margin: ${(props) => (props.$compact === false ? 0 : "8px 0")};
+  padding: ${(props) => (props.$compact === false ? "8px 0" : 0)};
   border-bottom: 1px solid
     ${(props) =>
       props.$border === false ? "transparent" : props.theme.divider};

app/components/Sidebar/Settings.tsx

@@ -1,6 +1,6 @@
 import { observer } from "mobx-react";
 import {
-  DocumentIcon,
+  NewDocumentIcon,
   EmailIcon,
   ProfileIcon,
   PadlockIcon,
@@ -11,6 +11,7 @@ import {
   TeamIcon,
   ExpandedIcon,
   BeakerIcon,
+  DownloadIcon,
 } from "outline-icons";
 import * as React from "react";
 import { useTranslation } from "react-i18next";
@@ -118,11 +119,18 @@ function SettingsSidebar() {
               icon={<LinkIcon color="currentColor" />}
               label={t("Share Links")}
             />
+            {can.manage && (
+              <SidebarLink
+                to="/settings/import"
+                icon={<NewDocumentIcon color="currentColor" />}
+                label={t("Import")}
+              />
+            )}
             {can.export && (
               <SidebarLink
-                to="/settings/import-export"
-                icon={<DocumentIcon color="currentColor" />}
-                label={`${t("Import")} / ${t("Export")}`}
+                to="/settings/export"
+                icon={<DownloadIcon color="currentColor" />}
+                label={t("Export")}
               />
             )}
           </Section>

app/components/SocketProvider.tsx

@@ -328,15 +328,17 @@ class SocketProvider extends React.Component<Props> {
       }
     });
 
+    this.socket.on("fileOperations.create", async (event: any) => {
+      const user = auth.user;
+      if (user) {
+        fileOperations.add({ ...event, user });
+      }
+    });
+
     this.socket.on("fileOperations.update", async (event: any) => {
       const user = auth.user;
-      let collection = null;
-      if (event.collectionId) {
-        collection = await collections.fetch(event.collectionId);
-      }
-
       if (user) {
-        fileOperations.add({ ...event, user, collection });
+        fileOperations.add({ ...event, user });
       }
     });
 

app/components/Spinner.tsx

@@ -0,0 +1,59 @@
+import * as React from "react";
+import styled from "styled-components";
+
+export default function Spinner(props: React.HTMLAttributes<HTMLOrSVGElement>) {
+  return (
+    <SVG
+      width="16px"
+      height="16px"
+      viewBox="0 0 16 16"
+      xmlns="http://www.w3.org/2000/svg"
+      {...props}
+    >
+      <Circle
+        fill="none"
+        strokeWidth="2"
+        strokeLinecap="round"
+        cx="8"
+        cy="8"
+        r="6"
+      ></Circle>
+    </SVG>
+  );
+}
+
+const SVG = styled.svg`
+  @keyframes rotator {
+    0% {
+      transform: rotate(0deg);
+    }
+    100% {
+      transform: rotate(270deg);
+    }
+  }
+
+  animation: rotator 1.4s linear infinite;
+  margin: 4px;
+`;
+
+const Circle = styled.circle`
+  @keyframes dash {
+    0% {
+      stroke-dashoffset: 47;
+    }
+    50% {
+      stroke-dashoffset: 11;
+      transform: rotate(135deg);
+    }
+    100% {
+      stroke-dashoffset: 47;
+      transform: rotate(450deg);
+    }
+  }
+
+  stroke: ${(props) => props.theme.textSecondary};
+  stroke-dasharray: 46;
+  stroke-dashoffset: 0;
+  transform-origin: center;
+  animation: dash 1.4s ease-in-out infinite;
+`;

app/models/FileOperation.ts

@@ -1,6 +1,5 @@
 import { computed } from "mobx";
 import BaseModal from "./BaseModel";
-import Collection from "./Collection";
 import User from "./User";
 
 class FileOperation extends BaseModal {
@@ -8,11 +7,15 @@ class FileOperation extends BaseModal {
 
   state: string;
 
-  collection: Collection | null | undefined;
+  name: string;
+
+  error: string | null;
+
+  collectionId: string | null;
 
   size: number;
 
-  type: string;
+  type: "import" | "export";
 
   user: User;
 

app/routes/settings.tsx

@@ -1,9 +1,10 @@
 import * as React from "react";
 import { Switch, Redirect } from "react-router-dom";
 import Details from "~/scenes/Settings/Details";
+import Export from "~/scenes/Settings/Export";
 import Features from "~/scenes/Settings/Features";
 import Groups from "~/scenes/Settings/Groups";
-import ImportExport from "~/scenes/Settings/ImportExport";
+import Import from "~/scenes/Settings/Import";
 import Notifications from "~/scenes/Settings/Notifications";
 import People from "~/scenes/Settings/People";
 import Profile from "~/scenes/Settings/Profile";
@@ -33,7 +34,11 @@ export default function SettingsRoutes() {
       {isHosted && (
         <Route exact path="/settings/integrations/zapier" component={Zapier} />
       )}
-      <Route exact path="/settings/import-export" component={ImportExport} />
+      <Route exact path="/settings/import" component={Import} />
+      <Route exact path="/settings/export" component={Export} />
+
+      {/* old routes */}
+      <Redirect from="/settings/import-export" to="/settings/export" />
       <Redirect from="/settings/people" to="/settings/members" />
     </Switch>
   );

app/scenes/Settings/Export.tsx

@@ -0,0 +1,105 @@
+import { observer } from "mobx-react";
+import { DownloadIcon } from "outline-icons";
+import * as React from "react";
+import { useTranslation, Trans } from "react-i18next";
+import FileOperation from "~/models/FileOperation";
+import Button from "~/components/Button";
+import Heading from "~/components/Heading";
+import HelpText from "~/components/HelpText";
+import PaginatedList from "~/components/PaginatedList";
+import Scene from "~/components/Scene";
+import Subheading from "~/components/Subheading";
+import useCurrentUser from "~/hooks/useCurrentUser";
+import useStores from "~/hooks/useStores";
+import useToasts from "~/hooks/useToasts";
+import FileOperationListItem from "./components/FileOperationListItem";
+
+function Export() {
+  const { t } = useTranslation();
+  const user = useCurrentUser();
+  const { fileOperations, collections } = useStores();
+  const { showToast } = useToasts();
+  const [isLoading, setLoading] = React.useState(false);
+  const [isExporting, setExporting] = React.useState(false);
+
+  const handleExport = React.useCallback(
+    async (ev: React.SyntheticEvent) => {
+      ev.preventDefault();
+      setLoading(true);
+
+      try {
+        await collections.export();
+        setExporting(true);
+        showToast(t("Export in progress…"));
+      } finally {
+        setLoading(false);
+      }
+    },
+    [t, collections, showToast]
+  );
+
+  const handleDelete = React.useCallback(
+    async (fileOperation: FileOperation) => {
+      try {
+        await fileOperations.delete(fileOperation);
+        showToast(t("Export deleted"));
+      } catch (err) {
+        showToast(err.message, {
+          type: "error",
+        });
+      }
+    },
+    [fileOperations, showToast, t]
+  );
+
+  return (
+    <Scene title={t("Export")} icon={<DownloadIcon color="currentColor" />}>
+      <Heading>{t("Export")}</Heading>
+      <HelpText>
+        <Trans
+          defaults="A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it’s complete."
+          values={{
+            userEmail: user.email,
+          }}
+          components={{
+            em: <strong />,
+          }}
+        />
+      </HelpText>
+      <Button
+        type="submit"
+        onClick={handleExport}
+        disabled={isLoading || isExporting}
+        primary
+      >
+        {isExporting
+          ? t("Export Requested")
+          : isLoading
+          ? `${t("Requesting Export")}…`
+          : t("Export Data")}
+      </Button>
+      <br />
+      <PaginatedList
+        items={fileOperations.exports}
+        fetch={fileOperations.fetchPage}
+        options={{
+          type: "export",
+        }}
+        heading={
+          <Subheading>
+            <Trans>Recent exports</Trans>
+          </Subheading>
+        }
+        renderItem={(item) => (
+          <FileOperationListItem
+            key={item.id}
+            fileOperation={item}
+            handleDelete={handleDelete}
+          />
+        )}
+      />
+    </Scene>
+  );
+}
+
+export default observer(Export);

app/scenes/Settings/Import.tsx

@@ -0,0 +1,154 @@
+import invariant from "invariant";
+import { observer } from "mobx-react";
+import { NewDocumentIcon } from "outline-icons";
+import * as React from "react";
+import { useTranslation, Trans } from "react-i18next";
+import { VisuallyHidden } from "reakit/VisuallyHidden";
+import getDataTransferFiles from "@shared/utils/getDataTransferFiles";
+import { cdnPath } from "@shared/utils/urls";
+import Button from "~/components/Button";
+import Heading from "~/components/Heading";
+import HelpText from "~/components/HelpText";
+import Item from "~/components/List/Item";
+import OutlineLogo from "~/components/OutlineLogo";
+import PaginatedList from "~/components/PaginatedList";
+import Scene from "~/components/Scene";
+import Subheading from "~/components/Subheading";
+import useStores from "~/hooks/useStores";
+import useToasts from "~/hooks/useToasts";
+import { uploadFile } from "~/utils/uploadFile";
+import FileOperationListItem from "./components/FileOperationListItem";
+
+function Import() {
+  const { t } = useTranslation();
+  const fileRef = React.useRef<HTMLInputElement>(null);
+  const { collections, fileOperations } = useStores();
+  const { showToast } = useToasts();
+  const [isImporting, setImporting] = React.useState(false);
+
+  const handleFilePicked = React.useCallback(
+    async (ev) => {
+      const files = getDataTransferFiles(ev);
+      const file = files[0];
+      invariant(file, "File must exist to upload");
+
+      setImporting(true);
+
+      try {
+        const attachment = await uploadFile(file, {
+          name: file.name,
+        });
+        await collections.import(attachment.id);
+        showToast(
+          t("Your import is being processed, you can safely leave this page"),
+          {
+            type: "success",
+            timeout: 8000,
+          }
+        );
+      } catch (err) {
+        showToast(err.message);
+      } finally {
+        if (fileRef.current) {
+          fileRef.current.value = "";
+        }
+
+        setImporting(false);
+      }
+    },
+    [t, collections, showToast]
+  );
+
+  const handlePickFile = React.useCallback(
+    (ev) => {
+      ev.preventDefault();
+
+      if (fileRef.current) {
+        fileRef.current.click();
+      }
+    },
+    [fileRef]
+  );
+
+  return (
+    <Scene title={t("Import")} icon={<NewDocumentIcon color="currentColor" />}>
+      <Heading>{t("Import")}</Heading>
+      <HelpText>
+        <Trans>
+          Quickly transfer your existing documents, pages, and files from other
+          tools and services into Outline. You can also drag and drop any HTML,
+          Markdown, and text documents directly into Collections in the app.
+        </Trans>
+      </HelpText>
+      <VisuallyHidden>
+        <input
+          type="file"
+          ref={fileRef}
+          onChange={handleFilePicked}
+          accept="application/zip"
+        />
+      </VisuallyHidden>
+
+      <div>
+        <Item
+          border={false}
+          image={<OutlineLogo size={28} fill="currentColor" />}
+          title="Outline"
+          subtitle={t(
+            "Import a backup file that was previously exported from Outline"
+          )}
+          actions={
+            <Button
+              type="submit"
+              onClick={handlePickFile}
+              disabled={isImporting}
+              neutral
+            >
+              {isImporting ? `${t("Uploading")}…` : t("Import")}
+            </Button>
+          }
+        />
+        <Item
+          border={false}
+          image={<img src={cdnPath("/images/confluence.png")} width={28} />}
+          title="Confluence"
+          subtitle={t("Import pages from a Confluence instance")}
+          actions={
+            <Button type="submit" onClick={handlePickFile} disabled neutral>
+              {t("Coming soon")}
+            </Button>
+          }
+        />
+        <Item
+          border={false}
+          image={<img src={cdnPath("/images/notion.png")} width={28} />}
+          title="Notion"
+          subtitle={t("Import documents from Notion")}
+          actions={
+            <Button type="submit" onClick={handlePickFile} disabled neutral>
+              {t("Coming soon")}
+            </Button>
+          }
+        />
+      </div>
+      <br />
+      <PaginatedList
+        items={fileOperations.imports}
+        fetch={fileOperations.fetchPage}
+        options={{
+          type: "import",
+        }}
+        heading={
+          <Subheading>
+            <Trans>Recent imports</Trans>
+          </Subheading>
+        }
+        renderItem={(item) => (
+          <FileOperationListItem key={item.id} fileOperation={item} />
+        )}
+      />
+    </Scene>
+  );
+}
+
+export default observer(Import);

app/scenes/Settings/ImportExport.tsx

@@ -1,274 +0,0 @@
-import invariant from "invariant";
-import { observer } from "mobx-react";
-import { CollectionIcon, DocumentIcon } from "outline-icons";
-import * as React from "react";
-import { useTranslation, Trans } from "react-i18next";
-import { VisuallyHidden } from "reakit/VisuallyHidden";
-import styled from "styled-components";
-import getDataTransferFiles from "@shared/utils/getDataTransferFiles";
-import { parseOutlineExport, Item } from "@shared/utils/zip";
-import FileOperation from "~/models/FileOperation";
-import Button from "~/components/Button";
-import Heading from "~/components/Heading";
-import HelpText from "~/components/HelpText";
-import Notice from "~/components/Notice";
-import PaginatedList from "~/components/PaginatedList";
-import Scene from "~/components/Scene";
-import Subheading from "~/components/Subheading";
-import useCurrentUser from "~/hooks/useCurrentUser";
-import useStores from "~/hooks/useStores";
-import useToasts from "~/hooks/useToasts";
-import { uploadFile } from "~/utils/uploadFile";
-import FileOperationListItem from "./components/FileOperationListItem";
-
-function ImportExport() {
-  const { t } = useTranslation();
-  const user = useCurrentUser();
-  const fileRef = React.useRef<HTMLInputElement>(null);
-  const { fileOperations, collections } = useStores();
-  const { showToast } = useToasts();
-  const [isLoading, setLoading] = React.useState(false);
-  const [isImporting, setImporting] = React.useState(false);
-  const [isImported, setImported] = React.useState(false);
-  const [isExporting, setExporting] = React.useState(false);
-  const [file, setFile] = React.useState<File>();
-  const [importDetails, setImportDetails] = React.useState<
-    Item[] | undefined
-  >();
-
-  const handleImport = React.useCallback(async () => {
-    setImported(false);
-    setImporting(true);
-
-    try {
-      invariant(file, "File must exist to upload");
-      const attachment = await uploadFile(file, {
-        name: file.name,
-      });
-      await collections.import(attachment.id);
-      showToast(t("Import started"));
-      setImported(true);
-    } catch (err) {
-      showToast(err.message);
-    } finally {
-      if (fileRef.current) {
-        fileRef.current.value = "";
-      }
-
-      setImporting(false);
-      setFile(undefined);
-      setImportDetails(undefined);
-    }
-  }, [t, file, collections, showToast]);
-
-  const handleFilePicked = React.useCallback(async (ev) => {
-    ev.preventDefault();
-    const files = getDataTransferFiles(ev);
-    const file = files[0];
-    setFile(file);
-
-    try {
-      setImportDetails(await parseOutlineExport(file));
-    } catch (err) {
-      setImportDetails([]);
-    }
-  }, []);
-
-  const handlePickFile = React.useCallback(
-    (ev) => {
-      ev.preventDefault();
-
-      if (fileRef.current) {
-        fileRef.current.click();
-      }
-    },
-    [fileRef]
-  );
-
-  const handleExport = React.useCallback(
-    async (ev: React.SyntheticEvent) => {
-      ev.preventDefault();
-      setLoading(true);
-
-      try {
-        await collections.export();
-        setExporting(true);
-        showToast(t("Export in progress…"));
-      } finally {
-        setLoading(false);
-      }
-    },
-    [t, collections, showToast]
-  );
-
-  const handleDelete = React.useCallback(
-    async (fileOperation: FileOperation) => {
-      try {
-        await fileOperations.delete(fileOperation);
-        showToast(t("Export deleted"));
-      } catch (err) {
-        showToast(err.message, {
-          type: "error",
-        });
-      }
-    },
-    [fileOperations, showToast, t]
-  );
-
-  const hasCollections = importDetails
-    ? !!importDetails.filter((detail) => detail.type === "collection").length
-    : false;
-  const hasDocuments = importDetails
-    ? !!importDetails.filter((detail) => detail.type === "document").length
-    : false;
-  const isImportable = hasCollections && hasDocuments;
-
-  return (
-    <Scene
-      title={`${t("Import")} / ${t("Export")}`}
-      icon={<DocumentIcon color="currentColor" />}
-    >
-      <Heading>{t("Import")}</Heading>
-      <HelpText>
-        <Trans>
-          It is possible to import a zip file of folders and Markdown files
-          previously exported from an Outline instance. Support will soon be
-          added for importing from other services.
-        </Trans>
-      </HelpText>
-      <VisuallyHidden>
-        <input
-          type="file"
-          ref={fileRef}
-          onChange={handleFilePicked}
-          accept="application/zip"
-        />
-      </VisuallyHidden>
-      {isImported && (
-        <Notice>
-          <Trans>
-            Your file has been uploaded and the import is currently being
-            processed, you can safely leave this page while it completes.
-          </Trans>
-        </Notice>
-      )}
-      {file && !isImportable && (
-        <ImportPreview>
-          <Trans
-            defaults="Sorry, the file <em>{{ fileName }}</em> is missing valid collections or documents."
-            values={{
-              fileName: file.name,
-            }}
-            components={{
-              em: <strong />,
-            }}
-          />
-        </ImportPreview>
-      )}
-      {file && importDetails && isImportable ? (
-        <>
-          <ImportPreview as="div">
-            <Trans
-              defaults="<em>{{ fileName }}</em> looks good, the following collections and their documents will be imported:"
-              values={{
-                fileName: file.name,
-              }}
-              components={{
-                em: <strong />,
-              }}
-            />
-            <List>
-              {importDetails
-                .filter((detail) => detail.type === "collection")
-                .map((detail) => (
-                  <ImportPreviewItem key={detail.path}>
-                    <CollectionIcon />
-                    <CollectionName>{detail.name}</CollectionName>
-                  </ImportPreviewItem>
-                ))}
-            </List>
-          </ImportPreview>
-          <Button
-            type="submit"
-            onClick={handleImport}
-            disabled={isImporting}
-            primary
-          >
-            {isImporting ? `${t("Uploading")}…` : t("Confirm & Import")}
-          </Button>
-        </>
-      ) : (
-        <Button type="submit" onClick={handlePickFile} primary>
-          {t("Choose File")}…
-        </Button>
-      )}
-      <Heading>{t("Export")}</Heading>
-      <HelpText>
-        <Trans
-          defaults="A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it’s complete."
-          values={{
-            userEmail: user.email,
-          }}
-          components={{
-            em: <strong />,
-          }}
-        />
-      </HelpText>
-      <Button
-        type="submit"
-        onClick={handleExport}
-        disabled={isLoading || isExporting}
-        primary
-      >
-        {isExporting
-          ? t("Export Requested")
-          : isLoading
-          ? `${t("Requesting Export")}…`
-          : t("Export Data")}
-      </Button>
-      <br />
-      <br />
-      <PaginatedList
-        items={fileOperations.orderedDataExports}
-        fetch={fileOperations.fetchPage}
-        options={{
-          type: "export",
-        }}
-        heading={
-          <Subheading>
-            <Trans>Recent exports</Trans>
-          </Subheading>
-        }
-        renderItem={(item) => (
-          <FileOperationListItem
-            key={item.id + item.state}
-            fileOperation={item}
-            handleDelete={handleDelete}
-          />
-        )}
-      />
-    </Scene>
-  );
-}
-
-const List = styled.ul`
-  padding: 0;
-  margin: 8px 0 0;
-`;
-
-const ImportPreview = styled(Notice)`
-  margin-bottom: 16px;
-`;
-
-const ImportPreviewItem = styled.li`
-  display: flex;
-  align-items: center;
-  list-style: none;
-`;
-
-const CollectionName = styled.span`
-  font-weight: 500;
-  margin-left: 4px;
-`;
-
-export default observer(ImportExport);

app/scenes/Settings/components/FileOperationListItem.tsx

@@ -1,38 +1,54 @@
+import { observer } from "mobx-react";
+import { DoneIcon, WarningIcon } from "outline-icons";
 import * as React from "react";
 import { useTranslation } from "react-i18next";
+import { useTheme } from "styled-components";
 import FileOperation from "~/models/FileOperation";
 import { Action } from "~/components/Actions";
 import ListItem from "~/components/List/Item";
+import Spinner from "~/components/Spinner";
 import Time from "~/components/Time";
 import useCurrentUser from "~/hooks/useCurrentUser";
 import FileOperationMenu from "~/menus/FileOperationMenu";
 
 type Props = {
   fileOperation: FileOperation;
-  handleDelete: (arg0: FileOperation) => Promise<void>;
+  handleDelete?: (arg0: FileOperation) => Promise<void>;
 };
 
 const FileOperationListItem = ({ fileOperation, handleDelete }: Props) => {
   const { t } = useTranslation();
   const user = useCurrentUser();
+  const theme = useTheme();
   const stateMapping = {
     creating: t("Processing"),
     expired: t("Expired"),
     uploading: t("Processing"),
-    error: t("Error"),
+    error: t("Failed"),
   };
+
+  const iconMapping = {
+    creating: <Spinner />,
+    uploading: <Spinner />,
+    complete: <DoneIcon color={theme.primary} />,
+    error: <WarningIcon color={theme.danger} />,
+  };
+
+  const title =
+    fileOperation.type === "import" || fileOperation.collectionId
+      ? fileOperation.name
+      : t("All collections");
+
   return (
     <ListItem
-      title={
-        fileOperation.collection
-          ? fileOperation.collection.name
-          : t("All collections")
-      }
+      title={title}
+      image={iconMapping[fileOperation.state]}
       subtitle={
         <>
           {fileOperation.state !== "complete" && (
             <>{stateMapping[fileOperation.state]}&nbsp;•&nbsp;</>
           )}
+          {fileOperation.error && <>{fileOperation.error}&nbsp;•&nbsp;</>}
           {t(`{{userName}} requested`, {
             userName:
               user.id === fileOperation.user.id
@@ -45,7 +61,7 @@ const FileOperationListItem = ({ fileOperation, handleDelete }: Props) => {
         </>
       }
       actions={
-        fileOperation.state === "complete" ? (
+        fileOperation.state === "complete" && handleDelete ? (
           <Action>
             <FileOperationMenu
               id={fileOperation.id}
@@ -61,4 +77,4 @@ const FileOperationListItem = ({ fileOperation, handleDelete }: Props) => {
   );
 };
 
-export default FileOperationListItem;
+export default observer(FileOperationListItem);

app/stores/FileOperationsStore.ts

@@ -12,15 +12,26 @@ export default class FileOperationsStore extends BaseStore<FileOperation> {
   }
 
   @computed
-  get exports(): FileOperation[] {
-    return Array.from(this.data.values()).reduce(
-      (acc, fileOp) => (fileOp.type === "export" ? [...acc, fileOp] : acc),
-      []
+  get imports(): FileOperation[] {
+    return orderBy(
+      Array.from(this.data.values()).reduce(
+        (acc, fileOp) => (fileOp.type === "import" ? [...acc, fileOp] : acc),
+        []
+      ),
+      "createdAt",
+      "desc"
     );
   }
 
   @computed
-  get orderedDataExports(): FileOperation[] {
-    return orderBy(this.exports, "createdAt", "desc");
+  get exports(): FileOperation[] {
+    return orderBy(
+      Array.from(this.data.values()).reduce(
+        (acc, fileOp) => (fileOp.type === "export" ? [...acc, fileOp] : acc),
+        []
+      ),
+      "createdAt",
+      "desc"
+    );
   }
 }

server/commands/collectionImporter.ts

@@ -5,9 +5,9 @@ import File from "formidable/lib/file";
 import invariant from "invariant";
 import { values, keys } from "lodash";
 import { v4 as uuidv4 } from "uuid";
-import { parseOutlineExport } from "@shared/utils/zip";
 import Logger from "@server/logging/logger";
 import { Attachment, Event, Document, Collection, User } from "@server/models";
+import { parseOutlineExport, Item } from "@server/utils/zip";
 import { FileImportError } from "../errors";
 import attachmentCreator from "./attachmentCreator";
 import documentCreator from "./documentCreator";
@@ -30,10 +30,10 @@ export default async function collectionImporter({
 }) {
   // load the zip structure into memory
   const zipData = await fs.promises.readFile(file.path);
-  let items;
+  let items: Item[];
 
   try {
-    items = await await parseOutlineExport(zipData);
+    items = await parseOutlineExport(zipData);
   } catch (err) {
     throw FileImportError(err.message);
   }

server/commands/fileOperationDeleter.ts

@@ -2,14 +2,14 @@ import { sequelize } from "@server/database/sequelize";
 import { FileOperation, Event, User } from "@server/models";
 
 export default async function fileOperationDeleter(
-  fileOp: FileOperation,
+  fileOperation: FileOperation,
   user: User,
   ip: string
 ) {
   const transaction = await sequelize.transaction();
 
   try {
-    await fileOp.destroy({
+    await fileOperation.destroy({
       transaction,
     });
     await Event.create(
@@ -17,8 +17,7 @@ export default async function fileOperationDeleter(
         name: "fileOperations.delete",
         teamId: user.teamId,
         actorId: user.id,
-        // @ts-expect-error dataValues does exist
-        data: fileOp.dataValues,
+        modelId: fileOperation.id,
         ip,
       },
       {

server/migrations/20220206225006-add-error-to-file-operation.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn("file_operations", "error", {
+      type: Sequelize.STRING,
+      allowNull: true,
+    });
+  },
+  down: async (queryInterface) => {
+    await queryInterface.removeColumn("file_operations", "error");
+  },
+};

server/models/FileOperation.ts

@@ -45,6 +45,9 @@ class FileOperation extends BaseModel {
   @Column
   url: string;
 
+  @Column
+  error: string | null;
+
   @Column(DataType.BIGINT)
   size: number;
 

server/presenters/fileOperation.ts

@@ -1,13 +1,16 @@
+import path from "path";
 import { FileOperation } from "@server/models";
-import { presentCollection, presentUser } from ".";
+import { presentUser } from ".";
 
 export default function present(data: FileOperation) {
   return {
     id: data.id,
     type: data.type,
+    name: data.collection?.name || path.basename(data.key || ""),
     state: data.state,
-    collection: data.collection ? presentCollection(data.collection) : null,
+    error: data.error,
     size: data.size,
+    collectionId: data.collectionId,
     user: presentUser(data.user),
     createdAt: data.createdAt,
   };

server/queues/processors/exports.ts

@@ -19,60 +19,62 @@ export default class ExportsProcessor {
         const user = await User.findByPk(actorId);
         invariant(user, "user operation not found");
 
-        const exportData = await FileOperation.findByPk(event.modelId);
-        invariant(exportData, "exportData not found");
+        const fileOperation = await FileOperation.findByPk(event.modelId);
+        invariant(fileOperation, "fileOperation not found");
 
         const collectionIds =
-          // @ts-expect-error ts-migrate(2339) FIXME: Property 'collectionId' does not exist on type 'Co... Remove this comment to see the full error message
-          event.collectionId || (await user.collectionIds());
+          "collectionId" in event
+            ? event.collectionId
+            : await user.collectionIds();
         const collections = await Collection.findAll({
           where: {
             id: collectionIds,
           },
         });
-        this.updateFileOperation(exportData, actorId, teamId, {
+        this.updateFileOperation(fileOperation, actorId, teamId, {
           state: "creating",
         });
         // heavy lifting of creating the zip file
         Logger.info(
           "processor",
-          `Archiving collections for file operation ${exportData.id}`
+          `Archiving collections for file operation ${fileOperation.id}`
         );
         const filePath = await archiveCollections(collections);
-        let url, state;
+        let url;
+        let state: any = "creating";
 
         try {
           // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
           const readBuffer = await fs.promises.readFile(filePath);
           // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
           const stat = await fs.promises.stat(filePath);
-          this.updateFileOperation(exportData, actorId, teamId, {
+          this.updateFileOperation(fileOperation, actorId, teamId, {
             state: "uploading",
             size: stat.size,
           });
           Logger.info(
             "processor",
-            `Uploading archive for file operation ${exportData.id}`
+            `Uploading archive for file operation ${fileOperation.id}`
           );
           url = await uploadToS3FromBuffer(
             readBuffer,
             "application/zip",
-            exportData.key,
+            fileOperation.key,
             "private"
           );
           Logger.info(
             "processor",
-            `Upload complete for file operation ${exportData.id}`
+            `Upload complete for file operation ${fileOperation.id}`
           );
           state = "complete";
         } catch (error) {
           Logger.error("Error exporting collection data", error, {
-            fileOperationId: exportData.id,
+            fileOperationId: fileOperation.id,
           });
           state = "error";
-          url = null;
+          url = undefined;
         } finally {
-          this.updateFileOperation(exportData, actorId, teamId, {
+          this.updateFileOperation(fileOperation, actorId, teamId, {
             state,
             url,
           });
@@ -85,7 +87,7 @@ export default class ExportsProcessor {
           } else {
             mailer.sendTemplate("exportSuccess", {
               to: user.email,
-              id: exportData.id,
+              id: fileOperation.id,
               teamUrl: team.url,
             });
           }
@@ -101,15 +103,14 @@ export default class ExportsProcessor {
     fileOperation: FileOperation,
     actorId: string,
     teamId: string,
-    data: Record<string, any>
+    data: Partial<FileOperation>
   ) {
     await fileOperation.update(data);
     await Event.add({
       name: "fileOperations.update",
       teamId,
       actorId,
-      // @ts-expect-error dataValues exists
-      data: fileOperation.dataValues,
+      modelId: fileOperation.id,
     });
   }
 }

server/queues/processors/imports.ts

@@ -3,13 +3,14 @@ import os from "os";
 import File from "formidable/lib/file";
 import invariant from "invariant";
 import collectionImporter from "@server/commands/collectionImporter";
-import { Attachment, User } from "@server/models";
-import { Event } from "../../types";
+import { Event, FileOperation, Attachment, User } from "@server/models";
+import { Event as TEvent } from "../../types";
 
 export default class ImportsProcessor {
-  async on(event: Event) {
+  async on(event: TEvent) {
     switch (event.name) {
       case "collections.import": {
+        let state, error;
         const { type } = event.data;
         const attachment = await Attachment.findByPk(event.modelId);
         invariant(attachment, "attachment not found");
@@ -17,22 +18,55 @@ export default class ImportsProcessor {
         const user = await User.findByPk(event.actorId);
         invariant(user, "user not found");
 
-        const buffer: any = await attachment.buffer;
-        const tmpDir = os.tmpdir();
-        const tmpFilePath = `${tmpDir}/upload-${event.modelId}`;
-        await fs.promises.writeFile(tmpFilePath, buffer);
-        const file = new File({
-          name: attachment.name,
-          type: attachment.contentType,
-          path: tmpFilePath,
+        const fileOperation = await FileOperation.create({
+          type: "import",
+          state: "creating",
+          size: attachment.size,
+          key: attachment.key,
+          userId: user.id,
+          teamId: user.teamId,
         });
-        await collectionImporter({
-          file,
-          user,
-          type,
-          ip: event.ip,
+
+        await Event.add({
+          name: "fileOperations.create",
+          modelId: fileOperation.id,
+          teamId: user.teamId,
+          actorId: user.id,
         });
-        await attachment.destroy();
+
+        try {
+          const buffer = await attachment.buffer;
+          const tmpDir = os.tmpdir();
+          const tmpFilePath = `${tmpDir}/upload-${event.modelId}`;
+          await fs.promises.writeFile(tmpFilePath, buffer as Uint8Array);
+          const file = new File({
+            name: attachment.name,
+            type: attachment.contentType,
+            path: tmpFilePath,
+          });
+
+          await collectionImporter({
+            file,
+            user,
+            type,
+            ip: event.ip,
+          });
+          await attachment.destroy();
+
+          state = "complete";
+        } catch (err) {
+          state = "error";
+          error = err.message;
+        } finally {
+          await fileOperation.update({ state, error });
+          await Event.add({
+            name: "fileOperations.update",
+            modelId: fileOperation.id,
+            teamId: user.teamId,
+            actorId: user.id,
+          });
+        }
+
         return;
       }
 

server/queues/processors/websockets.ts

@@ -3,13 +3,18 @@ import { Op } from "sequelize";
 import {
   Document,
   Collection,
+  FileOperation,
   Group,
   CollectionGroup,
   GroupUser,
   Pin,
   Star,
 } from "@server/models";
-import { presentPin, presentStar } from "@server/presenters";
+import {
+  presentFileOperation,
+  presentPin,
+  presentStar,
+} from "@server/presenters";
 import { Event } from "../../types";
 
 export default class WebsocketsProcessor {
@@ -354,10 +359,14 @@ export default class WebsocketsProcessor {
         return;
       }
 
+      case "fileOperations.create":
       case "fileOperations.update": {
-        return socketio
-          .to(`user-${event.actorId}`)
-          .emit("fileOperations.update", event.data);
+        const fileOperation = await FileOperation.findByPk(event.modelId);
+        if (!fileOperation) {
+          return;
+        }
+        const data = await presentFileOperation(fileOperation);
+        return socketio.to(`user-${event.actorId}`).emit(event.name, data);
       }
 
       case "pins.create":

server/routes/api/fileOperations.test.ts

@@ -118,7 +118,7 @@ describe("#fileOperations.list", () => {
     expect(data.id).toBe(exportData.id);
     expect(data.key).toBe(undefined);
     expect(data.state).toBe(exportData.state);
-    expect(data.collection.id).toBe(collection.id);
+    expect(data.collectionId).toBe(collection.id);
   });
 
   it("should return exports with collection data even if collection is deleted", async () => {
@@ -152,7 +152,7 @@ describe("#fileOperations.list", () => {
     expect(data.id).toBe(exportData.id);
     expect(data.key).toBe(undefined);
     expect(data.state).toBe(exportData.state);
-    expect(data.collection.id).toBe(collection.id);
+    expect(data.collectionId).toBe(collection.id);
   });
 
   it("should return exports with user data even if user is deleted", async () => {

server/types.ts

@@ -130,9 +130,13 @@ export type CollectionExportAllEvent = {
 };
 
 export type FileOperationEvent = {
-  name: "fileOperations.update" | "fileOperation.delete";
+  name:
+    | "fileOperations.create"
+    | "fileOperations.update"
+    | "fileOperation.delete";
   teamId: string;
   actorId: string;
+  modelId: string;
   data: {
     type: string;
     state: string;

server/utils/zip.ts

@@ -1,5 +1,6 @@
 import fs from "fs";
-import JSZip from "jszip";
+import path from "path";
+import JSZip, { JSZipObject } from "jszip";
 import tmp from "tmp";
 import Logger from "@server/logging/logger";
 import Attachment from "@server/models/Attachment";
@@ -10,6 +11,18 @@ import { serializeFilename } from "./fs";
 import parseAttachmentIds from "./parseAttachmentIds";
 import { getFileByKey } from "./s3";
 
+type ItemType = "collection" | "document" | "attachment";
+
+export type Item = {
+  path: string;
+  dir: string;
+  name: string;
+  depth: number;
+  metadata: Record<string, any>;
+  type: ItemType;
+  item: JSZipObject;
+};
+
 async function addToArchive(zip: JSZip, documents: NavigationNode[]) {
   for (const doc of documents) {
     const document = await Document.findByPk(doc.id);
@@ -104,3 +117,78 @@ export async function archiveCollections(collections: Collection[]) {
 
   return archiveToPath(zip);
 }
+
+export async function parseOutlineExport(
+  input: File | Buffer
+): Promise<Item[]> {
+  const zip = await JSZip.loadAsync(input);
+  // this is so we can use async / await a little easier
+  const items: Item[] = [];
+
+  for (const rawPath in zip.files) {
+    const item = zip.files[rawPath];
+
+    if (!item) {
+      throw new Error(
+        `No item at ${rawPath} in zip file. This zip file might be corrupt.`
+      );
+    }
+
+    const itemPath = rawPath.replace(/\/$/, "");
+    const dir = path.dirname(itemPath);
+    const name = path.basename(item.name);
+    const depth = itemPath.split("/").length - 1;
+
+    // known skippable items
+    if (itemPath.startsWith("__MACOSX") || itemPath.endsWith(".DS_Store")) {
+      continue;
+    }
+
+    // attempt to parse extra metadata from zip comment
+    let metadata = {};
+
+    try {
+      metadata = item.comment ? JSON.parse(item.comment) : {};
+    } catch (err) {
+      console.log(
+        `ZIP comment found for ${item.name}, but could not be parsed as metadata: ${item.comment}`
+      );
+    }
+
+    if (depth === 0 && !item.dir) {
+      throw new Error(
+        "Root of zip file must only contain folders representing collections"
+      );
+    }
+
+    let type: ItemType | undefined;
+
+    if (depth === 0 && item.dir && name) {
+      type = "collection";
+    }
+
+    if (depth > 0 && !item.dir && item.name.endsWith(".md")) {
+      type = "document";
+    }
+
+    if (depth > 0 && !item.dir && itemPath.includes("uploads")) {
+      type = "attachment";
+    }
+
+    if (!type) {
+      continue;
+    }
+
+    items.push({
+      path: itemPath,
+      dir,
+      name,
+      depth,
+      type,
+      metadata,
+      item,
+    });
+  }
+
+  return items;
+}

shared/i18n/locales/en_US/translation.json

@@ -514,7 +514,7 @@
   "No documents found for your search filters.": "No documents found for your search filters.",
   "Processing": "Processing",
   "Expired": "Expired",
-  "Error": "Error",
+  "Failed": "Failed",
   "All collections": "All collections",
   "{{userName}} requested": "{{userName}} requested",
   "Upload": "Upload",
@@ -541,6 +541,13 @@
   "Logo": "Logo",
   "Subdomain": "Subdomain",
   "Your knowledge base will be accessible at": "Your knowledge base will be accessible at",
+  "Export in progress…": "Export in progress…",
+  "Export deleted": "Export deleted",
+  "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it’s complete.": "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it’s complete.",
+  "Export Requested": "Export Requested",
+  "Requesting Export": "Requesting Export",
+  "Export Data": "Export Data",
+  "Recent exports": "Recent exports",
   "Manage optional and beta features. Changing these settings will affect the experience for all team members.": "Manage optional and beta features. Changing these settings will affect the experience for all team members.",
   "Collaborative editing": "Collaborative editing",
   "When enabled multiple people can edit documents at the same time with shared presence and live cursors.": "When enabled multiple people can edit documents at the same time with shared presence and live cursors.",
@@ -548,21 +555,14 @@
   "Groups can be used to organize and manage the people on your team.": "Groups can be used to organize and manage the people on your team.",
   "All groups": "All groups",
   "No groups have been created yet": "No groups have been created yet",
-  "Import started": "Import started",
-  "Export in progress…": "Export in progress…",
-  "Export deleted": "Export deleted",
-  "It is possible to import a zip file of folders and Markdown files previously exported from an Outline instance. Support will soon be added for importing from other services.": "It is possible to import a zip file of folders and Markdown files previously exported from an Outline instance. Support will soon be added for importing from other services.",
-  "Your file has been uploaded and the import is currently being processed, you can safely leave this page while it completes.": "Your file has been uploaded and the import is currently being processed, you can safely leave this page while it completes.",
-  "Sorry, the file <em>{{ fileName }}</em> is missing valid collections or documents.": "Sorry, the file <em>{{ fileName }}</em> is missing valid collections or documents.",
-  "<em>{{ fileName }}</em> looks good, the following collections and their documents will be imported:": "<em>{{ fileName }}</em> looks good, the following collections and their documents will be imported:",
+  "Your import is being processed, you can safely leave this page": "Your import is being processed, you can safely leave this page",
+  "Quickly transfer your existing documents, pages, and files from other tools and services into Outline. You can also drag and drop any HTML, Markdown, and text documents directly into Collections in the app.": "Quickly transfer your existing documents, pages, and files from other tools and services into Outline. You can also drag and drop any HTML, Markdown, and text documents directly into Collections in the app.",
+  "Import a backup file that was previously exported from Outline": "Import a backup file that was previously exported from Outline",
   "Uploading": "Uploading",
-  "Confirm & Import": "Confirm & Import",
-  "Choose File": "Choose File",
-  "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it’s complete.": "A full export might take some time, consider exporting a single document or collection. The exported data is a zip of your documents in Markdown format. You may leave this page once the export has started – we will email a link to <em>{{ userEmail }}</em> when it’s complete.",
-  "Export Requested": "Export Requested",
-  "Requesting Export": "Requesting Export",
-  "Export Data": "Export Data",
-  "Recent exports": "Recent exports",
+  "Import pages from a Confluence instance": "Import pages from a Confluence instance",
+  "Coming soon": "Coming soon",
+  "Import documents from Notion": "Import documents from Notion",
+  "Recent imports": "Recent imports",
   "Receive a notification whenever a new document is published": "Receive a notification whenever a new document is published",
   "Receive a notification when a document you created is edited": "Receive a notification when a document you created is edited",
   "Collection created": "Collection created",

shared/utils/zip.ts

@@ -1,79 +0,0 @@
-import path from "path";
-import JSZip, { JSZipObject } from "jszip";
-
-type ItemType = "collection" | "document" | "attachment";
-
-export type Item = {
-  path: string;
-  dir: string;
-  name: string;
-  depth: number;
-  metadata: Record<string, any>;
-  type: ItemType;
-  item: JSZipObject;
-};
-
-export async function parseOutlineExport(
-  input: File | Buffer
-): Promise<Item[]> {
-  const zip = await JSZip.loadAsync(input);
-  // this is so we can use async / await a little easier
-  const items: Item[] = [];
-  zip.forEach(async function (rawPath, item) {
-    const itemPath = rawPath.replace(/\/$/, "");
-    const dir = path.dirname(itemPath);
-    const name = path.basename(item.name);
-    const depth = itemPath.split("/").length - 1;
-
-    // known skippable items
-    if (itemPath.startsWith("__MACOSX") || itemPath.endsWith(".DS_Store")) {
-      return;
-    }
-
-    // attempt to parse extra metadata from zip comment
-    let metadata = {};
-
-    try {
-      metadata = item.comment ? JSON.parse(item.comment) : {};
-    } catch (err) {
-      console.log(
-        `ZIP comment found for ${item.name}, but could not be parsed as metadata: ${item.comment}`
-      );
-    }
-
-    if (depth === 0 && !item.dir) {
-      throw new Error(
-        "Root of zip file must only contain folders representing collections"
-      );
-    }
-
-    let type: ItemType | undefined;
-
-    if (depth === 0 && item.dir && name) {
-      type = "collection";
-    }
-
-    if (depth > 0 && !item.dir && item.name.endsWith(".md")) {
-      type = "document";
-    }
-
-    if (depth > 0 && !item.dir && itemPath.includes("uploads")) {
-      type = "attachment";
-    }
-
-    if (!type) {
-      return;
-    }
-
-    items.push({
-      path: itemPath,
-      dir,
-      name,
-      depth,
-      type,
-      metadata,
-      item,
-    });
-  });
-  return items;
-}