[PM-18969] CSV importers should create nested collections (#14007)

libs/importer/src/importers/base-importer.spec.ts

@@ -1,6 +1,9 @@
 import { CipherType } from "@bitwarden/common/vault/enums";
 import { CardView } from "@bitwarden/common/vault/models/view/card.view";
 import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view";
+import { FolderView } from "@bitwarden/common/vault/models/view/folder.view";
+
+import { ImportResult } from "../models";
 
 import { BaseImporter } from "./base-importer";
 
@@ -16,8 +19,169 @@ class FakeBaseImporter extends BaseImporter {
   parseXml(data: string): Document {
     return super.parseXml(data);
   }
+
+  processFolder(result: ImportResult, folderName: string, addRelationship: boolean = true): void {
+    return super.processFolder(result, folderName, addRelationship);
+  }
 }
 
+describe("processFolder method", () => {
+  let result: ImportResult;
+  const importer = new FakeBaseImporter();
+
+  beforeEach(() => {
+    result = {
+      folders: [],
+      folderRelationships: [],
+      collections: [],
+      collectionRelationships: [],
+      ciphers: [],
+      success: false,
+      errorMessage: "",
+    };
+  });
+
+  it("should add a new folder and relationship when folderName is unique", () => {
+    // arrange
+    // a folder exists - but it is not the same as the one we are importing
+    result = {
+      folders: [{ name: "ABC" } as FolderView],
+      folderRelationships: [],
+      collections: [],
+      collectionRelationships: [],
+      ciphers: [{ name: "cipher1", id: "cipher1" } as CipherView],
+      success: false,
+      errorMessage: "",
+    };
+    importer.processFolder(result, "Folder1");
+
+    expect(result.folders).toHaveLength(2);
+    expect(result.folders[0].name).toBe("ABC");
+    expect(result.folders[1].name).toBe("Folder1");
+    expect(result.folderRelationships).toHaveLength(1);
+    expect(result.folderRelationships[0]).toEqual([1, 1]); // cipher1 -> Folder1
+  });
+
+  it("should not add duplicate folders and should add relationships", () => {
+    // setup
+    // folder called "Folder1" already exists
+    result = {
+      folders: [{ name: "Folder1" } as FolderView],
+      folderRelationships: [],
+      collections: [],
+      collectionRelationships: [],
+      ciphers: [{ name: "cipher1", id: "cipher1" } as CipherView],
+      success: false,
+      errorMessage: "",
+    };
+
+    // import an existing folder should not add to the result.folders
+    importer.processFolder(result, "Folder1");
+
+    expect(result.folders).toHaveLength(1);
+    expect(result.folders[0].name).toBe("Folder1");
+    expect(result.folderRelationships).toHaveLength(1);
+    expect(result.folderRelationships[0]).toEqual([1, 0]); // cipher1 -> folder1
+  });
+
+  it("should create parent folders for nested folder names but not duplicates", () => {
+    // arrange
+    result = {
+      folders: [
+        { name: "Ancestor/Parent/Child" } as FolderView,
+        { name: "Ancestor" } as FolderView,
+      ],
+      folderRelationships: [],
+      collections: [],
+      collectionRelationships: [],
+      ciphers: [{ name: "cipher1", id: "cipher1" } as CipherView],
+      success: false,
+      errorMessage: "",
+    };
+
+    // act
+    // importing an existing folder with a relationship should not change the result.folders
+    // nor should it change the result.folderRelationships
+    importer.processFolder(result, "Ancestor/Parent/Child/Grandchild/GreatGrandchild");
+
+    expect(result.folders).toHaveLength(5);
+    expect(result.folders.map((f) => f.name)).toEqual([
+      "Ancestor/Parent/Child",
+      "Ancestor",
+      "Ancestor/Parent/Child/Grandchild/GreatGrandchild",
+      "Ancestor/Parent/Child/Grandchild",
+      "Ancestor/Parent",
+    ]);
+    expect(result.folderRelationships).toHaveLength(1);
+    expect(result.folderRelationships[0]).toEqual([1, 2]); // cipher1 -> grandchild
+  });
+
+  it("should not affect existing relationships", () => {
+    // arrange
+    // "Parent" is a folder with no relationship
+    // "Child" is a folder with 2 ciphers
+    result = {
+      folders: [{ name: "Parent" } as FolderView, { name: "Parent/Child" } as FolderView],
+      folderRelationships: [
+        [1, 1],
+        [2, 1],
+      ],
+      collections: [],
+      collectionRelationships: [],
+      ciphers: [
+        { name: "cipher1", id: "cipher1" } as CipherView,
+        { name: "cipher2", id: "cipher2" } as CipherView,
+        { name: "cipher3", id: "cipher3" } as CipherView,
+      ],
+      success: false,
+      errorMessage: "",
+    };
+
+    // act
+    // importing an existing folder with a relationship should not change the result.folders
+    // nor should it change the result.folderRelationships
+    importer.processFolder(result, "Parent/Child/Grandchild");
+
+    expect(result.folders).toHaveLength(3);
+    expect(result.folders.map((f) => f.name)).toEqual([
+      "Parent",
+      "Parent/Child",
+      "Parent/Child/Grandchild",
+    ]);
+    expect(result.folderRelationships).toHaveLength(3);
+    expect(result.folderRelationships[0]).toEqual([1, 1]); // cipher1 -> child
+    expect(result.folderRelationships[1]).toEqual([2, 1]); // cipher2 -> child
+    expect(result.folderRelationships[2]).toEqual([3, 2]); // cipher3 -> grandchild
+  });
+
+  it("should not add relationships if addRelationship is false", () => {
+    importer.processFolder(result, "Folder1", false);
+
+    expect(result.folders).toHaveLength(1);
+    expect(result.folders[0].name).toBe("Folder1");
+    expect(result.folderRelationships).toHaveLength(0);
+  });
+
+  it("should replace backslashes with forward slashes in folder names", () => {
+    importer.processFolder(result, "Parent\\Child\\Grandchild");
+
+    expect(result.folders).toHaveLength(3);
+    expect(result.folders.map((f) => f.name)).toEqual([
+      "Parent/Child/Grandchild",
+      "Parent/Child",
+      "Parent",
+    ]);
+  });
+
+  it("should handle empty or null folder names gracefully", () => {
+    importer.processFolder(result, null);
+    importer.processFolder(result, "");
+
+    expect(result.folders).toHaveLength(0);
+    expect(result.folderRelationships).toHaveLength(0);
+  });
+});
+
 describe("BaseImporter class", () => {
   const importer = new FakeBaseImporter();
   let cipher: CipherView;

libs/importer/src/importers/base-importer.ts

@@ -366,7 +366,7 @@ export abstract class BaseImporter {
 
     let folderIndex = result.folders.length;
     // Replace backslashes with forward slashes, ensuring we create sub-folders
-    folderName = folderName.replace("\\", "/");
+    folderName = folderName.replace(/\\/g, "/");
     let addFolder = true;
 
     for (let i = 0; i < result.folders.length; i++) {
@@ -387,6 +387,17 @@ export abstract class BaseImporter {
     if (addRelationship) {
       result.folderRelationships.push([result.ciphers.length, folderIndex]);
     }
+
+    // if the folder name is a/b/c/d, we need to create a/b/c and a/b and a
+    const parts = folderName.split("/");
+    for (let i = parts.length - 1; i > 0; i--) {
+      const parentName = parts.slice(0, i).join("/") as string;
+      if (result.folders.find((c) => c.name === parentName) == null) {
+        const folder = new FolderView();
+        folder.name = parentName;
+        result.folders.push(folder);
+      }
+    }
   }
 
   protected convertToNoteIfNeeded(cipher: CipherView) {

libs/importer/src/importers/bitwarden/bitwarden-csv-importer.ts

@@ -1,6 +1,5 @@
 // FIXME: Update this file to be type safe and remove this and next line
 // @ts-strict-ignore
-import { CollectionView } from "@bitwarden/admin-console/common";
 import { FieldType, SecureNoteType, CipherType } from "@bitwarden/common/vault/enums";
 import { CipherRepromptType } from "@bitwarden/common/vault/enums/cipher-reprompt-type";
 import { CipherView } from "@bitwarden/common/vault/models/view/cipher.view";
@@ -25,35 +24,11 @@ export class BitwardenCsvImporter extends BaseImporter implements Importer {
       if (this.organization && !this.isNullOrWhitespace(value.collections)) {
         const collections = (value.collections as string).split(",");
         collections.forEach((col) => {
-          let addCollection = true;
-          let collectionIndex = result.collections.length;
-
-          for (let i = 0; i < result.collections.length; i++) {
-            if (result.collections[i].name === col) {
-              addCollection = false;
-              collectionIndex = i;
-              break;
-            }
-          }
-
-          if (addCollection) {
-            const collection = new CollectionView();
-            collection.name = col;
-            result.collections.push(collection);
-          }
-
-          result.collectionRelationships.push([result.ciphers.length, collectionIndex]);
-
-          // if the collection name is a/b/c/d, we need to create a/b/c and a/b and a
-          const parts = col.split("/");
-          for (let i = parts.length - 1; i > 0; i--) {
-            const parentCollectionName = parts.slice(0, i).join("/") as string;
-            if (result.collections.find((c) => c.name === parentCollectionName) == null) {
-              const parentCollection = new CollectionView();
-              parentCollection.name = parentCollectionName;
-              result.collections.push(parentCollection);
-            }
-          }
+          // here processFolder is used to create collections
+          // In an Organization folders are converted to collections
+          // see line just before this function terminates
+          // where all folders are turned to collections
+          this.processFolder(result, col);
         });
       } else if (!this.organization) {
         this.processFolder(result, value.folder);
@@ -125,6 +100,10 @@ export class BitwardenCsvImporter extends BaseImporter implements Importer {
       result.ciphers.push(cipher);
     });
 
+    if (this.organization) {
+      this.moveFoldersToCollections(result);
+    }
+
     result.success = true;
     return Promise.resolve(result);
   }

libs/importer/src/importers/keeper/keeper-csv-importer.spec.ts

@@ -1,6 +1,9 @@
 import { Utils } from "@bitwarden/common/platform/misc/utils";
 
-import { testData as TestData } from "../spec-data/keeper-csv/testdata.csv";
+import {
+  testData as TestData,
+  testDataMultiCollection,
+} from "../spec-data/keeper-csv/testdata.csv";
 
 import { KeeperCsvImporter } from "./keeper-csv-importer";
 
@@ -121,4 +124,32 @@ describe("Keeper CSV Importer", () => {
     expect(result.collectionRelationships[1]).toEqual([1, 0]);
     expect(result.collectionRelationships[2]).toEqual([2, 1]);
   });
+
+  it("should create collections tree, with child collections and relationships", async () => {
+    importer.organizationId = Utils.newGuid();
+    const result = await importer.parse(testDataMultiCollection);
+    expect(result != null).toBe(true);
+
+    const collections = result.collections;
+    expect(collections).not.toBeNull();
+    expect(collections.length).toBe(3);
+
+    // collection with the cipher
+    const collections1 = collections.shift();
+    expect(collections1.name).toBe("Foo/Baz/Bar");
+
+    //second level collection
+    const collections2 = collections.shift();
+    expect(collections2.name).toBe("Foo/Baz");
+
+    //third level
+    const collections3 = collections.shift();
+    expect(collections3.name).toBe("Foo");
+
+    // [Cipher, Folder]
+    expect(result.collectionRelationships.length).toBe(3);
+    expect(result.collectionRelationships[0]).toEqual([0, 0]);
+    expect(result.collectionRelationships[1]).toEqual([1, 1]);
+    expect(result.collectionRelationships[2]).toEqual([2, 2]);
+  });
 });

libs/importer/src/importers/netwrix/netwrix-passwordsecure-csv-importer.spec.ts

@@ -1,6 +1,9 @@
 import { Utils } from "@bitwarden/common/platform/misc/utils";
 
-import { credentialsData } from "../spec-data/netwrix-csv/login-export.csv";
+import {
+  credentialsData,
+  credentialsDataWithFolders,
+} from "../spec-data/netwrix-csv/login-export.csv";
 
 import { NetwrixPasswordSecureCsvImporter } from "./netwrix-passwordsecure-csv-importer";
 
@@ -88,4 +91,18 @@ describe("Netwrix Password Secure CSV Importer", () => {
     expect(result.collectionRelationships[1]).toEqual([1, 1]);
     expect(result.collectionRelationships[2]).toEqual([2, 0]);
   });
+
+  it("should parse multiple collections", async () => {
+    importer.organizationId = Utils.newGuid();
+    const result = await importer.parse(credentialsDataWithFolders);
+
+    expect(result).not.toBeNull();
+    expect(result.success).toBe(true);
+    expect(result.collections.length).toBe(3);
+    expect(result.collections[0].name).toBe("folder1/folder2/folder3");
+    expect(result.collections[1].name).toBe("folder1/folder2");
+    expect(result.collections[2].name).toBe("folder1");
+    expect(result.collectionRelationships.length).toBe(1);
+    expect(result.collectionRelationships[0]).toEqual([0, 0]);
+  });
 });

libs/importer/src/importers/passsordxp/passwordxp-csv-importer.spec.ts

@@ -4,7 +4,10 @@ import { ImportResult } from "../../models/import-result";
 import { dutchHeaders } from "../spec-data/passwordxp-csv/dutch-headers";
 import { germanHeaders } from "../spec-data/passwordxp-csv/german-headers";
 import { noFolder } from "../spec-data/passwordxp-csv/no-folder.csv";
-import { withFolders } from "../spec-data/passwordxp-csv/passwordxp-with-folders.csv";
+import {
+  withFolders,
+  withMultipleFolders,
+} from "../spec-data/passwordxp-csv/passwordxp-with-folders.csv";
 import { withoutFolders } from "../spec-data/passwordxp-csv/passwordxp-without-folders.csv";
 
 import { PasswordXPCsvImporter } from "./passwordxp-csv-importer";
@@ -167,4 +170,22 @@ describe("PasswordXPCsvImporter", () => {
     expect(collectionRelationship).toEqual([4, 2]);
     collectionRelationship = result.collectionRelationships.shift();
   });
+
+  it("should convert multi-level folders to collections when importing into an organization", async () => {
+    importer.organizationId = "someOrg";
+    const result: ImportResult = await importer.parse(withMultipleFolders);
+    expect(result.success).toBe(true);
+    expect(result.ciphers.length).toBe(5);
+
+    expect(result.collections.length).toBe(3);
+    expect(result.collections[0].name).toEqual("Test Folder");
+    expect(result.collections[1].name).toEqual("Test Folder/Level 2 Folder");
+    expect(result.collections[2].name).toEqual("Test Folder/Level 2 Folder/Level 3 Folder");
+
+    expect(result.collectionRelationships.length).toBe(4);
+    expect(result.collectionRelationships[0]).toEqual([1, 0]);
+    expect(result.collectionRelationships[1]).toEqual([2, 1]);
+    expect(result.collectionRelationships[2]).toEqual([3, 1]);
+    expect(result.collectionRelationships[3]).toEqual([4, 2]);
+  });
 });

libs/importer/src/importers/roboform-csv-importer.spec.ts

@@ -2,7 +2,7 @@ import { CipherType } from "@bitwarden/common/vault/enums";
 
 import { RoboFormCsvImporter } from "./roboform-csv-importer";
 import { data as dataNoFolder } from "./spec-data/roboform-csv/empty-folders";
-import { data as dataFolder } from "./spec-data/roboform-csv/with-folders";
+import { data as dataFolder, dataWithFolderHierarchy } from "./spec-data/roboform-csv/with-folders";
 
 describe("Roboform CSV Importer", () => {
   beforeEach(() => {
@@ -39,4 +39,19 @@ describe("Roboform CSV Importer", () => {
     expect(result.ciphers[4].notes).toBe("This is a safe note");
     expect(result.ciphers[4].name).toBe("note - 2023-03-31");
   });
+
+  it("should parse CSV data with folder hierarchy", async () => {
+    const importer = new RoboFormCsvImporter();
+    const result = await importer.parse(dataWithFolderHierarchy);
+    expect(result != null).toBe(true);
+
+    expect(result.folders.length).toBe(5);
+    expect(result.ciphers.length).toBe(5);
+
+    expect(result.folders[0].name).toBe("folder1");
+    expect(result.folders[1].name).toBe("folder2");
+    expect(result.folders[2].name).toBe("folder2/folder3");
+    expect(result.folders[3].name).toBe("folder1/folder2/folder3");
+    expect(result.folders[4].name).toBe("folder1/folder2");
+  });
 });

libs/importer/src/importers/spec-data/keeper-csv/testdata.csv.ts

@@ -2,3 +2,9 @@ export const testData = `"Foo","Bar","john.doe@example.com","1234567890abcdef","
 "Foo","Bar 1","john.doe1@example.com","234567890abcdef1","https://an.example.com/","","","Account ID","12345","Org ID","54321"
 "Foo\\Baz","Bar 2","john.doe2@example.com","34567890abcdef12","https://another.example.com/","","","Account ID","23456","TFC:Keeper","otpauth://totp/Amazon:me@company.com?secret=JBSWY3DPEHPK3PXP&issuer=Amazon&algorithm=SHA1&digits=6&period=30"
 `;
+
+export const testDataMultiCollection = `
+"Foo\\Baz\\Bar","Bar 2","john.doe2@example.com","34567890abcdef12","https://another.example.com/","","","Account ID","23456","TFC:Keeper","otpauth://totp/Amazon:me@company.com?secret=JBSWY3DPEHPK3PXP&issuer=Amazon&algorithm=SHA1&digits=6&period=30"
+"Foo\\Baz","Bar 2","john.doe2@example.com","34567890abcdef12","https://another.example.com/","","","Account ID","23456","TFC:Keeper","otpauth://totp/Amazon:me@company.com?secret=JBSWY3DPEHPK3PXP&issuer=Amazon&algorithm=SHA1&digits=6&period=30"
+"Foo","Bar 2","john.doe2@example.com","34567890abcdef12","https://another.example.com/","","","Account ID","23456","TFC:Keeper","otpauth://totp/Amazon:me@company.com?secret=JBSWY3DPEHPK3PXP&issuer=Amazon&algorithm=SHA1&digits=6&period=30"
+`;

libs/importer/src/importers/spec-data/netwrix-csv/login-export.csv.ts

@@ -2,3 +2,6 @@
 "folderOrCollection1";"tag1, tag2, tag3";"Test Entry 1";"someUser";"somePassword";"https://www.example.com";"some note for example.com";"someTOTPSeed"
 "folderOrCollection2";"tag2";"Test Entry 2";"jdoe";"})9+Kg2fz_O#W1§H1-<ox>0Zio";"www.123.com";"Description123";"anotherTOTP"
 "folderOrCollection1";"someTag";"Test Entry 3";"username";"password";"www.internetsite.com";"Information";""`;
+
+export const credentialsDataWithFolders = `"Organisationseinheit";"DataTags";"Beschreibung";"Benutzername";"Passwort";"Internetseite";"Informationen";"One-Time Passwort"
+"folder1\\folder2\\folder3";"tag1, tag2, tag3";"Test Entry 1";"someUser";"somePassword";"https://www.example.com";"some note for example.com";"someTOTPSeed"`;

libs/importer/src/importers/spec-data/passwordxp-csv/passwordxp-with-folders.csv.ts

@@ -11,3 +11,17 @@ test;testtest;;http://test;test;27-3-2024 12:36:59;27-3-2024 12:36:59;;;
 
 [Cert folder\\Nested folder];
 test2;testtest;;http://test;test;27-3-2024 12:36:59;27-3-2024 12:36:59;;;`;
+
+export const withMultipleFolders = `Title;User name;Account;URL;Password;Modified;Created;Expire on;Description;Modified by
+>>>
+Title2;Username2;Account2;http://URL2.com;12345678;27-3-2024 08:11:21;27-3-2024 08:11:21;;;
+
+[Test Folder]
+Title Test 1;Username1;Account1;http://URL1.com;Password1;27-3-2024 08:10:52;27-3-2024 08:10:52;;;
+
+[Test Folder\\Level 2 Folder]
+Certificate 1;;;;;27-3-2024 10:22:39;27-3-2024 10:22:39;;;
+test;testtest;;http://test;test;27-3-2024 12:36:59;27-3-2024 12:36:59;;;
+
+[Test Folder\\Level 2 Folder\\Level 3 Folder]
+test2;testtest;;http://test;test;27-3-2024 12:36:59;27-3-2024 12:36:59;;;`;

libs/importer/src/importers/spec-data/roboform-csv/with-folders.ts

@@ -4,3 +4,10 @@ Test,https://www.test.com/,https://www.test.com/,test@gmail.com,:testPassword,te
 LoginWebsite,https://login.Website.com/,https://login.Website.com/,test@outlook.com,123password,,folder2,"User ID$,,,txt,test@outlook.com","Password$,,,pwd,123password"
 Website,https://signin.website.com/,https://signin.website.com/,user@bitwarden.com,password123,Website ,folder3,"User ID$,,,txt,user@bitwarden.com","Password$,,,pwd,password123"
 note - 2023-03-31,,,,,This is a safe note,`;
+
+export const dataWithFolderHierarchy = `Name,Url,MatchUrl,Login,Pwd,Note,Folder,RfFieldsV2
+Bitwarden,https://bitwarden.com,https://bitwarden.com,user@bitwarden.com,password,,folder1,"User ID$,,,txt,user@bitwarden.com","Password$,,,pwd,password"
+Test,https://www.test.com/,https://www.test.com/,test@gmail.com,:testPassword,test,folder1,"User ID$,,,txt,test@gmail.com","Password$,,,pwd,:testPassword"
+LoginWebsite,https://login.Website.com/,https://login.Website.com/,test@outlook.com,123password,,folder2,"User ID$,,,txt,test@outlook.com","Password$,,,pwd,123password"
+Website,https://signin.website.com/,https://signin.website.com/,user@bitwarden.com,password123,Website ,folder2\\folder3,"User ID$,,,txt,user@bitwarden.com","Password$,,,pwd,password123"
+Website,https://signin.website.com/,https://signin.website.com/,user@bitwarden.com,password123,Website ,folder1\\folder2\\folder3,"User ID$,,,txt,user@bitwarden.com","Password$,,,pwd,password123"`;