feat(common): support simultaneous imports of collections and environment files (#3719)

packages/hoppscotch-common/locales/en.json

@@ -315,7 +315,8 @@
     "proxy_error": "Proxy error",
     "script_fail": "Could not execute pre-request script",
     "something_went_wrong": "Something went wrong",
-    "test_script_fail": "Could not execute post-request script"
+    "test_script_fail": "Could not execute post-request script",
+    "reading_files": "Error while reading one or more files."
   },
   "export": {
     "as_json": "Export as JSON",
@@ -413,7 +414,10 @@
     "json_description": "Import collections from a Hoppscotch Collections JSON file",
     "postman_environment": "Postman Environment",
     "postman_environment_description": "Import Postman Environment from a JSON file",
-    "title": "Import"
+    "title": "Import",
+    "file_size_limit_exceeded_warning_multiple_files": "Chosen files exceed the recommended limit of 10MB. Only the first {files} selected will be imported",
+    "file_size_limit_exceeded_warning_single_file": "The currently chosen file exceeds the recommended limit of 10MB. Please select another file.",
+    "success": "Successfully imported"
   },
   "inspections": {
     "description": "Inspect possible errors",

packages/hoppscotch-common/src/components/collections/ImportExport.vue

@@ -263,7 +263,7 @@ const HoppOpenAPIImporter: ImporterOrExporter = {
       step: UrlSource({
         caption: "import.from_url",
         onImportFromURL: async (content) => {
-          const res = await hoppOpenAPIImporter(content)()
+          const res = await hoppOpenAPIImporter([content])()
 
           if (E.isRight(res)) {
             handleImportToStore(res.right)

packages/hoppscotch-common/src/components/collections/MyCollections.vue

@@ -694,7 +694,7 @@ class MyCollectionsAdapter implements SmartTreeAdapter<MyCollectionNode> {
     let target = collections[indexPaths.shift() as number]
 
     while (indexPaths.length > 0)
-      target = target.folders[indexPaths.shift() as number]
+      target = target?.folders[indexPaths.shift() as number]
 
     return target !== undefined ? target : null
   }

packages/hoppscotch-common/src/components/environments/ImportExport.vue

@@ -133,7 +133,7 @@ const PostmanEnvironmentsImport: ImporterOrExporter = {
         return
       }
 
-      handleImportToStore([res.right])
+      handleImportToStore(res.right)
 
       platform.analytics?.logEvent({
         type: "HOPP_IMPORT_ENVIRONMENT",
@@ -166,19 +166,14 @@ const insomniaEnvironmentsImport: ImporterOrExporter = {
         return
       }
 
-      const globalEnvIndex = res.right.findIndex(
+      const globalEnvs = res.right.filter(
         (env) => env.name === "Base Environment"
       )
+      const otherEnvs = res.right.filter(
+        (env) => env.name !== "Base Environment"
+      )
 
-      const globalEnv =
-        globalEnvIndex !== -1 ? res.right[globalEnvIndex] : undefined
-
-      // remove the global env from the environments array to prevent it from being imported twice
-      if (globalEnvIndex !== -1) {
-        res.right.splice(globalEnvIndex, 1)
-      }
-
-      handleImportToStore(res.right, globalEnv)
+      handleImportToStore(otherEnvs, globalEnvs)
 
       platform.analytics?.logEvent({
         type: "HOPP_IMPORT_ENVIRONMENT",
@@ -340,14 +335,14 @@ const showImportFailedError = () => {
 
 const handleImportToStore = async (
   environments: Environment[],
-  globalEnv?: NonSecretEnvironment
+  globalEnvs: NonSecretEnvironment[] = []
 ) => {
-  // if there's a global env, add them to the store
-  if (globalEnv) {
-    globalEnv.variables.forEach(({ key, value, secret }) =>
+  // Add global envs to the store
+  globalEnvs.forEach(({ variables }) => {
+    variables.forEach(({ key, value, secret }) => {
       addGlobalEnvVariable({ key, value, secret })
-    )
-  }
+    })
+  })
 
   if (props.environmentType === "MY_ENV") {
     appendEnvironments(environments)

packages/hoppscotch-common/src/components/importExport/ImportExportSteps/FileImport.vue

@@ -13,6 +13,7 @@
         {{ t(`${caption}`) }}
       </span>
     </p>
+
     <div
       class="flex flex-col ml-10 border border-dashed rounded border-dividerDark"
     >
@@ -23,15 +24,30 @@
         type="file"
         class="p-4 cursor-pointer transition file:transition file:cursor-pointer text-secondary hover:text-secondaryDark file:mr-2 file:py-2 file:px-4 file:rounded file:border-0 file:text-secondary hover:file:text-secondaryDark file:bg-primaryLight hover:file:bg-primaryDark"
         :accept="acceptedFileTypes"
+        multiple
         @change="onFileChange"
       />
     </div>
 
+    <p v-if="showFileSizeLimitExceededWarning" class="text-red-500 ml-10">
+      <template v-if="importFilesCount">
+        {{
+          t("import.file_size_limit_exceeded_warning_multiple_files", {
+            files:
+              importFilesCount === 1 ? "file" : `${importFilesCount} files`,
+          })
+        }}
+      </template>
+
+      <template v-else>
+        {{ t("import.file_size_limit_exceeded_warning_single_file") }}
+      </template>
+    </p>
     <div>
       <HoppButtonPrimary
         class="w-full"
         :label="t('import.title')"
-        :disabled="!hasFile"
+        :disabled="!hasFile || showFileSizeLimitExceededWarning"
         @click="emit('importFromFile', fileContent)"
       />
     </div>
@@ -51,16 +67,30 @@ defineProps<{
 const t = useI18n()
 const toast = useToast()
 
+const ALLOWED_FILE_SIZE_LIMIT = 10 // 10 MB
+
+const importFilesCount = ref(0)
+
 const hasFile = ref(false)
-const fileContent = ref("")
+const showFileSizeLimitExceededWarning = ref(false)
+const fileContent = ref<string[]>([])
 
 const inputChooseFileToImportFrom = ref<HTMLInputElement | any>()
 
 const emit = defineEmits<{
-  (e: "importFromFile", content: string): void
+  (e: "importFromFile", content: string[]): void
 }>()
 
-const onFileChange = () => {
+const onFileChange = async () => {
+  // Reset the state on entering the handler to avoid any stale state
+  if (showFileSizeLimitExceededWarning.value) {
+    showFileSizeLimitExceededWarning.value = false
+  }
+
+  if (importFilesCount.value) {
+    importFilesCount.value = 0
+  }
+
   const inputFileToImport = inputChooseFileToImportFrom.value
 
   if (!inputFileToImport) {
@@ -69,27 +99,52 @@ const onFileChange = () => {
   }
 
   if (!inputFileToImport.files || inputFileToImport.files.length === 0) {
-    inputChooseFileToImportFrom.value[0].value = ""
+    inputChooseFileToImportFrom.value = ""
     hasFile.value = false
     toast.show(t("action.choose_file").toString())
     return
   }
 
-  const reader = new FileReader()
+  const readerPromises: Promise<string | null>[] = []
 
-  reader.onload = ({ target }) => {
-    const content = target!.result as string | null
-    if (!content) {
-      hasFile.value = false
-      toast.show(t("action.choose_file").toString())
-      return
+  let totalFileSize = 0
+
+  for (let i = 0; i < inputFileToImport.files.length; i++) {
+    const file = inputFileToImport.files[i]
+
+    totalFileSize += file.size / 1024 / 1024
+
+    if (totalFileSize > ALLOWED_FILE_SIZE_LIMIT) {
+      showFileSizeLimitExceededWarning.value = true
+      break
     }
 
-    fileContent.value = content
+    const reader = new FileReader()
 
-    hasFile.value = !!content?.length
+    readerPromises.push(
+      new Promise((resolve, reject) => {
+        reader.onload = () => resolve(reader.result as string | null)
+        reader.onerror = reject
+        reader.readAsText(file)
+      })
+    )
   }
 
-  reader.readAsText(inputFileToImport.files[0])
+  importFilesCount.value = readerPromises.length
+
+  const results = await Promise.allSettled(readerPromises)
+
+  const contentsArr = results
+    .filter((result) => result.status === "fulfilled")
+    .map((result) => (result as { value: string | null }).value)
+    .filter(Boolean) as string[]
+
+  const errors = results.filter((result) => result.status === "rejected")
+  if (errors.length) {
+    toast.error(t("error.reading_files"))
+  }
+
+  fileContent.value = contentsArr
+  hasFile.value = contentsArr.length > 0
 }
 </script>

packages/hoppscotch-common/src/helpers/import-export/import/hopp.ts

@@ -2,6 +2,7 @@ import { pipe, flow } from "fp-ts/function"
 import * as TE from "fp-ts/TaskEither"
 import * as O from "fp-ts/Option"
 import * as RA from "fp-ts/ReadonlyArray"
+import * as A from "fp-ts/Array"
 import { translateToNewRESTCollection, HoppCollection } from "@hoppscotch/data"
 import { isPlainObject as _isPlainObject } from "lodash-es"
 
@@ -9,11 +10,13 @@ import { IMPORTER_INVALID_FILE_FORMAT } from "."
 import { safeParseJSON } from "~/helpers/functional/json"
 import { translateToNewGQLCollection } from "@hoppscotch/data"
 
-export const hoppRESTImporter = (content: string) =>
+export const hoppRESTImporter = (content: string[]) =>
   pipe(
-    safeParseJSON(content),
+    content,
+    A.traverse(O.Applicative)((str) => safeParseJSON(str, true)),
     O.chain(
       flow(
+        A.flatten,
         makeCollectionsArray,
         RA.map(validateCollection),
         O.sequenceArray,

packages/hoppscotch-common/src/helpers/import-export/import/hoppEnv.ts

@@ -8,17 +8,35 @@ import { IMPORTER_INVALID_FILE_FORMAT } from "."
 import { Environment } from "@hoppscotch/data"
 import { z } from "zod"
 
-export const hoppEnvImporter = (content: string) => {
-  const parsedContent = safeParseJSON(content, true)
+export const hoppEnvImporter = (contents: string[]) => {
+  const parsedContents = contents.map((str) => safeParseJSON(str, true))
 
-  // parse json from the environments string
-  if (O.isNone(parsedContent)) {
+  if (parsedContents.some((parsed) => O.isNone(parsed))) {
     return TE.left(IMPORTER_INVALID_FILE_FORMAT)
   }
 
+  const parsedValues = parsedContents.flatMap((content) => {
+    const unwrappedContent = O.toNullable(content) as Environment[] | null
+
+    if (unwrappedContent) {
+      return unwrappedContent.map((contentEntry) => {
+        return {
+          ...contentEntry,
+          variables: contentEntry.variables?.map((valueEntry) => ({
+            ...valueEntry,
+            ...("value" in valueEntry
+              ? { value: String(valueEntry.value) }
+              : {}),
+          })),
+        }
+      })
+    }
+    return null
+  })
+
   const validationResult = z
     .array(entityReference(Environment))
-    .safeParse(parsedContent.value)
+    .safeParse(parsedValues)
 
   if (!validationResult.success) {
     return TE.left(IMPORTER_INVALID_FILE_FORMAT)

packages/hoppscotch-common/src/helpers/import-export/import/hoppGql.ts

@@ -3,10 +3,10 @@ import * as E from "fp-ts/Either"
 
 // TODO: add zod validation
 export const hoppGqlCollectionsImporter = (
-  content: string
+  contents: string[]
 ): E.Either<"INVALID_JSON", HoppCollection[]> => {
   return E.tryCatch(
-    () => JSON.parse(content) as HoppCollection[],
+    () => contents.flatMap((content) => JSON.parse(content)),
     () => "INVALID_JSON"
   )
 }

packages/hoppscotch-common/src/helpers/import-export/import/import-sources/FileSource.ts

@@ -6,7 +6,7 @@ import { v4 as uuidv4 } from "uuid"
 export function FileSource(metadata: {
   acceptedFileTypes: string
   caption: string
-  onImportFromFile: (content: string) => any | Promise<any>
+  onImportFromFile: (content: string[]) => any | Promise<any>
 }) {
   const stepID = uuidv4()
 

packages/hoppscotch-common/src/helpers/import-export/import/import-sources/GistSource.ts

@@ -10,14 +10,14 @@ import { v4 as uuidv4 } from "uuid"
 export function GistSource(metadata: {
   caption: string
   onImportFromGist: (
-    importResult: E.Either<string, string>
+    importResult: E.Either<string, string[]>
   ) => any | Promise<any>
 }) {
   const stepID = uuidv4()
 
   return defineStep(stepID, UrlImport, () => ({
     caption: metadata.caption,
-    onImportFromURL: (gistResponse) => {
+    onImportFromURL: (gistResponse: Record<string, unknown>) => {
       const fileSchema = z.object({
         files: z.record(z.object({ content: z.string() })),
       })
@@ -29,9 +29,11 @@ export function GistSource(metadata: {
         return
       }
 
-      const content = Object.values(parseResult.data.files)[0].content
+      const contents = Object.values(parseResult.data.files).map(
+        ({ content }) => content
+      )
 
-      metadata.onImportFromGist(E.right(content))
+      metadata.onImportFromGist(E.right(contents))
     },
     fetchLogic: fetchGistFromUrl,
   }))

packages/hoppscotch-common/src/helpers/import-export/import/insomnia.ts

@@ -1,19 +1,21 @@
-import { convert, ImportRequest } from "insomnia-importers"
-import { pipe } from "fp-ts/function"
 import {
+  HoppCollection,
   HoppRESTAuth,
   HoppRESTHeader,
   HoppRESTParam,
   HoppRESTReqBody,
   HoppRESTRequest,
   knownContentTypes,
-  makeRESTRequest,
-  HoppCollection,
   makeCollection,
+  makeRESTRequest,
 } from "@hoppscotch/data"
+
 import * as A from "fp-ts/Array"
-import * as TO from "fp-ts/TaskOption"
 import * as TE from "fp-ts/TaskEither"
+import * as TO from "fp-ts/TaskOption"
+import { pipe } from "fp-ts/function"
+import { ImportRequest, convert } from "insomnia-importers"
+
 import { IMPORTER_INVALID_FILE_FORMAT } from "."
 import { replaceInsomniaTemplating } from "./insomniaEnv"
 
@@ -203,15 +205,18 @@ const getHoppFolder = (
     headers: [],
   })
 
-const getHoppCollections = (doc: InsomniaDoc) =>
-  getFoldersIn(null, doc.data.resources).map((f) =>
-    getHoppFolder(f, doc.data.resources)
-  )
+const getHoppCollections = (docs: InsomniaDoc[]) => {
+  return docs.flatMap((doc) => {
+    return getFoldersIn(null, doc.data.resources).map((f) =>
+      getHoppFolder(f, doc.data.resources)
+    )
+  })
+}
 
-export const hoppInsomniaImporter = (fileContent: string) =>
+export const hoppInsomniaImporter = (fileContents: string[]) =>
   pipe(
-    fileContent,
-    parseInsomniaDoc,
+    fileContents,
+    A.traverse(TO.ApplicativeSeq)(parseInsomniaDoc),
     TO.map(getHoppCollections),
     TE.fromTaskOption(() => IMPORTER_INVALID_FILE_FORMAT)
   )

packages/hoppscotch-common/src/helpers/import-export/import/insomniaEnv.ts

@@ -29,33 +29,36 @@ export const replaceInsomniaTemplating = (expression: string) => {
   return expression.replaceAll(regex, "<<$1>>")
 }
 
-export const insomniaEnvImporter = (content: string) => {
-  const parsedContent = safeParseJSONOrYAML(content)
-
-  if (O.isNone(parsedContent)) {
+export const insomniaEnvImporter = (contents: string[]) => {
+  const parsedContents = contents.map((str) => safeParseJSONOrYAML(str))
+  if (parsedContents.some((parsed) => O.isNone(parsed))) {
     return TE.left(IMPORTER_INVALID_FILE_FORMAT)
   }
 
-  const validationResult = insomniaResourcesSchema.safeParse(
-    parsedContent.value
-  )
+  const parsedValues = parsedContents.map((parsed) => O.toNullable(parsed))
+
+  const validationResult = z
+    .array(insomniaResourcesSchema)
+    .safeParse(parsedValues)
 
   if (!validationResult.success) {
     return TE.left(IMPORTER_INVALID_FILE_FORMAT)
   }
 
-  const insomniaEnvs = validationResult.data.resources
-    .filter((resource) => resource._type === "environment")
-    .map((envResource) => {
-      const envResourceData = envResource.data as Record<string, unknown>
-      const stringifiedData: Record<string, string> = {}
+  const insomniaEnvs = validationResult.data.flatMap(({ resources }) => {
+    return resources
+      .filter((resource) => resource._type === "environment")
+      .map((envResource) => {
+        const envResourceData = envResource.data as Record<string, unknown>
+        const stringifiedData: Record<string, string> = {}
 
-      Object.keys(envResourceData).forEach((key) => {
-        stringifiedData[key] = String(envResourceData[key])
+        Object.keys(envResourceData).forEach((key) => {
+          stringifiedData[key] = String(envResourceData[key])
+        })
+
+        return { ...envResource, data: stringifiedData }
       })
-
-      return { ...envResource, data: stringifiedData }
-    })
+  })
 
   const environments: NonSecretEnvironment[] = []
 

packages/hoppscotch-common/src/helpers/import-export/import/openapi.ts

@@ -584,24 +584,28 @@ const convertPathToHoppReqs = (
     RA.toArray
   )
 
-const convertOpenApiDocToHopp = (
-  doc: OpenAPI.Document
+const convertOpenApiDocsToHopp = (
+  docs: OpenAPI.Document[]
 ): TE.TaskEither<never, HoppCollection[]> => {
-  const name = doc.info.title
+  const collections = docs.map((doc) => {
+    const name = doc.info.title
 
-  const paths = Object.entries(doc.paths ?? {})
-    .map(([pathName, pathObj]) => convertPathToHoppReqs(doc, pathName, pathObj))
-    .flat()
+    const paths = Object.entries(doc.paths ?? {})
+      .map(([pathName, pathObj]) =>
+        convertPathToHoppReqs(doc, pathName, pathObj)
+      )
+      .flat()
 
-  return TE.of([
-    makeCollection({
+    return makeCollection({
       name,
       folders: [],
       requests: paths,
       auth: { authType: "inherit", authActive: true },
       headers: [],
-    }),
-  ])
+    })
+  })
+
+  return TE.of(collections)
 }
 
 const parseOpenAPIDocContent = (str: string) =>
@@ -614,29 +618,49 @@ const parseOpenAPIDocContent = (str: string) =>
     )
   )
 
-export const hoppOpenAPIImporter = (fileContent: string) =>
+export const hoppOpenAPIImporter = (fileContents: string[]) =>
   pipe(
     // See if we can parse JSON properly
-    fileContent,
-    parseOpenAPIDocContent,
-    TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT),
+    fileContents,
+    A.traverse(O.Applicative)(parseOpenAPIDocContent),
+    TE.fromOption(() => {
+      return IMPORTER_INVALID_FILE_FORMAT
+    }),
     // Try validating, else the importer is invalid file format
-    TE.chainW((obj) =>
-      pipe(
+    TE.chainW((docArr) => {
+      return pipe(
         TE.tryCatch(
-          () => SwaggerParser.validate(obj),
+          async () => {
+            const resultDoc = []
+
+            for (const docObj of docArr) {
+              const validatedDoc = await SwaggerParser.validate(docObj)
+              resultDoc.push(validatedDoc)
+            }
+
+            return resultDoc
+          },
           () => IMPORTER_INVALID_FILE_FORMAT
         )
       )
-    ),
+    }),
     // Deference the references
-    TE.chainW((obj) =>
+    TE.chainW((docArr) =>
       pipe(
         TE.tryCatch(
-          () => SwaggerParser.dereference(obj),
+          async () => {
+            const resultDoc = []
+
+            for (const docObj of docArr) {
+              const validatedDoc = await SwaggerParser.dereference(docObj)
+              resultDoc.push(validatedDoc)
+            }
+
+            return resultDoc
+          },
           () => OPENAPI_DEREF_ERROR
         )
       )
     ),
-    TE.chainW(convertOpenApiDocToHopp)
+    TE.chainW(convertOpenApiDocsToHopp)
   )

packages/hoppscotch-common/src/helpers/import-export/import/postman.ts

@@ -55,7 +55,11 @@ const readPMCollection = (def: string) =>
   pipe(
     def,
     safeParseJSON,
-    O.chain((data) => O.tryCatch(() => new PMCollection(data)))
+    O.chain((data) =>
+      O.tryCatch(() => {
+        return new PMCollection(data)
+      })
+    )
   )
 
 const getHoppReqHeaders = (item: Item): HoppRESTHeader[] =>
@@ -296,15 +300,17 @@ const getHoppFolder = (ig: ItemGroup<Item>): HoppCollection =>
     headers: [],
   })
 
-export const getHoppCollection = (coll: PMCollection) => getHoppFolder(coll)
+export const getHoppCollections = (collections: PMCollection[]) => {
+  return collections.map(getHoppFolder)
+}
 
-export const hoppPostmanImporter = (fileContent: string) =>
+export const hoppPostmanImporter = (fileContents: string[]) =>
   pipe(
     // Try reading
-    fileContent,
-    readPMCollection,
+    fileContents,
+    A.traverse(O.Applicative)(readPMCollection),
 
-    O.map(flow(getHoppCollection, A.of)),
+    O.map(flow(getHoppCollections)),
 
     TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT)
   )

packages/hoppscotch-common/src/helpers/import-export/import/postmanEnv.ts

@@ -1,12 +1,11 @@
-import * as TE from "fp-ts/TaskEither"
-import * as O from "fp-ts/Option"
-
-import { IMPORTER_INVALID_FILE_FORMAT } from "."
-import { safeParseJSON } from "~/helpers/functional/json"
-
-import { z } from "zod"
 import { Environment } from "@hoppscotch/data"
+import * as O from "fp-ts/Option"
+import * as TE from "fp-ts/TaskEither"
 import { uniqueId } from "lodash-es"
+import { z } from "zod"
+
+import { safeParseJSON } from "~/helpers/functional/json"
+import { IMPORTER_INVALID_FILE_FORMAT } from "."
 
 const postmanEnvSchema = z.object({
   name: z.string(),
@@ -18,32 +17,44 @@ const postmanEnvSchema = z.object({
   ),
 })
 
-export const postmanEnvImporter = (content: string) => {
-  const parsedContent = safeParseJSON(content)
+type PostmanEnv = z.infer<typeof postmanEnvSchema>
 
-  // parse json from the environments string
-  if (O.isNone(parsedContent)) {
+export const postmanEnvImporter = (contents: string[]) => {
+  const parsedContents = contents.map((str) => safeParseJSON(str, true))
+  if (parsedContents.some((parsed) => O.isNone(parsed))) {
     return TE.left(IMPORTER_INVALID_FILE_FORMAT)
   }
 
-  const validationResult = postmanEnvSchema.safeParse(parsedContent.value)
+  const parsedValues = parsedContents.flatMap((parsed) => {
+    const unwrappedEntry = O.toNullable(parsed) as PostmanEnv[] | null
+
+    if (unwrappedEntry) {
+      return unwrappedEntry.map((entry) => ({
+        ...entry,
+        values: entry.values?.map((valueEntry) => ({
+          ...valueEntry,
+          value: String(valueEntry.value),
+        })),
+      }))
+    }
+    return null
+  })
+
+  const validationResult = z.array(postmanEnvSchema).safeParse(parsedValues)
 
   if (!validationResult.success) {
     return TE.left(IMPORTER_INVALID_FILE_FORMAT)
   }
 
-  const postmanEnv = validationResult.data
-
-  const environment: Environment = {
-    id: uniqueId(),
-    v: 1,
-    name: postmanEnv.name,
-    variables: [],
-  }
-
-  postmanEnv.values.forEach(({ key, value }) =>
-    environment.variables.push({ key, value, secret: false })
+  // Convert `values` to `variables` to match the format expected by the system
+  const environments: Environment[] = validationResult.data.map(
+    ({ name, values }) => ({
+      id: uniqueId(),
+      v: 1,
+      name,
+      variables: values.map((entires) => ({ ...entires, secret: false })),
+    })
   )
 
-  return TE.right(environment)
+  return TE.right(environments)
 }

packages/hoppscotch-common/src/newstore/collections.ts

@@ -61,7 +61,7 @@ export function navigateToFolderWithIndexPath(
 
   let target = collections[indexPaths.shift() as number]
 
-  while (indexPaths.length > 0)
+  while (indexPaths.length > 0 && target)
     target = target.folders[indexPaths.shift() as number]
 
   return target !== undefined ? target : null