feat: import collections from URL (#2262)

Co-authored-by: Andrew Bastin <andrewbastin.k@gmail.com>

packages/hoppscotch-app/helpers/functional/object.ts

@@ -1,6 +1,7 @@
 import { pipe } from "fp-ts/function"
 import cloneDeep from "lodash/cloneDeep"
 import isEqual from "lodash/isEqual"
+import { JSPrimitive, TypeFromPrimitive } from "./primtive"
 
 export const objRemoveKey =
   <T, K extends keyof T>(key: K) =>
@@ -19,34 +20,15 @@ export const objFieldMatches =
   (obj: T): obj is T & { [_ in K]: V } =>
     matches.findIndex((x) => isEqual(obj[fieldName], x)) !== -1
 
-type JSPrimitive =
-  | "undefined"
-  | "object"
-  | "boolean"
-  | "number"
-  | "bigint"
-  | "string"
-  | "symbol"
-  | "function"
-
-type TypeFromPrimitive<P extends JSPrimitive | undefined> =
-  P extends "undefined"
-    ? undefined
-    : P extends "object"
-    ? object | null // typeof null === "object"
-    : P extends "boolean"
-    ? boolean
-    : P extends "number"
-    ? number
-    : P extends "bigint"
-    ? BigInt
-    : P extends "string"
-    ? string
-    : P extends "symbol"
-    ? Symbol
-    : P extends "function"
-    ? Function
-    : unknown
+export const objHasProperty =
+  <O extends object, K extends string, P extends JSPrimitive | undefined>(
+    prop: K,
+    type?: P
+  ) =>
+  // eslint-disable-next-line
+  (obj: O): obj is O & { [_ in K]: TypeFromPrimitive<P> } =>
+    // eslint-disable-next-line
+    prop in obj && (type === undefined || typeof (obj as any)[prop] === type)
 
 type TypeFromPrimitiveArray<P extends JSPrimitive | undefined> =
   P extends "undefined"
@@ -67,16 +49,6 @@ type TypeFromPrimitiveArray<P extends JSPrimitive | undefined> =
     ? Function[]
     : unknown[]
 
-export const objHasProperty =
-  <O extends object, K extends string, P extends JSPrimitive | undefined>(
-    prop: K,
-    type?: P
-  ) =>
-  // eslint-disable-next-line
-  (obj: O): obj is O & { [_ in K]: TypeFromPrimitive<P> } =>
-    // eslint-disable-next-line
-    prop in obj && (type === undefined || typeof (obj as any)[prop] === type)
-
 export const objHasArrayProperty =
   <O extends object, K extends string, P extends JSPrimitive>(
     prop: K,

packages/hoppscotch-app/helpers/functional/primtive.ts

@@ -0,0 +1,34 @@
+export type JSPrimitive =
+  | "undefined"
+  | "object"
+  | "boolean"
+  | "number"
+  | "bigint"
+  | "string"
+  | "symbol"
+  | "function"
+
+export type TypeFromPrimitive<P extends JSPrimitive | undefined> =
+  P extends "undefined"
+    ? undefined
+    : P extends "object"
+    ? object | null // typeof null === "object"
+    : P extends "boolean"
+    ? boolean
+    : P extends "number"
+    ? number
+    : P extends "bigint"
+    ? BigInt
+    : P extends "string"
+    ? string
+    : P extends "symbol"
+    ? Symbol
+    : P extends "function"
+    ? Function
+    : unknown
+
+export const isOfType =
+  <T extends JSPrimitive>(type: T) =>
+  (value: unknown): value is T =>
+    // eslint-disable-next-line valid-typeof
+    typeof value === type

packages/hoppscotch-app/helpers/functional/taskEither.ts

@@ -0,0 +1,13 @@
+import * as TE from "fp-ts/TaskEither"
+
+/**
+ * A utility type which gives you the type of the left value of a TaskEither
+ */
+export type TELeftType<T extends TE.TaskEither<any, any>> =
+  T extends TE.TaskEither<
+    infer U,
+    // eslint-disable-next-line
+    infer _
+  >
+    ? U
+    : never

packages/hoppscotch-app/helpers/import-export/import/gist.ts

@@ -10,7 +10,7 @@ import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
 // TODO: Add validation to output
 const fetchGist = (
   url: string
-): TO.TaskOption<HoppCollection<HoppRESTRequest>> =>
+): TO.TaskOption<HoppCollection<HoppRESTRequest>[]> =>
   pipe(
     TO.tryCatch(() =>
       axios.get(`https://api.github.com/gists/${url.split("/").pop()}`, {
@@ -30,8 +30,10 @@ const fetchGist = (
   )
 
 export default defineImporter({
+  id: "gist",
   name: "import.from_gist",
   icon: "github",
+  applicableTo: ["my-collections", "team-collections"],
   steps: [
     step({
       stepName: "URL_IMPORT",

packages/hoppscotch-app/helpers/import-export/import/hopp.ts

@@ -1,13 +1,22 @@
-import { pipe } from "fp-ts/function"
+import { pipe, flow } from "fp-ts/function"
 import * as TE from "fp-ts/TaskEither"
-import * as E from "fp-ts/Either"
-import { translateToNewRESTCollection } from "@hoppscotch/data"
+import * as O from "fp-ts/Option"
+import * as RA from "fp-ts/ReadonlyArray"
+import {
+  translateToNewRESTCollection,
+  HoppCollection,
+  HoppRESTRequest,
+} from "@hoppscotch/data"
+import _isPlainObject from "lodash/isPlainObject"
 import { step } from "../steps"
 import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
+import { safeParseJSON } from "~/helpers/functional/json"
 
 export default defineImporter({
+  id: "hoppscotch",
   name: "import.from_json",
   icon: "folder-plus",
+  applicableTo: ["my-collections", "team-collections", "url-import"],
   steps: [
     step({
       stepName: "FILE_IMPORT",
@@ -19,16 +28,40 @@ export default defineImporter({
   ] as const,
   importer: ([content]) =>
     pipe(
-      E.tryCatch(
-        () => {
-          const x = JSON.parse(content)
-
-          return Array.isArray(x)
-            ? x.map((coll: any) => translateToNewRESTCollection(coll))
-            : [translateToNewRESTCollection(x)]
-        },
-        () => IMPORTER_INVALID_FILE_FORMAT
+      safeParseJSON(content),
+      O.chain(
+        flow(
+          makeCollectionsArray,
+          RA.map(
+            flow(
+              O.fromPredicate(isValidCollection),
+              O.map(translateToNewRESTCollection)
+            )
+          ),
+          O.sequenceArray,
+          O.map(RA.toArray)
+        )
       ),
-      TE.fromEither
+      TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT)
     ),
 })
+
+/**
+ * checks if a value is a plain object
+ */
+const isPlainObject = (value: any): value is object => _isPlainObject(value)
+
+/**
+ * checks if a collection matches the schema for a hoppscotch collection.
+ * as of now we are only checking if the collection has a "v" key in it.
+ */
+const isValidCollection = (
+  collection: unknown
+): collection is HoppCollection<HoppRESTRequest> =>
+  isPlainObject(collection) && "v" in collection
+
+/**
+ * convert single collection object into an array so it can be handled the same as multiple collections
+ */
+const makeCollectionsArray = (collections: unknown | unknown[]): unknown[] =>
+  Array.isArray(collections) ? collections : [collections]

packages/hoppscotch-app/helpers/import-export/import/importers.ts

@@ -13,3 +13,10 @@ export const RESTCollectionImporters = [
   GistImporter,
   MyCollectionsImporter,
 ] as const
+
+export const URLImporters = [
+  HoppRESTCollImporter,
+  OpenAPIImporter,
+  PostmanImporter,
+  InsomniaImporter,
+] as const

packages/hoppscotch-app/helpers/import-export/import/index.ts

@@ -13,10 +13,18 @@ type HoppImporter<T, StepsType, Errors> = (
   stepValues: StepsOutputList<StepsType>
 ) => TE.TaskEither<Errors, T>
 
+type HoppImporterApplicableTo = Array<
+  "team-collections" | "my-collections" | "url-import"
+>
+
 /**
  * Definition for importers
  */
 type HoppImporterDefinition<T, Y, E> = {
+  /**
+   * the id
+   */
+  id: string
   /**
    * Name of the importer, shown on the Select Importer dropdown
    */
@@ -30,7 +38,7 @@ type HoppImporterDefinition<T, Y, E> = {
   /**
    * Identifier for the importer
    */
-  applicableTo?: Array<"team-collections" | "my-collections">
+  applicableTo: HoppImporterApplicableTo
 
   /**
    * The importer function, It is a Promise because its supposed to be loaded in lazily (dynamic imports ?)
@@ -47,10 +55,11 @@ type HoppImporterDefinition<T, Y, E> = {
  * Defines a Hoppscotch importer
  */
 export const defineImporter = <ReturnType, StepType, Errors>(input: {
+  id: string
   name: string
   icon: string
   importer: HoppImporter<ReturnType, StepType, Errors>
-  applicableTo?: Array<"team-collections" | "my-collections">
+  applicableTo: HoppImporterApplicableTo
   steps: StepType
 }) => {
   return <HoppImporterDefinition<ReturnType, StepType, Errors>>{

packages/hoppscotch-app/helpers/import-export/import/insomnia.ts

@@ -210,7 +210,9 @@ const getHoppCollections = (doc: InsomniaDoc) =>
   )
 
 export default defineImporter({
+  id: "insomnia",
   name: "import.from_insomnia",
+  applicableTo: ["my-collections", "team-collections", "url-import"],
   icon: "insomnia",
   steps: [
     step({

packages/hoppscotch-app/helpers/import-export/import/myCollections.ts

@@ -6,6 +6,7 @@ import { defineImporter } from "."
 import { getRESTCollection } from "~/newstore/collections"
 
 export default defineImporter({
+  id: "myCollections",
   name: "import.from_my_collections",
   icon: "user",
   applicableTo: ["team-collections"],

packages/hoppscotch-app/helpers/import-export/import/openapi.ts

@@ -27,7 +27,7 @@ import * as RA from "fp-ts/ReadonlyArray"
 import { step } from "../steps"
 import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
 
-const OPENAPI_DEREF_ERROR = "openapi/deref_error" as const
+export const OPENAPI_DEREF_ERROR = "openapi/deref_error" as const
 
 // TODO: URL Import Support
 
@@ -586,7 +586,9 @@ const parseOpenAPIDocContent = (str: string) =>
   )
 
 export default defineImporter({
+  id: "openapi",
   name: "import.from_openapi",
+  applicableTo: ["my-collections", "team-collections", "url-import"],
   icon: "file",
   steps: [
     step({
@@ -603,7 +605,6 @@ export default defineImporter({
       fileContent,
       parseOpenAPIDocContent,
       TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT),
-
       // Try validating, else the importer is invalid file format
       TE.chainW((obj) =>
         pipe(
@@ -613,7 +614,6 @@ export default defineImporter({
           )
         )
       ),
-
       // Deference the references
       TE.chainW((obj) =>
         pipe(
@@ -623,7 +623,6 @@ export default defineImporter({
           )
         )
       ),
-
       TE.chainW(convertOpenApiDocToHopp)
     ),
 })

packages/hoppscotch-app/helpers/import-export/import/postman.ts

@@ -297,7 +297,9 @@ const getHoppFolder = (ig: ItemGroup<Item>): HoppCollection<HoppRESTRequest> =>
 export const getHoppCollection = (coll: PMCollection) => getHoppFolder(coll)
 
 export default defineImporter({
+  id: "postman",
   name: "import.from_postman",
+  applicableTo: ["my-collections", "team-collections", "url-import"],
   icon: "postman",
   steps: [
     step({

packages/hoppscotch-app/locales/en.json

@@ -266,7 +266,11 @@
     "from_url": "Import from URL",
     "gist_url": "Enter Gist URL",
     "json_description": "Import collections from a Hoppscotch Collections JSON file",
-    "title": "Import"
+    "title": "Import",
+    "import_from_url_success": "Collections Imported",
+    "import_from_url_invalid_file_format": "Error while importing collections",
+    "import_from_url_invalid_type": "Unsupported type. accepted values are 'hoppscotch', 'openapi', 'postman', 'insomnia'",
+    "import_from_url_invalid_fetch": "Couldn't get data from the url"
   },
   "layout": {
     "collapse_collection": "Collapse or Expand Collections",

packages/hoppscotch-app/pages/import.vue

@@ -0,0 +1,102 @@
+<template>
+  <div></div>
+</template>
+
+<script setup lang="ts">
+import axios from "axios"
+import * as TO from "fp-ts/TaskOption"
+import * as TE from "fp-ts/TaskEither"
+import * as RA from "fp-ts/ReadonlyArray"
+
+import { useRoute, useRouter, onMounted } from "@nuxtjs/composition-api"
+import * as E from "fp-ts/Either"
+import { pipe } from "fp-ts/function"
+import { HoppRESTRequest, HoppCollection } from "@hoppscotch/data"
+import { appendRESTCollections } from "~/newstore/collections"
+import { useToast, useI18n } from "~/helpers/utils/composables"
+import { URLImporters } from "~/helpers/import-export/import/importers"
+import { IMPORTER_INVALID_FILE_FORMAT } from "~/helpers/import-export/import"
+import { OPENAPI_DEREF_ERROR } from "~/helpers/import-export/import/openapi"
+import { isOfType } from "~/helpers/functional/primtive"
+import { TELeftType } from "~/helpers/functional/taskEither"
+
+const route = useRoute()
+const router = useRouter()
+const toast = useToast()
+const t = useI18n()
+
+const IMPORTER_INVALID_TYPE = "importer_invalid_type" as const
+const IMPORTER_INVALID_FETCH = "importer_invalid_fetch" as const
+
+const importCollections = (url: unknown, type: unknown) =>
+  pipe(
+    TE.Do,
+    TE.bind("importer", () =>
+      pipe(
+        URLImporters,
+        RA.findFirst(
+          (importer) =>
+            importer.applicableTo.includes("url-import") && importer.id === type
+        ),
+        TE.fromOption(() => IMPORTER_INVALID_TYPE)
+      )
+    ),
+    TE.bindW("content", () =>
+      pipe(
+        url,
+        TO.fromPredicate(isOfType("string")),
+        TO.chain(fetchUrlData),
+        TE.fromTaskOption(() => IMPORTER_INVALID_FETCH)
+      )
+    ),
+    TE.chainW(({ importer, content }) =>
+      pipe(
+        content.data,
+        TO.fromPredicate(isOfType("string")),
+        TE.fromTaskOption(() => IMPORTER_INVALID_FILE_FORMAT),
+        TE.chain((data) => importer.importer([data]))
+      )
+    )
+  )
+
+type ImportCollectionsError = TELeftType<ReturnType<typeof importCollections>>
+
+onMounted(async () => {
+  const { query } = route.value
+
+  const url = query.url
+  const type = query.type
+
+  const result = await importCollections(url, type)()
+
+  pipe(result, E.fold(handleImportFailure, handleImportSuccess))
+
+  router.replace("/")
+})
+
+const IMPORT_ERROR_MAP: Record<ImportCollectionsError, string> = {
+  [IMPORTER_INVALID_TYPE]: "import.import_from_url_invalid_type",
+  [IMPORTER_INVALID_FETCH]: "import.import_from_url_invalid_fetch",
+  [IMPORTER_INVALID_FILE_FORMAT]: "import.import_from_url_invalid_file_format",
+  [OPENAPI_DEREF_ERROR]: "import.import_from_url_invalid_file_format",
+} as const
+
+const handleImportFailure = (error: ImportCollectionsError) => {
+  toast.error(t(IMPORT_ERROR_MAP[error]).toString())
+}
+
+const handleImportSuccess = (
+  collections: HoppCollection<HoppRESTRequest>[]
+) => {
+  appendRESTCollections(collections)
+  toast.success(t("import.import_from_url_success").toString())
+}
+
+const fetchUrlData = (url: string) =>
+  TO.tryCatch(() =>
+    axios.get(url, {
+      responseType: "text",
+      transitional: { forcedJSONParsing: false },
+    })
+  )
+</script>