refactor: revamp the importers & exporters systems to be reused (#3425)

Co-authored-by: jamesgeorge007 <jamesgeorge998001@gmail.com>
This commit is contained in:
Akash K
2023-12-06 21:24:29 +05:30
committed by GitHub
parent d9c75ed79e
commit ab7c29d228
90 changed files with 2399 additions and 1892 deletions

View File

@@ -0,0 +1,5 @@
import { Environment } from "@hoppscotch/data"
export const environmentsExporter = (myEnvironments: Environment[]) => {
return JSON.stringify(myEnvironments, null, 2)
}

View File

@@ -0,0 +1,18 @@
import * as E from "fp-ts/Either"
import { createGist } from "~/helpers/gist"
export const environmentsGistExporter = async (
environmentsJSON: string,
accessToken: string
) => {
const res = await createGist(
environmentsJSON,
"hoppscotch-collections.json",
accessToken
)()
if (E.isLeft(res)) {
return E.left(res.left)
}
return E.right(res.right.data.html_url as string)
}

View File

@@ -0,0 +1,22 @@
import { createGist } from "~/helpers/gist"
import * as E from "fp-ts/Either"
export const collectionsGistExporter = async (
collectionJSON: string,
accessToken: string
) => {
if (!accessToken) {
return E.left("Invalid User")
}
const res = await createGist(
collectionJSON,
"hoppscotch-collections.json",
accessToken
)()
if (E.isLeft(res)) {
return E.left(res.left)
}
return E.right(true)
}

View File

@@ -0,0 +1,7 @@
import { HoppCollection, HoppGQLRequest } from "@hoppscotch/data"
export const gqlCollectionsExporter = (
gqlCollections: HoppCollection<HoppGQLRequest>[]
) => {
return JSON.stringify(gqlCollections, null, 2)
}

View File

@@ -0,0 +1,18 @@
import * as E from "fp-ts/Either"
import { createGist } from "~/helpers/gist"
export const gqlCollectionsGistExporter = async (
gqlCollectionsJSON: string,
accessToken: string
) => {
const res = await createGist(
gqlCollectionsJSON,
"hoppscotch-collections.json",
accessToken
)()
if (E.isLeft(res)) {
return E.left(res.left)
}
return E.right(res.right.data.html_url as string)
}

View File

@@ -1,9 +0,0 @@
import { HoppRESTRequest, HoppCollection } from "@hoppscotch/data"
import { pipe } from "fp-ts/function"
import * as TE from "fp-ts/TaskEither"
import { HoppExporter } from "."
const exporter: HoppExporter<HoppCollection<HoppRESTRequest>> = (content) =>
pipe(content, JSON.stringify, TE.right)
export default exporter

View File

@@ -1,18 +1,32 @@
import * as TE from "fp-ts/TaskEither"
import { HoppRESTRequest, HoppCollection } from "@hoppscotch/data"
import * as E from "fp-ts/Either"
export type HoppExporter<T> = (content: T) => TE.TaskEither<string, string>
/**
* Create a downloadable file from a collection and prompts the user to download it.
* @param collectionJSON - JSON string of the collection
* @param name - Name of the collection set as the file name
*/
export const initializeDownloadCollection = (
collectionJSON: string,
name: string | null
) => {
const file = new Blob([collectionJSON], { type: "application/json" })
const a = document.createElement("a")
const url = URL.createObjectURL(file)
a.href = url
export type HoppExporterDefinition<T> = {
name: string
exporter: () => Promise<HoppExporter<T>>
if (name) {
a.download = `${name}.json`
} else {
a.download = `${url.split("/").pop()!.split("#")[0].split("?")[0]}.json`
}
document.body.appendChild(a)
a.click()
setTimeout(() => {
document.body.removeChild(a)
URL.revokeObjectURL(url)
}, 1000)
return E.right("state.download_started")
}
export const RESTCollectionExporters: HoppExporterDefinition<
HoppCollection<HoppRESTRequest>
>[] = [
{
name: "Hoppscotch REST Collection JSON",
exporter: () => import("./hopp").then((m) => m.default),
},
]

View File

@@ -0,0 +1,7 @@
import { HoppCollection, HoppRESTRequest } from "@hoppscotch/data"
export const myCollectionsExporter = (
myCollections: HoppCollection<HoppRESTRequest>[]
) => {
return JSON.stringify(myCollections, null, 2)
}

View File

@@ -0,0 +1,5 @@
import { getTeamCollectionJSON } from "~/helpers/backend/helpers"
export const teamCollectionsExporter = (teamID: string) => {
return getTeamCollectionJSON(teamID)
}

View File

@@ -1,52 +0,0 @@
import IconGithub from "~icons/lucide/github"
import { pipe } from "fp-ts/function"
import * as TE from "fp-ts/TaskEither"
import * as TO from "fp-ts/TaskOption"
import * as O from "fp-ts/Option"
import axios from "axios"
import { HoppRESTRequest, HoppCollection } from "@hoppscotch/data"
import { step } from "../steps"
import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
// TODO: Add validation to output
const fetchGist = (
url: string
): TO.TaskOption<HoppCollection<HoppRESTRequest>[]> =>
pipe(
TO.tryCatch(() =>
axios.get(`https://api.github.com/gists/${url.split("/").pop()}`, {
headers: {
Accept: "application/vnd.github.v3+json",
},
})
),
TO.chain((res) =>
pipe(
O.tryCatch(() =>
JSON.parse((Object.values(res.data.files)[0] as any).content)
),
TO.fromOption
)
)
)
export default defineImporter({
id: "gist",
name: "import.from_gist",
icon: IconGithub,
applicableTo: ["my-collections", "team-collections"],
steps: [
step({
stepName: "URL_IMPORT",
metadata: {
caption: "import.from_gist_description",
placeholder: "import.gist_url",
},
}),
] as const,
importer: ([content]) =>
pipe(
fetchGist(content),
TE.fromTaskOption(() => IMPORTER_INVALID_FILE_FORMAT)
),
})

View File

@@ -1,4 +1,3 @@
import IconFolderPlus from "~icons/lucide/folder-plus"
import { pipe, flow } from "fp-ts/function"
import * as TE from "fp-ts/TaskEither"
import * as O from "fp-ts/Option"
@@ -9,38 +8,23 @@ import {
HoppRESTRequest,
} from "@hoppscotch/data"
import { isPlainObject as _isPlainObject } from "lodash-es"
import { step } from "../steps"
import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
import { IMPORTER_INVALID_FILE_FORMAT } from "."
import { safeParseJSON } from "~/helpers/functional/json"
export default defineImporter({
id: "hoppscotch",
name: "import.from_json",
icon: IconFolderPlus,
applicableTo: ["my-collections", "team-collections", "url-import"],
steps: [
step({
stepName: "FILE_IMPORT",
metadata: {
caption: "import.from_json_description",
acceptedFileTypes: "application/json",
},
}),
] as const,
importer: ([content]) =>
pipe(
safeParseJSON(content),
O.chain(
flow(
makeCollectionsArray,
RA.map(validateCollection),
O.sequenceArray,
O.map(RA.toArray)
)
),
TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT)
export const hoppRESTImporter = (content: string) =>
pipe(
safeParseJSON(content),
O.chain(
flow(
makeCollectionsArray,
RA.map(validateCollection),
O.sequenceArray,
O.map(RA.toArray)
)
),
})
TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT)
)
/**
* checks if a value is a plain object
@@ -63,9 +47,8 @@ const isValidCollection = (
const validateCollection = (collection: unknown) => {
if (isValidCollection(collection)) {
return O.some(collection)
} else {
return O.some(translateToNewRESTCollection(collection))
}
return O.some(translateToNewRESTCollection(collection))
}
/**

View File

@@ -0,0 +1,37 @@
import * as TE from "fp-ts/TaskEither"
import * as O from "fp-ts/Option"
import { IMPORTER_INVALID_FILE_FORMAT } from "."
import { safeParseJSON } from "~/helpers/functional/json"
import { z } from "zod"
const hoppEnvSchema = z.object({
id: z.string().optional(),
name: z.string(),
variables: z.array(
z.object({
key: z.string(),
value: z.string(),
})
),
})
export const hoppEnvImporter = (content: string) => {
const parsedContent = safeParseJSON(content)
// parse json from the environments string
if (O.isNone(parsedContent)) {
return TE.left(IMPORTER_INVALID_FILE_FORMAT)
}
const validationResult = z.array(hoppEnvSchema).safeParse(parsedContent.value)
if (!validationResult.success) {
return TE.left(IMPORTER_INVALID_FILE_FORMAT)
}
const environments = validationResult.data
return TE.right(environments)
}

View File

@@ -0,0 +1,12 @@
import { HoppCollection, HoppGQLRequest } from "@hoppscotch/data"
import * as E from "fp-ts/Either"
// TODO: add zod validation
export const hoppGqlCollectionsImporter = (
content: string
): E.Either<"INVALID_JSON", HoppCollection<HoppGQLRequest>[]> => {
return E.tryCatch(
() => JSON.parse(content) as HoppCollection<HoppGQLRequest>[],
() => "INVALID_JSON"
)
}

View File

@@ -0,0 +1,18 @@
import FileImportVue from "~/components/importExport/ImportExportSteps/FileImport.vue"
import { defineStep } from "~/composables/step-components"
import { v4 as uuidv4 } from "uuid"
export function FileSource(metadata: {
acceptedFileTypes: string
caption: string
onImportFromFile: (content: string) => any | Promise<any>
}) {
const stepID = uuidv4()
return defineStep(stepID, FileImportVue, () => ({
acceptedFileTypes: metadata.acceptedFileTypes,
caption: metadata.caption,
onImportFromFile: metadata.onImportFromFile,
}))
}

View File

@@ -0,0 +1,45 @@
import axios from "axios"
import UrlImport from "~/components/importExport/ImportExportSteps/UrlImport.vue"
import { defineStep } from "~/composables/step-components"
import * as E from "fp-ts/Either"
import { z } from "zod"
import { v4 as uuidv4 } from "uuid"
export function GistSource(metadata: {
caption: string
onImportFromGist: (
importResult: E.Either<string, string>
) => any | Promise<any>
}) {
const stepID = uuidv4()
return defineStep(stepID, UrlImport, () => ({
caption: metadata.caption,
onImportFromURL: (gistResponse) => {
const fileSchema = z.object({
files: z.record(z.object({ content: z.string() })),
})
const parseResult = fileSchema.safeParse(gistResponse)
if (!parseResult.success) {
metadata.onImportFromGist(E.left("INVALID_GIST"))
return
}
const content = Object.values(parseResult.data.files)[0].content
metadata.onImportFromGist(E.right(content))
},
fetchLogic: fetchGistFromUrl,
}))
}
const fetchGistFromUrl = (url: string) =>
axios.get(`https://api.github.com/gists/${url.split("/").pop()}`, {
headers: {
Accept: "application/vnd.github.v3+json",
},
})

View File

@@ -0,0 +1,21 @@
import UrlImport from "~/components/importExport/ImportExportSteps/UrlImport.vue"
import { defineStep } from "~/composables/step-components"
import { v4 as uuidv4 } from "uuid"
export function UrlSource(metadata: {
caption: string
onImportFromURL: (content: string) => any | Promise<any>
fetchLogic?: (url: string) => Promise<any>
}) {
const stepID = uuidv4()
return defineStep(stepID, UrlImport, () => ({
caption: metadata.caption,
onImportFromURL: (content: unknown) => {
if (typeof content === "string") {
metadata.onImportFromURL(content)
}
},
}))
}

View File

@@ -1,22 +1,5 @@
import HoppRESTCollImporter from "./hopp"
import OpenAPIImporter from "./openapi"
import PostmanImporter from "./postman"
import InsomniaImporter from "./insomnia"
import GistImporter from "./gist"
import MyCollectionsImporter from "./myCollections"
export const RESTCollectionImporters = [
HoppRESTCollImporter,
OpenAPIImporter,
PostmanImporter,
InsomniaImporter,
GistImporter,
MyCollectionsImporter,
] as const
export const URLImporters = [
HoppRESTCollImporter,
OpenAPIImporter,
PostmanImporter,
InsomniaImporter,
] as const
export { hoppRESTImporter } from "./hopp"
export { hoppOpenAPIImporter } from "./openapi"
export { hoppPostmanImporter } from "./postman"
export { hoppInsomniaImporter } from "./insomnia"
export { toTeamsImporter } from "./myCollections"

View File

@@ -1,4 +1,3 @@
import IconInsomnia from "~icons/hopp/insomnia"
import { convert, ImportRequest } from "insomnia-importers"
import { pipe, flow } from "fp-ts/function"
import {
@@ -16,8 +15,7 @@ import * as A from "fp-ts/Array"
import * as S from "fp-ts/string"
import * as TO from "fp-ts/TaskOption"
import * as TE from "fp-ts/TaskEither"
import { step } from "../steps"
import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
import { IMPORTER_INVALID_FILE_FORMAT } from "."
// TODO: Insomnia allows custom prefixes for Bearer token auth, Hoppscotch doesn't. We just ignore the prefix for now
@@ -210,27 +208,10 @@ const getHoppCollections = (doc: InsomniaDoc) =>
getHoppFolder(f, doc.data.resources)
)
export default defineImporter({
id: "insomnia",
name: "import.from_insomnia",
applicableTo: ["my-collections", "team-collections", "url-import"],
icon: IconInsomnia,
steps: [
step({
stepName: "FILE_IMPORT",
metadata: {
caption: "import.from_insomnia_description",
acceptedFileTypes: ".json, .yaml",
},
}),
] as const,
importer: ([fileContent]) =>
pipe(
fileContent,
parseInsomniaDoc,
TO.map(getHoppCollections),
TE.fromTaskOption(() => IMPORTER_INVALID_FILE_FORMAT)
),
})
export const hoppInsomniaImporter = (fileContent: string) =>
pipe(
fileContent,
parseInsomniaDoc,
TO.map(getHoppCollections),
TE.fromTaskOption(() => IMPORTER_INVALID_FILE_FORMAT)
)

View File

@@ -1,23 +1,5 @@
import IconUser from "~icons/lucide/user"
import * as TE from "fp-ts/TaskEither"
import * as A from "fp-ts/Array"
import { pipe } from "fp-ts/function"
import { step } from "../steps"
import { defineImporter } from "."
import { getRESTCollection } from "~/newstore/collections"
import { importJSONToTeam } from "~/helpers/backend/mutations/TeamCollection"
export default defineImporter({
id: "myCollections",
name: "import.from_my_collections",
icon: IconUser,
applicableTo: ["team-collections"],
steps: [
step({
stepName: "TARGET_MY_COLLECTION",
metadata: {
caption: "import.from_my_collections_description",
},
}),
] as const,
importer: ([content]) => pipe(content, getRESTCollection, A.of, TE.of),
})
export function toTeamsImporter(content: string, teamID: string) {
return importJSONToTeam(content, teamID)
}

View File

@@ -1,4 +1,3 @@
import IconOpenAPI from "~icons/lucide/file"
import {
OpenAPI,
OpenAPIV2,
@@ -25,8 +24,7 @@ import * as S from "fp-ts/string"
import * as O from "fp-ts/Option"
import * as TE from "fp-ts/TaskEither"
import * as RA from "fp-ts/ReadonlyArray"
import { step } from "../steps"
import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
import { IMPORTER_INVALID_FILE_FORMAT } from "."
export const OPENAPI_DEREF_ERROR = "openapi/deref_error" as const
@@ -167,14 +165,12 @@ const parseOpenAPIV3BodyFormData = (
contentType,
body: keys.map((key) => `${key}: `).join("\n"),
}
} else {
return {
contentType,
body: keys.map(
(key) =>
<FormDataKeyValue>{ key, value: "", isFile: false, active: true }
),
}
}
return {
contentType,
body: keys.map(
(key) => <FormDataKeyValue>{ key, value: "", isFile: false, active: true }
),
}
}
@@ -233,10 +229,9 @@ const resolveOpenAPIV3SecurityObj = (
} else if (scheme.scheme === "bearer") {
// Bearer
return { authType: "bearer", authActive: true, token: "" }
} else {
// Unknown/Unsupported Scheme
return { authType: "none", authActive: true }
}
// Unknown/Unsupported Scheme
return { authType: "none", authActive: true }
} else if (scheme.type === "apiKey") {
if (scheme.in === "header") {
return {
@@ -301,17 +296,16 @@ const resolveOpenAPIV3SecurityObj = (
scope: _schemeData.join(" "),
token: "",
}
} else {
return {
authType: "oauth-2",
authActive: true,
accessTokenURL: "",
authURL: "",
clientID: "",
oidcDiscoveryURL: "",
scope: _schemeData.join(" "),
token: "",
}
}
return {
authType: "oauth-2",
authActive: true,
accessTokenURL: "",
authURL: "",
clientID: "",
oidcDiscoveryURL: "",
scope: _schemeData.join(" "),
token: "",
}
} else if (scheme.type === "openIdConnect") {
return {
@@ -339,7 +333,7 @@ const resolveOpenAPIV3SecurityScheme = (
| undefined
if (!scheme) return { authType: "none", authActive: true }
else return resolveOpenAPIV3SecurityObj(scheme, schemeData)
return resolveOpenAPIV3SecurityObj(scheme, schemeData)
}
const resolveOpenAPIV3Security = (
@@ -439,17 +433,16 @@ const resolveOpenAPIV2SecurityScheme = (
scope: _schemeData.join(" "),
token: "",
}
} else {
return {
authType: "oauth-2",
authActive: true,
accessTokenURL: "",
authURL: "",
clientID: "",
oidcDiscoveryURL: "",
scope: _schemeData.join(" "),
token: "",
}
}
return {
authType: "oauth-2",
authActive: true,
accessTokenURL: "",
authURL: "",
clientID: "",
oidcDiscoveryURL: "",
scope: _schemeData.join(" "),
token: "",
}
}
@@ -614,44 +607,29 @@ const parseOpenAPIDocContent = (str: string) =>
)
)
export default defineImporter({
id: "openapi",
name: "import.from_openapi",
applicableTo: ["my-collections", "team-collections", "url-import"],
icon: IconOpenAPI,
steps: [
step({
stepName: "FILE_IMPORT",
metadata: {
caption: "import.from_openapi_description",
acceptedFileTypes: ".json, .yaml, .yml",
},
}),
] as const,
importer: ([fileContent]) =>
pipe(
// See if we can parse JSON properly
fileContent,
parseOpenAPIDocContent,
TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT),
// Try validating, else the importer is invalid file format
TE.chainW((obj) =>
pipe(
TE.tryCatch(
() => SwaggerParser.validate(obj),
() => IMPORTER_INVALID_FILE_FORMAT
)
export const hoppOpenAPIImporter = (fileContent: string) =>
pipe(
// See if we can parse JSON properly
fileContent,
parseOpenAPIDocContent,
TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT),
// Try validating, else the importer is invalid file format
TE.chainW((obj) =>
pipe(
TE.tryCatch(
() => SwaggerParser.validate(obj),
() => IMPORTER_INVALID_FILE_FORMAT
)
),
// Deference the references
TE.chainW((obj) =>
pipe(
TE.tryCatch(
() => SwaggerParser.dereference(obj),
() => OPENAPI_DEREF_ERROR
)
)
),
TE.chainW(convertOpenApiDocToHopp)
)
),
})
// Deference the references
TE.chainW((obj) =>
pipe(
TE.tryCatch(
() => SwaggerParser.dereference(obj),
() => OPENAPI_DEREF_ERROR
)
)
),
TE.chainW(convertOpenApiDocToHopp)
)

View File

@@ -1,4 +1,3 @@
import IconPostman from "~icons/hopp/postman"
import {
Collection as PMCollection,
Item,
@@ -25,8 +24,7 @@ import * as S from "fp-ts/string"
import * as A from "fp-ts/Array"
import * as O from "fp-ts/Option"
import * as TE from "fp-ts/TaskEither"
import { step } from "../steps"
import { defineImporter, IMPORTER_INVALID_FILE_FORMAT } from "."
import { IMPORTER_INVALID_FILE_FORMAT } from "."
import { PMRawLanguage } from "~/types/pm-coll-exts"
import { stringArrayJoin } from "~/helpers/functional/array"
@@ -298,28 +296,13 @@ const getHoppFolder = (ig: ItemGroup<Item>): HoppCollection<HoppRESTRequest> =>
export const getHoppCollection = (coll: PMCollection) => getHoppFolder(coll)
export default defineImporter({
id: "postman",
name: "import.from_postman",
applicableTo: ["my-collections", "team-collections", "url-import"],
icon: IconPostman,
steps: [
step({
stepName: "FILE_IMPORT",
metadata: {
caption: "import.from_postman_description",
acceptedFileTypes: ".json",
},
}),
] as const,
importer: ([fileContent]) =>
pipe(
// Try reading
fileContent,
readPMCollection,
export const hoppPostmanImporter = (fileContent: string) =>
pipe(
// Try reading
fileContent,
readPMCollection,
O.map(flow(getHoppCollection, A.of)),
O.map(flow(getHoppCollection, A.of)),
TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT)
),
})
TE.fromOption(() => IMPORTER_INVALID_FILE_FORMAT)
)

View File

@@ -0,0 +1,46 @@
import * as TE from "fp-ts/TaskEither"
import * as O from "fp-ts/Option"
import { IMPORTER_INVALID_FILE_FORMAT } from "."
import { safeParseJSON } from "~/helpers/functional/json"
import { z } from "zod"
import { Environment } from "@hoppscotch/data"
const postmanEnvSchema = z.object({
name: z.string(),
values: z.array(
z.object({
key: z.string(),
value: z.string(),
})
),
})
export const postmanEnvImporter = (content: string) => {
const parsedContent = safeParseJSON(content)
// parse json from the environments string
if (O.isNone(parsedContent)) {
return TE.left(IMPORTER_INVALID_FILE_FORMAT)
}
const validationResult = postmanEnvSchema.safeParse(parsedContent.value)
if (!validationResult.success) {
return TE.left(IMPORTER_INVALID_FILE_FORMAT)
}
const postmanEnv = validationResult.data
const environment: Environment = {
name: postmanEnv.name,
variables: [],
}
postmanEnv.values.forEach(({ key, value }) =>
environment.variables.push({ key, value })
)
return TE.right(environment)
}

View File

@@ -1,82 +0,0 @@
/**
* Defines which type of content a Step returns.
* Add an entry here when you define a step
*/
export type StepDefinition = {
FILE_IMPORT: {
returnType: string
metadata: {
caption: string
acceptedFileTypes: string
}
} // String content of the file
TARGET_MY_COLLECTION: {
returnType: number
metadata: {
caption: string
}
} // folderPath
URL_IMPORT: {
returnType: string
metadata: {
caption: string
placeholder: string
}
} // String content of the url
}
export type StepReturnValue = StepDefinition[keyof StepDefinition]["returnType"]
/**
* Defines what the data structure of a step
*/
export type Step<T extends keyof StepDefinition> =
StepDefinition[T]["metadata"] extends never
? {
name: T
caption?: string
metadata: undefined
}
: {
name: T
caption?: string
metadata: StepDefinition[T]["metadata"]
}
/**
* The return output value of an individual step
*/
export type StepReturnType<T> = T extends Step<infer U>
? StepDefinition[U]["returnType"]
: never
export type StepMetadata<T> = T extends Step<infer U>
? StepDefinition[U]["metadata"]
: never
/**
* Defines the value of the output list generated by a step
*/
export type StepsOutputList<T> = {
[K in keyof T]: StepReturnType<T[K]>
}
type StepFuncInput<T extends keyof StepDefinition> =
StepDefinition[T]["metadata"] extends never
? {
stepName: T
caption?: string
}
: {
stepName: T
caption?: string
metadata: StepDefinition[T]["metadata"]
}
/** Use this function to define a step */
export const step = <T extends keyof StepDefinition>(input: StepFuncInput<T>) =>
<Step<T>>{
name: input.stepName,
metadata: (input as any).metadata ?? undefined,
caption: input.caption,
}