refactor: move rawKeyValue and templating to hopp/data + rewrite rawKeyValue parsing

packages/hoppscotch-app/components/environments/Edit.vue

@@ -118,15 +118,14 @@
 import clone from "lodash/clone"
 import { computed, defineComponent, PropType } from "@nuxtjs/composition-api"
 import * as E from "fp-ts/Either"
+import { Environment, parseTemplateStringE } from "@hoppscotch/data"
 import {
-  Environment,
   getEnviroment,
   getGlobalVariables,
   globalEnv$,
   setGlobalEnvVariables,
   updateEnvironment,
 } from "~/newstore/environments"
-import { parseTemplateStringE } from "~/helpers/templating"
 import { useReadonlyStream } from "~/helpers/utils/composables"
 
 export default defineComponent({

packages/hoppscotch-app/components/environments/ImportExport.vue

@@ -85,6 +85,7 @@
 
 <script setup lang="ts">
 import { computed, ref } from "@nuxtjs/composition-api"
+import { Environment } from "@hoppscotch/data"
 import { currentUser$ } from "~/helpers/fb/auth"
 import {
   useAxios,
@@ -96,7 +97,6 @@ import {
   environments$,
   replaceEnvironments,
   appendEnvironments,
-  Environment,
 } from "~/newstore/environments"
 
 defineProps<{

packages/hoppscotch-app/components/http/CodegenModal.vue

@@ -85,14 +85,14 @@
 <script setup lang="ts">
 import { computed, ref, watch } from "@nuxtjs/composition-api"
 import * as O from "fp-ts/Option"
-import { makeRESTRequest } from "@hoppscotch/data"
+import { Environment, makeRESTRequest } from "@hoppscotch/data"
 import { useCodemirror } from "~/helpers/editor/codemirror"
 import { copyToClipboard } from "~/helpers/utils/clipboard"
 import {
   getEffectiveRESTRequest,
   resolvesEnvsInBody,
 } from "~/helpers/utils/EffectiveURL"
-import { Environment, getAggregateEnvs } from "~/newstore/environments"
+import { getAggregateEnvs } from "~/newstore/environments"
 import { getRESTRequest } from "~/newstore/RESTSession"
 import { useI18n, useToast } from "~/helpers/utils/composables"
 import {

packages/hoppscotch-app/components/http/URLEncodedParams.vue

@@ -131,15 +131,15 @@
 import { computed, Ref, ref, watch } from "@nuxtjs/composition-api"
 import isEqual from "lodash/isEqual"
 import clone from "lodash/clone"
-import { HoppRESTReqBody } from "@hoppscotch/data"
-import { useCodemirror } from "~/helpers/editor/codemirror"
-import { useRESTRequestBody } from "~/newstore/RESTSession"
-import { pluckRef, useI18n, useToast } from "~/helpers/utils/composables"
 import {
+  HoppRESTReqBody,
   parseRawKeyValueEntries,
   rawKeyValueEntriesToString,
   RawKeyValueEntry,
-} from "~/helpers/rawKeyValue"
+} from "@hoppscotch/data"
+import { useCodemirror } from "~/helpers/editor/codemirror"
+import { useRESTRequestBody } from "~/newstore/RESTSession"
+import { pluckRef, useI18n, useToast } from "~/helpers/utils/composables"
 
 const t = useI18n()
 const toast = useToast()

packages/hoppscotch-app/helpers/editor/extensions/HoppEnvironment.ts

@@ -8,13 +8,13 @@ import {
   ViewPlugin,
 } from "@codemirror/view"
 import * as E from "fp-ts/Either"
+import { parseTemplateStringE } from "@hoppscotch/data"
 import { StreamSubscriberFunc } from "~/helpers/utils/composables"
 import {
   AggregateEnvironment,
   aggregateEnvs$,
   getAggregateEnvs,
 } from "~/newstore/environments"
-import { parseTemplateStringE } from "~/helpers/templating"
 
 const HOPP_ENVIRONMENT_REGEX = /(<<\w+>>)/g
 

packages/hoppscotch-app/helpers/fb/environments.ts

@@ -1,3 +1,4 @@
+import { Environment } from "@hoppscotch/data"
 import {
   collection,
   doc,
@@ -7,7 +8,6 @@ import {
 } from "firebase/firestore"
 import { currentUser$ } from "./auth"
 import {
-  Environment,
   environments$,
   globalEnv$,
   replaceEnvironments,

packages/hoppscotch-app/helpers/rawKeyValue.ts

@@ -1,40 +0,0 @@
-import * as A from "fp-ts/Array"
-import * as RA from "fp-ts/ReadonlyArray"
-import * as S from "fp-ts/string"
-import { pipe, flow } from "fp-ts/function"
-import { stringArrayJoin } from "./functional/array"
-
-export type RawKeyValueEntry = {
-  key: string
-  value: string
-  active: boolean
-}
-
-const parseRawKeyValueEntry = (str: string): RawKeyValueEntry => {
-  const trimmed = str.trim()
-  const inactive = trimmed.startsWith("#")
-
-  const [key, value] = trimmed.split(":").map(S.trim)
-
-  return {
-    key: inactive ? key.replaceAll(/^#+\s*/g, "") : key, // Remove comment hash and early space
-    value,
-    active: !inactive,
-  }
-}
-
-export const parseRawKeyValueEntries = flow(
-  S.split("\n"),
-  RA.filter((x) => x.trim().length > 0), // Remove lines which are empty
-  RA.map(parseRawKeyValueEntry),
-  RA.toArray
-)
-
-export const rawKeyValueEntriesToString = (entries: RawKeyValueEntry[]) =>
-  pipe(
-    entries,
-    A.map(({ key, value, active }) =>
-      active ? `${key}: ${value}` : `# ${key}: ${value}`
-    ),
-    stringArrayJoin("\n")
-  )

packages/hoppscotch-app/helpers/rules/BodyTransition.ts

@@ -9,12 +9,10 @@ import {
   FormDataKeyValue,
   HoppRESTReqBody,
   ValidContentTypes,
-} from "@hoppscotch/data"
-import {
   parseRawKeyValueEntries,
   rawKeyValueEntriesToString,
   RawKeyValueEntry,
-} from "../rawKeyValue"
+} from "@hoppscotch/data"
 
 const ANY_TYPE = Symbol("TRANSITION_RULESET_IGNORE_TYPE")
 // eslint-disable-next-line no-redeclare

packages/hoppscotch-app/helpers/utils/EffectiveURL.ts

@@ -7,13 +7,15 @@ import {
   FormDataKeyValue,
   HoppRESTReqBody,
   HoppRESTRequest,
+  parseTemplateString,
+  parseBodyEnvVariables,
+  parseRawKeyValueEntries,
+  Environment,
 } from "@hoppscotch/data"
-import { parseTemplateString, parseBodyEnvVariables } from "../templating"
 import { arrayFlatMap, arraySort } from "../functional/array"
 import { toFormData } from "../functional/formData"
 import { tupleToRecord } from "../functional/record"
-import { parseRawKeyValueEntries } from "../rawKeyValue"
-import { Environment, getGlobalVariables } from "~/newstore/environments"
+import { getGlobalVariables } from "~/newstore/environments"
 
 export interface EffectiveHoppRESTRequest extends HoppRESTRequest {
   /**

packages/hoppscotch-app/newstore/environments.ts

@@ -1,3 +1,4 @@
+import { Environment } from "@hoppscotch/data"
 import { cloneDeep } from "lodash"
 import isEqual from "lodash/isEqual"
 import { combineLatest, Observable } from "rxjs"
@@ -6,14 +7,6 @@ import DispatchingStore, {
   defineDispatchers,
 } from "~/newstore/DispatchingStore"
 
-export type Environment = {
-  name: string
-  variables: {
-    key: string
-    value: string
-  }[]
-}
-
 const defaultEnvironmentsState = {
   environments: [
     {

packages/hoppscotch-app/newstore/localpersistence.ts

@@ -10,6 +10,7 @@ import {
   translateToNewRequest,
   translateToNewRESTCollection,
   translateToNewGQLCollection,
+  Environment,
 } from "@hoppscotch/data"
 import { cloneDeep } from "lodash"
 import {
@@ -37,7 +38,6 @@ import {
 import {
   replaceEnvironments,
   environments$,
-  Environment,
   addGlobalEnvVariable,
   setGlobalEnvVariables,
   globalEnv$,

packages/hoppscotch-app/package.json

@@ -56,7 +56,7 @@
     "@codemirror/tooltip": "^0.19.0",
     "@codemirror/view": "^0.19.42",
     "@hoppscotch/codemirror-lang-graphql": "workspace:^0.1.0",
-    "@hoppscotch/data": "workspace:^0.3.0",
+    "@hoppscotch/data": "workspace:^0.4.0",
     "@hoppscotch/js-sandbox": "workspace:^1.0.0",
     "@nuxtjs/axios": "^5.13.6",
     "@nuxtjs/composition-api": "^0.31.0",

packages/hoppscotch-data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hoppscotch/data",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Data Types, Validations and Migrations for Hoppscotch Public Data Structures",
   "main": "dist/index.js",
   "module": "true",
@@ -14,7 +14,10 @@
   "exports": {
     ".": "./dist/index.js",
     "./graphql": "./dist/graphql/index.js",
-    "./rest": "./dist/rest/index.js"
+    "./rest": "./dist/rest/index.js",
+    "./rawKeyValue": "./dist/rawKeyValue.js",
+    "./collection": "./dist/index.js",
+    "./environment": "./dist/environment.js"
   },
   "repository": {
     "type": "git",
@@ -31,6 +34,9 @@
     "tsup": "^5.11.13"
   },
   "dependencies": {
-    "lodash": "^4.17.21"
+    "fp-ts": "^2.11.8",
+    "io-ts": "^2.2.16",
+    "lodash": "^4.17.21",
+    "parser-ts": "^0.6.16"
   }
 }

packages/hoppscotch-data/src/environment.ts

@@ -1,6 +1,13 @@
-import * as E from "fp-ts/Either"
 import { pipe } from "fp-ts/function"
-import { Environment } from "~/newstore/environments"
+import * as E from "fp-ts/Either"
+
+export type Environment = {
+  name: string
+  variables: {
+    key: string
+    value: string
+  }[]
+}
 
 const REGEX_ENV_VAR = /<<([^>]*)>>/g // "<<myVariable>>"
 

packages/hoppscotch-data/src/index.ts

@@ -1,3 +1,5 @@
 export * from "./rest"
 export * from "./graphql"
 export * from "./collection"
+export * from "./rawKeyValue"
+export * from "./environment"

packages/hoppscotch-data/src/rawKeyValue.ts

@@ -0,0 +1,178 @@
+import { not } from "fp-ts/Predicate"
+import { pipe, flow } from "fp-ts/function"
+import * as Str from "fp-ts/string"
+import * as RA from "fp-ts/ReadonlyArray"
+import * as A from "fp-ts/Array"
+import * as O from "fp-ts/Option"
+import * as E from "fp-ts/Either"
+import * as P from "parser-ts/Parser"
+import * as S from "parser-ts/string"
+import * as C from "parser-ts/char"
+
+export type RawKeyValueEntry = {
+  key: string
+  value: string
+  active: boolean
+}
+
+/* Beginning of Parser Definitions */
+
+const wsSurround = P.surroundedBy(S.spaces)
+
+const stringArrayJoin = (sep: string) => (input: string[]) => input.join(sep)
+
+const stringTakeUntilChars = (chars: C.Char[]) => pipe(
+  P.takeUntil((c: C.Char) => chars.includes(c)),
+  P.map(stringArrayJoin("")),
+)
+
+const stringTakeUntilCharsInclusive = flow(
+  stringTakeUntilChars,
+  P.chainFirst(() => P.sat(() => true)),
+)
+
+const key = pipe(
+  stringTakeUntilChars([":", "\n"]),
+  P.map(Str.trim)
+)
+
+const value = pipe(
+  stringTakeUntilChars(["\n"]),
+  P.map(Str.trim)
+)
+
+const commented = pipe(
+  S.maybe(S.string("#")),
+  P.map(not(Str.isEmpty))
+)
+
+const line = pipe(
+  wsSurround(commented),
+  P.bindTo("commented"),
+
+  P.bind("key", () => wsSurround(key)),
+
+  P.chainFirst(() => C.char(":")),
+
+  P.bind("value", () => value),
+)
+
+const lineWithNoColon = pipe(
+  wsSurround(commented),
+  P.bindTo("commented"),
+  P.bind("key", () => stringTakeUntilCharsInclusive(["\n"])),
+  P.map(flow(
+    O.fromPredicate(({ key }) => !Str.isEmpty(key))
+  ))
+)
+
+const file = pipe(
+  P.manyTill(line, P.eof()),
+)
+
+/**
+ * This Raw Key Value parser ignores the key value pair (no colon) issues
+ */
+const tolerantFile = pipe(
+  P.manyTill(
+    P.either(
+      pipe(line, P.map(O.some)),
+      () => pipe(
+        lineWithNoColon,
+        P.map(flow(
+          O.map((a) => ({ ...a, value: "" }))
+        ))
+      )
+    ),
+    P.eof()
+  ),
+  P.map(flow(
+    RA.filterMap(flow(
+      O.fromPredicate(O.isSome),
+      O.map((a) => a.value)
+    ))
+  ))
+)
+
+/* End of Parser Definitions */
+
+/**
+ * Converts Raw Key Value Entries to the file string format
+ * @param entries The entries array
+ * @returns The entries in string format
+ */
+export const rawKeyValueEntriesToString = (entries: RawKeyValueEntry[]) =>
+  pipe(
+    entries,
+    A.map(({ key, value, active }) =>
+      active ? `${key}: ${value}` : `# ${key}: ${value}`
+    ),
+    stringArrayJoin("\n")
+  )
+
+/**
+ * Parses raw key value entries string to array
+ * @param s The file string to parse from
+ * @returns Either the parser fail result or the raw key value entries
+ */
+export const parseRawKeyValueEntriesE = (s: string) =>
+  pipe(
+    tolerantFile,
+    S.run(s),
+    E.mapLeft((err) => ({
+      message: `Expected ${err.expected.map((x) => `'${x}'`).join(", ")}`,
+      expected: err.expected,
+      pos: err.input.cursor,
+    })),
+    E.map(
+      ({ value }) => pipe(
+        value,
+        RA.map(({ key, value, commented }) =>
+          <RawKeyValueEntry>{
+            active: !commented,
+            key,
+            value
+          }
+        )
+      )
+    )
+  )
+
+/**
+ * Less error tolerating version of `parseRawKeyValueEntriesE`
+ * @param s The file string to parse from
+ * @returns Either the parser fail result or the raw key value entries
+ */
+export const strictParseRawKeyValueEntriesE = (s: string) =>
+  pipe(
+    file,
+    S.run(s),
+    E.mapLeft((err) => ({
+      message: `Expected ${err.expected.map((x) => `'${x}'`).join(", ")}`,
+      expected: err.expected,
+      pos: err.input.cursor,
+    })),
+    E.map(
+      ({ value }) => pipe(
+        value,
+        RA.map(({ key, value, commented }) =>
+          <RawKeyValueEntry>{
+            active: !commented,
+            key,
+            value
+          }
+        )
+      )
+    )
+  )
+
+/**
+ * Kept for legacy code compatibility, parses raw key value entries.
+ * If failed, it returns an empty array
+ * @deprecated Use `parseRawKeyValueEntriesE` instead
+ */
+export const parseRawKeyValueEntries = flow(
+  parseRawKeyValueEntriesE,
+  E.map(RA.toArray),
+  E.getOrElse(() => [] as RawKeyValueEntry[])
+)