feat(cli): add support for request variables (#4275)

feat: add support for request variables in the CLI

packages/hoppscotch-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hoppscotch/cli",
-  "version": "0.10.2",
+  "version": "0.11.0",
   "description": "A CLI to run Hoppscotch test scripts in CI environments.",
   "homepage": "https://hoppscotch.io",
   "type": "module",

packages/hoppscotch-cli/src/__tests__/e2e/commands/test.spec.ts

@@ -335,6 +335,7 @@ describe("hopp test [options] <file_path_or_id>", () => {
             "secret-envs-persistence-scripting-envs.json",
             "environment"
           );
+
           const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
 
           const { error } = await runCLI(args);
@@ -343,6 +344,45 @@ describe("hopp test [options] <file_path_or_id>", () => {
       },
       { timeout: 20000 }
     );
+
+    describe("Request variables", () => {
+      test("Picks active request variables and ignores inactive entries", async () => {
+        const COLL_PATH = getTestJsonFilePath(
+          "request-vars-coll.json",
+          "collection"
+        );
+
+        const args = `test ${COLL_PATH}`;
+
+        const { error } = await runCLI(args);
+        expect(error).toBeNull();
+      });
+
+      test("Supports the usage of request variables along with environment variables", async () => {
+        const env = {
+          ...process.env,
+          secretBasicAuthUsernameEnvVar: "username",
+          secretBasicAuthPasswordEnvVar: "password",
+        };
+
+        const COLL_PATH = getTestJsonFilePath(
+          "request-vars-coll.json",
+          "collection"
+        );
+        const ENVS_PATH = getTestJsonFilePath(
+          "request-vars-envs.json",
+          "environment"
+        );
+
+        const args = `test ${COLL_PATH} --env ${ENVS_PATH}`;
+
+        const { error, stdout } = await runCLI(args, { env });
+        expect(stdout).toContain(
+          "https://echo.hoppscotch.io/********/********"
+        );
+        expect(error).toBeNull();
+      });
+    });
   });
 
   describe("Test `hopp test <file_path_or_id> --delay <delay_in_ms>` command:", () => {

packages/hoppscotch-cli/src/__tests__/e2e/fixtures/collections/request-vars-coll.json

@@ -0,0 +1,188 @@
+{
+  "v": 2,
+  "name": "Request variables",
+  "folders": [],
+  "requests": [
+    {
+      "v": "6",
+      "auth": {
+        "authType": "inherit",
+        "authActive": true
+      },
+      "body": {
+        "body": "{\n  \"<<httpBodyRawKey>>\": \"<<httpBodyRawValue>>\"\n}",
+        "contentType": "application/json"
+      },
+      "name": "request-variables-basic-usage",
+      "method": "POST",
+      "params": [
+        {
+          "key": "<<queryParamKey>>",
+          "value": "<<queryParamValue>>",
+          "active": true
+        },
+        {
+          "key": "inactive-query-param-key",
+          "value": "<<inactiveQueryParamValue>>",
+          "active": true
+        }
+      ],
+      "headers": [
+        {
+          "key": "<<customHeaderKey>>",
+          "value": "<<customHeaderValue>>",
+          "active": true
+        },
+        {
+          "key": "inactive-header-key",
+          "value": "<<inactiveHeaderValue>>",
+          "active": true
+        }
+      ],
+      "endpoint": "<<url>>",
+      "testScript": "pw.test(\"Accounts for active request variables\", ()=> {\n  pw.expect(pw.response.body.args[\"query-param-key\"]).toBe(\"query-param-value\");\n\n  const data = JSON.parse(pw.response.body.data)\n\n  pw.expect(data[\"http-body-raw-key\"]).toBe(\"http-body-raw-value\")\n\n  pw.expect(pw.response.body.headers[\"custom-header-key\"]).toBe(\"custom-header-value\");\n});\n\npw.test(\"Ignores inactive request variables\", () => {\n  pw.expect(pw.response.body.args[\"inactive-query-param-key\"]).toBe(\"\")\n  pw.expect(pw.response.body.args[\"inactive-header-key\"]).toBe(undefined)\n})",
+      "preRequestScript": "",
+      "requestVariables": [
+        {
+          "key": "url",
+          "value": "https://echo.hoppscotch.io",
+          "active": true
+        },
+        {
+          "key": "method",
+          "value": "POST",
+          "active": true
+        },
+        {
+          "key": "httpBodyRawKey",
+          "value": "http-body-raw-key",
+          "active": true
+        },
+        {
+          "key": "httpBodyRawValue",
+          "value": "http-body-raw-value",
+          "active": true
+        },
+        {
+          "key": "customHeaderKey",
+          "value": "custom-header-key",
+          "active": true
+        },
+        {
+          "key": "customHeaderValue",
+          "value": "custom-header-value",
+          "active": true
+        },
+        {
+          "key": "queryParamKey",
+          "value": "query-param-key",
+          "active": true
+        },
+        {
+          "key": "queryParamValue",
+          "value": "query-param-value",
+          "active": true
+        },
+        {
+          "key": "inactiveQueryParamValue",
+          "value": "inactive-query-param-value",
+          "active": false
+        },
+        {
+          "key": "inactiveHeaderValue",
+          "value": "inactive-header-value",
+          "active": false
+        }
+      ]
+    },
+    {
+      "v": "6",
+      "auth": {
+        "authType": "none",
+        "password": "<<password>>",
+        "username": "<<username>>",
+        "authActive": true
+      },
+      "body": {
+        "body": "{\n  \"username\": \"<<username>>\",\n  \"password\": \"<<password>>\"\n}",
+        "contentType": "application/json"
+      },
+      "name": "request-variables-alongside-environment-variables",
+      "method": "POST",
+      "params": [
+        {
+          "key": "method",
+          "value": "<<method>>",
+          "active": true
+        }
+      ],
+      "headers": [
+        {
+          "key": "test-header-key",
+          "value": "<<testHeaderValue>>",
+          "active": true
+        }
+      ],
+      "endpoint": "<<url>>/<<path>>",
+      "testScript": "pw.test(\"The first occurrence is picked for multiple request variable occurrences with the same key.\", () => {\n  pw.expect(pw.response.body.args.method).toBe(\"post\");\n});\n\npw.test(\"Request variables support recursive resolution and pick values from secret environment variables\", () => {\n  const { username, password } = JSON.parse(pw.response.body.data)\n\n  pw.expect(username).toBe(\"username\")\n  pw.expect(password).toBe(\"password\")\n\n})\n\npw.test(\"Resolves request variables that are clubbed together\", () => {\n  pw.expect(pw.response.body.path).toBe(\"/username/password\")\n})\n\npw.test(\"Request variables are prioritised over environment variables\", () => {\n  pw.expect(pw.response.body.headers.host).toBe(\"echo.hoppscotch.io\")\n})\n\npw.test(\"Environment variable is picked if the request variable under the same name is empty\", () => {\n  pw.expect(pw.response.body.headers[\"test-header-key\"]).toBe(\"test-header-value\")\n})",
+      "preRequestScript": "",
+      "requestVariables": [
+        {
+          "key": "url",
+          "value": "https://echo.hoppscotch.io",
+          "active": true
+        },
+        {
+          "key": "username",
+          "value": "<<recursiveBasicAuthUsernameReqVar>>",
+          "active": true
+        },
+        {
+          "key": "recursiveBasicAuthUsernameReqVar",
+          "value": "<<secretBasicAuthUsernameEnvVar>>",
+          "active": true
+        },
+        {
+          "key": "password",
+          "value": "<<recursiveBasicAuthPasswordReqVar>>",
+          "active": true
+        },
+        {
+          "key": "recursiveBasicAuthPasswordReqVar",
+          "value": "<<secretBasicAuthPasswordEnvVar>>",
+          "active": true
+        },
+        {
+          "key": "method",
+          "value": "post",
+          "active": true
+        },
+        {
+          "key": "method",
+          "value": "get",
+          "active": true
+        },
+        {
+          "key": "method",
+          "value": "put",
+          "active": true
+        },
+        {
+          "key": "path",
+          "value": "<<username>>/<<password>>",
+          "active": true
+        },
+        {
+          "key": "testHeaderValue",
+          "value": "",
+          "active": true
+        }
+      ]
+    }
+  ],
+  "auth": {
+    "authType": "inherit",
+    "authActive": true
+  },
+  "headers": []
+}

packages/hoppscotch-cli/src/__tests__/e2e/fixtures/environments/request-vars-envs.json

@@ -0,0 +1,34 @@
+{
+  "v": 1,
+  "id": "cm00r7kpb0006mbd2nq1560w6",
+  "name": "Request variables alongside environment variables",
+  "variables": [
+    {
+      "key": "url",
+      "value": "https://echo.hoppscotch.io",
+      "secret": false
+    },
+    {
+      "key": "secretBasicAuthPasswordEnvVar",
+      "secret": true
+    },
+    {
+      "key": "secretBasicAuthUsernameEnvVar",
+      "value": "username",
+      "secret": true
+    },
+    {
+      "key": "username",
+      "secret": true
+    },
+    {
+      "key": "password",
+      "secret": true
+    },
+    {
+      "key": "testHeaderValue",
+      "value": "test-header-value",
+      "secret": false
+    }
+  ]
+}

packages/hoppscotch-cli/src/__tests__/unit/getters.spec.ts

@@ -4,7 +4,6 @@ import { describe, expect, test, vi } from "vitest";
 
 import {
   CollectionSchemaVersion,
-  Environment,
   HoppCollection,
   getDefaultRESTRequest,
 } from "@hoppscotch/data";
@@ -13,6 +12,7 @@ import { DEFAULT_DURATION_PRECISION } from "../../utils/constants";
 import {
   getDurationInSeconds,
   getEffectiveFinalMetaData,
+  getResolvedVariables,
   getResourceContents,
 } from "../../utils/getters";
 import * as mutators from "../../utils/mutators";
@@ -43,13 +43,14 @@ describe("getters", () => {
   });
 
   describe("getEffectiveFinalMetaData", () => {
-    const DEFAULT_ENV = <Environment>{
+    const environmentVariables = [
-      name: "name",
+      { key: "PARAM", value: "parsed_param", secret: false },
-      variables: [{ key: "PARAM", value: "parsed_param" }],
+    ];
-    };
 
     test("Empty list of meta-data", () => {
-      expect(getEffectiveFinalMetaData([], DEFAULT_ENV)).toSubsetEqualRight([]);
+      expect(
+        getEffectiveFinalMetaData([], environmentVariables)
+      ).toSubsetEqualRight([]);
     });
 
     test("Non-empty active list of meta-data with unavailable ENV", () => {
@@ -62,7 +63,7 @@ describe("getters", () => {
               value: "<<UNKNOWN_VALUE>>",
             },
           ],
-          DEFAULT_ENV
+          environmentVariables
         )
       ).toSubsetEqualRight([{ active: true, key: "", value: "" }]);
     });
@@ -71,7 +72,7 @@ describe("getters", () => {
       expect(
         getEffectiveFinalMetaData(
           [{ active: false, key: "KEY", value: "<<PARAM>>" }],
-          DEFAULT_ENV
+          environmentVariables
         )
       ).toSubsetEqualRight([]);
     });
@@ -80,7 +81,7 @@ describe("getters", () => {
       expect(
         getEffectiveFinalMetaData(
           [{ active: true, key: "PARAM", value: "<<PARAM>>" }],
-          DEFAULT_ENV
+          environmentVariables
         )
       ).toSubsetEqualRight([
         { active: true, key: "PARAM", value: "parsed_param" },
@@ -386,4 +387,101 @@ describe("getters", () => {
       });
     });
   });
+
+  describe("getResolvedVariables", () => {
+    const requestVariables = [
+      {
+        key: "SHARED_KEY_I",
+        value: "request-variable-shared-value-I",
+        active: true,
+      },
+      {
+        key: "SHARED_KEY_II",
+        value: "",
+        active: true,
+      },
+      {
+        key: "REQUEST_VAR_III",
+        value: "request-variable-value-III",
+        active: true,
+      },
+      {
+        key: "REQUEST_VAR_IV",
+        value: "request-variable-value-IV",
+        active: false,
+      },
+      {
+        key: "REQUEST_VAR_V",
+        value: "request-variable-value-V",
+        active: false,
+      },
+    ];
+
+    const environmentVariables = [
+      {
+        key: "SHARED_KEY_I",
+        value: "environment-variable-shared-value-I",
+        secret: false,
+      },
+      {
+        key: "SHARED_KEY_II",
+        value: "environment-variable-shared-value-II",
+        secret: false,
+      },
+      {
+        key: "ENV_VAR_III",
+        value: "environment-variable-value-III",
+        secret: false,
+      },
+      {
+        key: "ENV_VAR_IV",
+        value: "environment-variable-value-IV",
+        secret: false,
+      },
+      {
+        key: "ENV_VAR_V",
+        value: "environment-variable-value-V",
+        secret: false,
+      },
+    ];
+
+    test("Filters request variables by active status and value fields, then remove environment variables sharing the same keys", () => {
+      const expected = [
+        {
+          key: "SHARED_KEY_I",
+          value: "request-variable-shared-value-I",
+          secret: false,
+        },
+        {
+          key: "REQUEST_VAR_III",
+          value: "request-variable-value-III",
+          secret: false,
+        },
+        {
+          key: "SHARED_KEY_II",
+          value: "environment-variable-shared-value-II",
+          secret: false,
+        },
+        {
+          key: "ENV_VAR_III",
+          value: "environment-variable-value-III",
+          secret: false,
+        },
+        {
+          key: "ENV_VAR_IV",
+          value: "environment-variable-value-IV",
+          secret: false,
+        },
+        {
+          key: "ENV_VAR_V",
+          value: "environment-variable-value-V",
+          secret: false,
+        },
+      ];
+
+      expect(
+        getResolvedVariables(requestVariables, environmentVariables)
+      ).toEqual(expected);
+    });
+  });
 });

packages/hoppscotch-cli/src/utils/getters.ts

@@ -1,8 +1,8 @@
 import {
-  Environment,
+  EnvironmentVariable,
-  HoppCollection,
   HoppRESTHeader,
   HoppRESTParam,
+  HoppRESTRequestVariables,
   parseTemplateStringE,
 } from "@hoppscotch/data";
 import axios, { AxiosError } from "axios";
@@ -58,12 +58,12 @@ export const getColorStatusCode = (
  * Replaces all template-string with their effective ENV values to generate effective
  * request headers/parameters meta-data.
  * @param metaData Headers/parameters on which ENVs will be applied.
- * @param environment Provides ENV variables for parsing template-string.
+ * @param resolvedVariables Provides ENV variables for parsing template-string.
  * @returns Active, non-empty-key, parsed headers/parameters pairs.
  */
 export const getEffectiveFinalMetaData = (
   metaData: HoppRESTHeader[] | HoppRESTParam[],
-  environment: Environment
+  resolvedVariables: EnvironmentVariable[]
 ) =>
   pipe(
     metaData,
@@ -72,11 +72,13 @@ export const getEffectiveFinalMetaData = (
      * Selecting only non-empty and active pairs.
      */
     A.filter(({ key, active }) => !S.isEmpty(key) && active),
-    A.map(({ key, value }) => ({
+    A.map(({ key, value }) => {
-      active: true,
+      return {
-      key: parseTemplateStringE(key, environment.variables),
+        active: true,
-      value: parseTemplateStringE(value, environment.variables),
+        key: parseTemplateStringE(key, resolvedVariables),
-    })),
+        value: parseTemplateStringE(value, resolvedVariables),
+      };
+    }),
     E.fromPredicate(
       /**
        * Check if every key-value is right either. Else return HoppCLIError with
@@ -253,3 +255,30 @@ export const getResourceContents = async (
 
   return contents;
 };
+
+/**
+ * Processes incoming request variables and environment variables and returns a list
+ * where active request variables are picked and prioritised over the supplied environment variables.
+ * Falls back to environment variables for an empty request variable.
+ *
+ * @param {HoppRESTRequestVariables} requestVariables - Incoming request variables.
+ * @param {EnvironmentVariable[]} environmentVariables - Incoming environment variables.
+ * @returns {EnvironmentVariable[]} The resolved list of variables that conforms to the shape of environment variables.
+ */
+export const getResolvedVariables = (
+  requestVariables: HoppRESTRequestVariables,
+  environmentVariables: EnvironmentVariable[]
+): EnvironmentVariable[] => {
+  const activeRequestVariables = requestVariables
+    .filter(({ active, value }) => active && value)
+    .map(({ key, value }) => ({ key, value, secret: false }));
+
+  const requestVariableKeys = activeRequestVariables.map(({ key }) => key);
+
+  // Request variables have higher priority, hence filtering out environment variables with the same keys
+  const filteredEnvironmentVariables = environmentVariables.filter(
+    ({ key }) => !requestVariableKeys.includes(key)
+  );
+
+  return [...activeRequestVariables, ...filteredEnvironmentVariables];
+};

packages/hoppscotch-cli/src/utils/pre-request.ts

@@ -1,5 +1,6 @@
 import {
   Environment,
+  EnvironmentVariable,
   HoppRESTRequest,
   parseBodyEnvVariablesE,
   parseRawKeyValueEntriesE,
@@ -22,7 +23,7 @@ import { HoppEnvs } from "../types/request";
 import { PreRequestMetrics } from "../types/response";
 import { isHoppCLIError } from "./checks";
 import { arrayFlatMap, arraySort, tupleToRecord } from "./functions/array";
-import { getEffectiveFinalMetaData } from "./getters";
+import { getEffectiveFinalMetaData, getResolvedVariables } from "./getters";
 import { toFormData } from "./mutators";
 
 /**
@@ -80,10 +81,15 @@ export function getEffectiveRESTRequest(
 > {
   const envVariables = environment.variables;
 
+  const resolvedVariables = getResolvedVariables(
+    request.requestVariables,
+    envVariables
+  );
+
   // Parsing final headers with applied ENVs.
   const _effectiveFinalHeaders = getEffectiveFinalMetaData(
     request.headers,
-    environment
+    resolvedVariables
   );
   if (E.isLeft(_effectiveFinalHeaders)) {
     return _effectiveFinalHeaders;
@@ -93,7 +99,7 @@ export function getEffectiveRESTRequest(
   // Parsing final parameters with applied ENVs.
   const _effectiveFinalParams = getEffectiveFinalMetaData(
     request.params,
-    environment
+    resolvedVariables
   );
   if (E.isLeft(_effectiveFinalParams)) {
     return _effectiveFinalParams;
@@ -104,8 +110,14 @@ export function getEffectiveRESTRequest(
   if (request.auth.authActive) {
     // TODO: Support a better b64 implementation than btoa ?
     if (request.auth.authType === "basic") {
-      const username = parseTemplateString(request.auth.username, envVariables);
+      const username = parseTemplateString(
-      const password = parseTemplateString(request.auth.password, envVariables);
+        request.auth.username,
+        resolvedVariables
+      );
+      const password = parseTemplateString(
+        request.auth.password,
+        resolvedVariables
+      );
 
       effectiveFinalHeaders.push({
         active: true,
@@ -116,7 +128,7 @@ export function getEffectiveRESTRequest(
       effectiveFinalHeaders.push({
         active: true,
         key: "Authorization",
-        value: `Bearer ${parseTemplateString(request.auth.token, envVariables)}`,
+        value: `Bearer ${parseTemplateString(request.auth.token, resolvedVariables)}`,
       });
     } else if (request.auth.authType === "oauth-2") {
       const { addTo } = request.auth;
@@ -125,7 +137,7 @@ export function getEffectiveRESTRequest(
         effectiveFinalHeaders.push({
           active: true,
           key: "Authorization",
-          value: `Bearer ${parseTemplateString(request.auth.grantTypeInfo.token, envVariables)}`,
+          value: `Bearer ${parseTemplateString(request.auth.grantTypeInfo.token, resolvedVariables)}`,
         });
       } else if (addTo === "QUERY_PARAMS") {
         effectiveFinalParams.push({
@@ -133,7 +145,7 @@ export function getEffectiveRESTRequest(
           key: "access_token",
           value: parseTemplateString(
             request.auth.grantTypeInfo.token,
-            envVariables
+            resolvedVariables
           ),
         });
       }
@@ -142,21 +154,24 @@ export function getEffectiveRESTRequest(
       if (addTo === "HEADERS") {
         effectiveFinalHeaders.push({
           active: true,
-          key: parseTemplateString(key, envVariables),
+          key: parseTemplateString(key, resolvedVariables),
-          value: parseTemplateString(value, envVariables),
+          value: parseTemplateString(value, resolvedVariables),
         });
       } else if (addTo === "QUERY_PARAMS") {
         effectiveFinalParams.push({
           active: true,
-          key: parseTemplateString(key, envVariables),
+          key: parseTemplateString(key, resolvedVariables),
-          value: parseTemplateString(value, envVariables),
+          value: parseTemplateString(value, resolvedVariables),
         });
       }
     }
   }
 
   // Parsing final-body with applied ENVs.
-  const _effectiveFinalBody = getFinalBodyFromRequest(request, envVariables);
+  const _effectiveFinalBody = getFinalBodyFromRequest(
+    request,
+    resolvedVariables
+  );
   if (E.isLeft(_effectiveFinalBody)) {
     return _effectiveFinalBody;
   }
@@ -175,10 +190,10 @@ export function getEffectiveRESTRequest(
     });
   }
 
-  // Parsing final-endpoint with applied ENVs.
+  // Parsing final-endpoint with applied ENVs (environment + request variables).
   const _effectiveFinalURL = parseTemplateStringE(
     request.endpoint,
-    envVariables
+    resolvedVariables
   );
   if (E.isLeft(_effectiveFinalURL)) {
     return E.left(
@@ -195,7 +210,7 @@ export function getEffectiveRESTRequest(
   if (envVariables.some(({ secret }) => secret)) {
     const _effectiveFinalDisplayURL = parseTemplateStringE(
       request.endpoint,
-      envVariables,
+      resolvedVariables,
       true
     );
 
@@ -213,7 +228,7 @@ export function getEffectiveRESTRequest(
       effectiveFinalParams,
       effectiveFinalBody,
     },
-    updatedEnvs: { global: [], selected: envVariables },
+    updatedEnvs: { global: [], selected: resolvedVariables },
   });
 }
 
@@ -221,14 +236,14 @@ export function getEffectiveRESTRequest(
  * Replaces template variables in request's body from the given set of ENVs,
  * to generate final request body without any template variables.
  * @param request Provides request's body, on which ENVs has to be applied.
- * @param envVariables Provides set of key-value pairs (environment variables),
+ * @param resolvedVariables Provides set of key-value pairs (request + environment variables),
  * used to parse-out template variables.
  * @returns Final request body without any template variables as value.
  * Or, HoppCLIError in case of error while parsing.
  */
 function getFinalBodyFromRequest(
   request: HoppRESTRequest,
-  envVariables: Environment["variables"]
+  resolvedVariables: EnvironmentVariable[]
 ): E.Either<HoppCLIError, string | null | FormData> {
   if (request.body.contentType === null) {
     return E.right(null);
@@ -252,8 +267,8 @@ function getFinalBodyFromRequest(
            * which will be resolved in further steps.
            */
           A.map(({ key, value }) => [
-            parseTemplateStringE(key, envVariables),
+            parseTemplateStringE(key, resolvedVariables),
-            parseTemplateStringE(value, envVariables),
+            parseTemplateStringE(value, resolvedVariables),
           ]),
 
           /**
@@ -289,13 +304,13 @@ function getFinalBodyFromRequest(
       arrayFlatMap((x) =>
         x.isFile
           ? x.value.map((v) => ({
-              key: parseTemplateString(x.key, envVariables),
+              key: parseTemplateString(x.key, resolvedVariables),
               value: v as string | Blob,
             }))
           : [
               {
-                key: parseTemplateString(x.key, envVariables),
+                key: parseTemplateString(x.key, resolvedVariables),
-                value: parseTemplateString(x.value, envVariables),
+                value: parseTemplateString(x.value, resolvedVariables),
               },
             ]
       ),
@@ -305,7 +320,7 @@ function getFinalBodyFromRequest(
   }
 
   return pipe(
-    parseBodyEnvVariablesE(request.body.body, envVariables),
+    parseBodyEnvVariablesE(request.body.body, resolvedVariables),
     E.mapLeft((e) =>
       error({
         code: "PARSING_ERROR",

packages/hoppscotch-cli/src/utils/request.ts

@@ -4,7 +4,7 @@ import {
   HoppRESTRequest,
   RESTReqSchemaVersion,
 } from "@hoppscotch/data";
-import axios, { AxiosResponse, Method } from "axios";
+import axios, { Method } from "axios";
 import * as A from "fp-ts/Array";
 import * as E from "fp-ts/Either";
 import * as T from "fp-ts/Task";