From dcdd0379d442747fa1eeeb46cf66b43db0cbf5ab Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Mon, 14 Mar 2022 18:09:32 +0000 Subject: [PATCH] chore: introduce curl parser tests and minor changes (#2145) Co-authored-by: Liyas Thomas Co-authored-by: Andrew Bastin --- .github/workflows/tests.yml | 2 +- .../helpers/curl/__tests__/curlparser.spec.js | 636 ++++++++++++++++++ .../curl/__tests__/detectContentType.spec.js | 160 +++++ .../helpers/curl/contentParser.ts | 26 +- .../hoppscotch-app/helpers/curl/curlparser.ts | 80 ++- packages/hoppscotch-app/helpers/curl/index.ts | 4 +- 6 files changed, 862 insertions(+), 46 deletions(-) create mode 100644 packages/hoppscotch-app/helpers/curl/__tests__/curlparser.spec.js create mode 100644 packages/hoppscotch-app/helpers/curl/__tests__/detectContentType.spec.js diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 43517768e..f76db860f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: ["lts/*"] steps: - uses: actions/checkout@v2 diff --git a/packages/hoppscotch-app/helpers/curl/__tests__/curlparser.spec.js b/packages/hoppscotch-app/helpers/curl/__tests__/curlparser.spec.js new file mode 100644 index 000000000..1f1bd1d17 --- /dev/null +++ b/packages/hoppscotch-app/helpers/curl/__tests__/curlparser.spec.js @@ -0,0 +1,636 @@ +// @ts-check +// ^^^ Enables Type Checking by the TypeScript compiler + +import { makeRESTRequest, rawKeyValueEntriesToString } from "@hoppscotch/data" +import { parseCurlToHoppRESTReq } from ".." + +const samples = [ + { + command: ` + curl --request GET \ + --url https://echo.hoppscotch.io/ \ + --header 'content-type: application/x-www-form-urlencoded' \ + --data a=b \ + --data c=d + `, + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "https://echo.hoppscotch.io/", + auth: { authType: "none", authActive: false }, + body: { + contentType: "application/x-www-form-urlencoded", + body: rawKeyValueEntriesToString([ + { + active: true, + key: "a", + value: "b", + }, + { + active: true, + key: "c", + value: "d", + }, + ]), + }, + headers: [], + params: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: ` + curl 'http://avs:def@127.0.0.1:8000/api/admin/crm/brand/4' + -X PUT + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0' + -H 'Accept: application/json, text/plain, */*' + -H 'Accept-Language: en' + --compressed + -H 'Content-Type: application/hal+json;charset=utf-8' + -H 'Origin: http://localhost:3012' + -H 'Connection: keep-alive' + -H 'Referer: http://localhost:3012/crm/company/4' + --data-raw '{"id":4,"crm_company_id":4,"industry_primary_id":2,"industry_head_id":2,"industry_body_id":2,"code":"01","barcode":"222010101","summary":"Healt-Seasoning-Basic-Hori-Kello","name":"Kellolaa","sub_code":"01","sub_name":"Hori","created_at":"2020-06-08 08:50:02","updated_at":"2020-06-08 08:50:02","company":4,"primary":{"id":2,"code":"2","name":"Healt","created_at":"2020-05-19 07:05:02","updated_at":"2020-05-19 07:09:28"},"head":{"id":2,"code":"2","name":"Seasoning","created_at":"2020-04-14 19:34:33","updated_at":"2020-04-14 19:34:33"},"body":{"id":2,"code":"2","name":"Basic","created_at":"2020-04-14 19:33:54","updated_at":"2020-04-14 19:33:54"},"contacts":[]}' + `, + response: makeRESTRequest({ + method: "PUT", + name: "Untitled request", + endpoint: "http://127.0.0.1:8000/api/admin/crm/brand/4", + auth: { + authType: "basic", + authActive: true, + username: "avs", + password: "def", + }, + body: { + contentType: "application/hal+json", + body: `{ + "id": 4, + "crm_company_id": 4, + "industry_primary_id": 2, + "industry_head_id": 2, + "industry_body_id": 2, + "code": "01", + "barcode": "222010101", + "summary": "Healt-Seasoning-Basic-Hori-Kello", + "name": "Kellolaa", + "sub_code": "01", + "sub_name": "Hori", + "created_at": "2020-06-08 08:50:02", + "updated_at": "2020-06-08 08:50:02", + "company": 4, + "primary": { + "id": 2, + "code": "2", + "name": "Healt", + "created_at": "2020-05-19 07:05:02", + "updated_at": "2020-05-19 07:09:28" + }, + "head": { + "id": 2, + "code": "2", + "name": "Seasoning", + "created_at": "2020-04-14 19:34:33", + "updated_at": "2020-04-14 19:34:33" + }, + "body": { + "id": 2, + "code": "2", + "name": "Basic", + "created_at": "2020-04-14 19:33:54", + "updated_at": "2020-04-14 19:33:54" + }, + "contacts": [] +}`, + }, + headers: [ + { + active: true, + key: "User-Agent", + value: + "Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0", + }, + { + active: true, + key: "Accept", + value: "application/json, text/plain, */*", + }, + { + active: true, + key: "Accept-Language", + value: "en", + }, + { + active: true, + key: "Origin", + value: "http://localhost:3012", + }, + { + active: true, + key: "Connection", + value: "keep-alive", + }, + { + active: true, + key: "Referer", + value: "http://localhost:3012/crm/company/4", + }, + ], + params: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl google.com`, + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "https://google.com/", + auth: { authType: "none", authActive: false }, + body: { + contentType: null, + body: null, + }, + headers: [], + params: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl -X POST -d '{"foo":"bar"}' http://localhost:1111/hello/world/?bar=baz&buzz`, + response: makeRESTRequest({ + method: "POST", + name: "Untitled request", + endpoint: "http://localhost:1111/hello/world/?buzz", + auth: { authType: "none", authActive: false }, + body: { + contentType: "application/json", + body: `{\n "foo": "bar"\n}`, + }, + headers: [], + params: [ + { + active: true, + key: "bar", + value: "baz", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl --get -d "tool=curl" -d "age=old" https://example.com`, + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "https://example.com/", + auth: { authType: "none", authActive: false }, + body: { + contentType: null, + body: null, + }, + headers: [], + params: [ + { + active: true, + key: "tool", + value: "curl", + }, + { + active: true, + key: "age", + value: "old", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl -F hello=hello2 -F hello3=@hello4.txt bing.com`, + response: makeRESTRequest({ + method: "POST", + name: "Untitled request", + endpoint: "https://bing.com/", + auth: { authType: "none", authActive: false }, + body: { + contentType: "multipart/form-data", + body: [ + { + active: true, + isFile: false, + key: "hello", + value: "hello2", + }, + { + active: true, + isFile: false, + key: "hello3", + value: "", + }, + ], + }, + headers: [], + params: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: + "curl -X GET localhost -H 'Accept: application/json' --user root:toor", + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "http://localhost/", + auth: { + authType: "basic", + authActive: true, + username: "root", + password: "toor", + }, + body: { + contentType: null, + body: null, + }, + params: [], + headers: [ + { + active: true, + key: "Accept", + value: "application/json", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: + "curl -X GET localhost --header 'Authorization: Basic dXNlcjpwYXNz'", + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "http://localhost/", + auth: { + authType: "basic", + authActive: true, + username: "user", + password: "pass", + }, + body: { + contentType: null, + body: null, + }, + params: [], + headers: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: + "curl -X GET localhost --header 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c'", + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "http://localhost/", + auth: { + authType: "bearer", + authActive: true, + token: + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", + }, + body: { + contentType: null, + body: null, + }, + params: [], + headers: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: + "curl -X GET localhost --header 'Authorization: Apikey dXNlcjpwYXNz'", + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "http://localhost/", + auth: { + authActive: true, + authType: "api-key", + key: "apikey", + value: "dXNlcjpwYXNz", + addTo: "headers", + }, + body: { + contentType: null, + body: null, + }, + params: [], + headers: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl --get -I -d "tool=curl" -d "platform=hoppscotch" -d"io" https://hoppscotch.io`, + response: makeRESTRequest({ + method: "HEAD", + name: "Untitled request", + endpoint: "https://hoppscotch.io/?io", + auth: { + authActive: false, + authType: "none", + }, + body: { + contentType: null, + body: null, + }, + params: [ + { + active: true, + key: "tool", + value: "curl", + }, + { + active: true, + key: "platform", + value: "hoppscotch", + }, + ], + headers: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl 'https://someshadywebsite.com/questionable/path/?and=params&so&stay=tuned&' \ + -H 'user-agent: Mozilla/5.0' \ + -H 'accept: text/html' \ + -H $'cookie: cookie-cookie' \ + --data $'------WebKitFormBoundaryj3oufpIISPa2DP7c\\r\\nContent-Disposition: form-data; name="EmailAddress"\\r\\n\\r\\ntest@test.com\\r\\n------WebKitFormBoundaryj3oufpIISPa2DP7c\\r\\nContent-Disposition: form-data; name="Entity"\\r\\n\\r\\n1\\r\\n------WebKitFormBoundaryj3oufpIISPa2DP7c--\\r\\n'`, + response: makeRESTRequest({ + method: "POST", + name: "Untitled request", + endpoint: "https://someshadywebsite.com/questionable/path/?so", + auth: { + authActive: false, + authType: "none", + }, + body: { + contentType: "multipart/form-data", + body: [ + { + active: true, + isFile: false, + key: "EmailAddress", + value: "test@test.com", + }, + { + active: true, + isFile: false, + key: "Entity", + value: "1", + }, + ], + }, + params: [ + { + active: true, + key: "and", + value: "params", + }, + { + active: true, + key: "stay", + value: "tuned", + }, + ], + headers: [ + { + active: true, + key: "user-agent", + value: "Mozilla/5.0", + }, + { + active: true, + key: "accept", + value: "text/html", + }, + { + active: true, + key: "cookie", + value: "cookie-cookie", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: + "curl localhost -H 'content-type: multipart/form-data; boundary=------------------------d74496d66958873e' --data '-----------------------------d74496d66958873e\\r\\nContent-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\\r\\nContent-Type: text/plain\\r\\n\\r\\nHello World\\r\\n\\r\\n-----------------------------d74496d66958873e--\\r\\n'", + response: makeRESTRequest({ + method: "POST", + name: "Untitled request", + endpoint: "http://localhost/", + auth: { + authActive: false, + authType: "none", + }, + body: { + contentType: "multipart/form-data", + body: [ + { + active: true, + isFile: false, + key: "file", + value: "", + }, + ], + }, + params: [], + headers: [], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl 'https://hoppscotch.io/' \ + -H 'authority: hoppscotch.io' \ + -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"' \ + -H 'accept: */*' \ + -H 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36' \ + -H 'sec-ch-ua-platform: "Windows"' \ + -H 'accept-language: en-US,en;q=0.9,ml;q=0.8' \ + --compressed`, + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "https://hoppscotch.io/", + auth: { authType: "none", authActive: false }, + body: { + contentType: null, + body: null, + }, + params: [], + headers: [ + { + active: true, + key: "authority", + value: "hoppscotch.io", + }, + { + active: true, + key: "sec-ch-ua", + value: + '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"', + }, + { + active: true, + key: "accept", + value: "*/*", + }, + { + active: true, + key: "user-agent", + value: + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", + }, + { + active: true, + key: "sec-ch-ua-platform", + value: '"Windows"', + }, + { + active: true, + key: "accept-language", + value: "en-US,en;q=0.9,ml;q=0.8", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl --request GET \ + --url 'https://echo.hoppscotch.io/?hello=there' \ + --header 'content-type: application/x-www-form-urlencoded' \ + --header 'something: other-thing' \ + --data a=b \ + --data c=d`, + response: makeRESTRequest({ + method: "GET", + name: "Untitled request", + endpoint: "https://echo.hoppscotch.io/", + auth: { authType: "none", authActive: false }, + body: { + contentType: "application/x-www-form-urlencoded", + body: rawKeyValueEntriesToString([ + { + key: "a", + value: "b", + active: true, + }, + { + key: "c", + value: "d", + active: true, + }, + ]), + }, + params: [ + { + active: true, + key: "hello", + value: "there", + }, + ], + headers: [ + { + active: true, + key: "something", + value: "other-thing", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: `curl --request POST \ + --url 'https://echo.hoppscotch.io/?hello=there' \ + --header 'content-type: multipart/form-data' \ + --header 'something: other-thing' \ + --form a=b \ + --form c=d`, + response: makeRESTRequest({ + name: "Untitled request", + endpoint: "https://echo.hoppscotch.io/", + method: "POST", + auth: { authType: "none", authActive: false }, + headers: [ + { + active: true, + key: "something", + value: "other-thing", + }, + ], + body: { + contentType: "multipart/form-data", + body: [ + { + active: true, + isFile: false, + key: "a", + value: "b", + }, + { + active: true, + isFile: false, + key: "c", + value: "d", + }, + ], + }, + params: [ + { + active: true, + key: "hello", + value: "there", + }, + ], + preRequestScript: "", + testScript: "", + }), + }, + { + command: "curl 'muxueqz.top/skybook.html'", + response: makeRESTRequest({ + name: "Untitled request", + endpoint: "https://muxueqz.top/skybook.html", + method: "GET", + auth: { authType: "none", authActive: false }, + headers: [], + body: { contentType: null, body: null }, + params: [], + preRequestScript: "", + testScript: "", + }), + }, +] + +describe("parseCurlToHoppRESTReq", () => { + for (const [i, { command, response }] of samples.entries()) { + test(`matches expectation for sample #${i + 1}`, () => { + expect(parseCurlToHoppRESTReq(command)).toEqual(response) + }) + } +}) diff --git a/packages/hoppscotch-app/helpers/curl/__tests__/detectContentType.spec.js b/packages/hoppscotch-app/helpers/curl/__tests__/detectContentType.spec.js new file mode 100644 index 000000000..3834d55cb --- /dev/null +++ b/packages/hoppscotch-app/helpers/curl/__tests__/detectContentType.spec.js @@ -0,0 +1,160 @@ +import { detectContentType } from "../contentParser" + +describe("detect content type", () => { + test("should return text/plain for blank/null/undefined input", () => { + expect(detectContentType("")).toBe("text/plain") + expect(detectContentType(null)).toBe("text/plain") + expect(detectContentType(undefined)).toBe("text/plain") + }) + + describe("application/json", () => { + test('should return text/plain for "{"', () => { + expect(detectContentType("{")).toBe("text/plain") + }) + + test('should return application/json for "{}"', () => { + expect(detectContentType("{}")).toBe("application/json") + }) + + test("should return application/json for valid json data", () => { + expect( + detectContentType(` + { + "body": "some text", + "name": "interesting name", + "code": [1, 5, 6, 2] + } + `) + ).toBe("application/json") + }) + }) + + describe("application/xml", () => { + test("should return text/html for XML data without XML declaration", () => { + expect( + detectContentType(` + + Everyday Italian + Giada De Laurentiis + 2005 + 30.00 + + `) + ).toBe("text/html") + }) + + test("should return application/xml for valid XML data", () => { + expect( + detectContentType(` + + + Everyday Italian + Giada De Laurentiis + 2005 + 30.00 + + `) + ).toBe("text/html") + }) + + test("should return text/html for invalid XML data", () => { + expect( + detectContentType(` + + Everyday Italian + <abcd>Giada De Laurentiis</abcd> + <year>2005</year> + <price>30.00</price> + `) + ).toBe("text/html") + }) + }) + + describe("text/html", () => { + test("should return text/html for valid HTML data", () => { + expect( + detectContentType(` + <!DOCTYPE html> + <html> + <head> + <title>Page Title + + +

This is a Heading

+

This is a paragraph.

+ + + `) + ).toBe("text/html") + }) + + test("should return text/html for invalid HTML data", () => { + expect( + detectContentType(` + + Page Title + +

This is a Heading

+ + + `) + ).toBe("text/html") + }) + + test("should return text/html for unmatched tag", () => { + expect(detectContentType("")).toBe("text/html") + }) + + test("should return text/plain for no valid tags in input", () => { + expect(detectContentType(" { + test("should return application/x-www-form-urlencoded for valid data", () => { + expect(detectContentType("hello=world&hopp=scotch")).toBe( + "application/x-www-form-urlencoded" + ) + }) + + test("should return application/x-www-form-urlencoded for empty pair", () => { + expect(detectContentType("hello=world&hopp=scotch&")).toBe( + "application/x-www-form-urlencoded" + ) + }) + + test("should return application/x-www-form-urlencoded for dangling param", () => { + expect(detectContentType("hello=world&hoppscotch")).toBe( + "application/x-www-form-urlencoded" + ) + }) + + test('should return text/plain for "="', () => { + expect(detectContentType("=")).toBe("text/plain") + }) + + test("should return application/x-www-form-urlencoded for no value field", () => { + expect(detectContentType("hello=")).toBe( + "application/x-www-form-urlencoded" + ) + }) + }) + + describe("multipart/form-data", () => { + test("should return multipart/form-data for valid data", () => { + expect( + detectContentType( + `------WebKitFormBoundaryj3oufpIISPa2DP7c\\r\\nContent-Disposition: form-data; name="EmailAddress"\\r\\n\\r\\ntest@test.com\\r\\n------WebKitFormBoundaryj3oufpIISPa2DP7c\\r\\nContent-Disposition: form-data; name="Entity"\\r\\n\\r\\n1\\r\\n------WebKitFormBoundaryj3oufpIISPa2DP7c--\\r\\n` + ) + ).toBe("multipart/form-data") + }) + + test("should return application/x-www-form-urlencoded for data with only one boundary", () => { + expect( + detectContentType( + `\\r\\nContent-Disposition: form-data; name="EmailAddress"\\r\\n\\r\\ntest@test.com\\r\\n\\r\\nContent-Disposition: form-data; name="Entity"\\r\\n\\r\\n1\\r\\n------WebKitFormBoundaryj3oufpIISPa2DP7c--\\r\\n` + ) + ).toBe("application/x-www-form-urlencoded") + }) + }) +}) diff --git a/packages/hoppscotch-app/helpers/curl/contentParser.ts b/packages/hoppscotch-app/helpers/curl/contentParser.ts index 714f7a5d1..d10aec09c 100644 --- a/packages/hoppscotch-app/helpers/curl/contentParser.ts +++ b/packages/hoppscotch-app/helpers/curl/contentParser.ts @@ -14,6 +14,8 @@ import { safeParseJSON } from "~/helpers/functional/json" export function detectContentType( rawData: string ): HoppRESTReqBody["contentType"] { + if (!rawData) return "text/plain" + let contentType: HoppRESTReqBody["contentType"] if (O.isSome(safeParseJSON(rawData))) { @@ -25,15 +27,21 @@ export function detectContentType( // everything is HTML contentType = "text/html" } - } else if (/([^&=]+)=([^&=]+)/.test(rawData)) { - contentType = "application/x-www-form-urlencoded" } else { contentType = pipe( - rawData.match(/^-{2,}.+\\r\\n/), + rawData.match(/^-{2,}[A-Za-z0-9]+\\r\\n/), O.fromNullable, - O.filter((boundaryMatch) => boundaryMatch && boundaryMatch.length > 1), + O.filter((boundaryMatch) => boundaryMatch.length > 0), O.match( - () => "text/plain", + () => + pipe( + rawData, + O.fromPredicate((rd) => /([^&=]+)=([^&=]*)/.test(rd)), + O.match( + () => "text/plain", + () => "application/x-www-form-urlencoded" + ) + ), () => "multipart/form-data" ) ) @@ -162,7 +170,7 @@ export function parseBody( O.fromNullable, O.map(decodeURIComponent), O.chain((rd) => - pipe(rd.match(/(([^&=]+)=?([^&=]+))/g), O.fromNullable) + pipe(rd.match(/(([^&=]+)=?([^&=]*))/g), O.fromNullable) ), O.map((pairs) => pairs.map((p) => p.replace("=", ": ")).join("\n")) ) @@ -190,7 +198,7 @@ export function parseBody( O.match( () => pipe( - rawData.match(/^-{2,}.+\\r\\n/), + rawData.match(/-{2,}[A-Za-z0-9]+\\r\\n/g), O.fromNullable, O.filter((boundaryMatch) => boundaryMatch.length > 1), O.map((matches) => matches[0]) @@ -220,14 +228,14 @@ export function parseBody( RA.filter((p) => p !== "" && p.includes("name")), RA.map((p) => pipe( - p.replaceAll(/[\r\n]+/g, "\r\n"), + p.replaceAll(/\\r\\n+/g, "\\r\\n"), S.split("\\r\\n"), RA.filter((q) => q !== "") ) ), RA.filterMap((p) => pipe( - p[0].match(/name=(.+)$/), + p[0].match(/ name="(\w+)"/), O.fromNullable, O.filter((nameMatch) => nameMatch.length > 0), O.map((nameMatch) => { diff --git a/packages/hoppscotch-app/helpers/curl/curlparser.ts b/packages/hoppscotch-app/helpers/curl/curlparser.ts index f6472c617..bff7f4bae 100644 --- a/packages/hoppscotch-app/helpers/curl/curlparser.ts +++ b/packages/hoppscotch-app/helpers/curl/curlparser.ts @@ -3,7 +3,6 @@ import parser from "yargs-parser" import * as RA from "fp-ts/ReadonlyArray" import * as O from "fp-ts/Option" import { pipe } from "fp-ts/function" - import { HoppRESTAuth, FormDataKeyValue, @@ -22,26 +21,40 @@ export const parseCurlCommand = (curlCommand: string) => { const parsedArguments = parser(curlCommand) const headers = getHeaders(parsedArguments) + const method = getMethod(parsedArguments) + const urlObject = parseURL(parsedArguments) + let rawContentType: string = "" + let rawData: string | string[] = parsedArguments?.d || "" + let body: string | null = "" + let contentType: HoppRESTReqBody["contentType"] = null + let hasBodyBeenParsed = false if (headers && rawContentType === "") rawContentType = headers["Content-Type"] || headers["content-type"] || "" - let rawData: string | string[] = parsedArguments?.d || "" - const urlObject = parseURL(parsedArguments) - let { queries, danglingParams } = getQueries( urlObject?.searchParams.entries() ) - // if method type is to be set as GET - if (parsedArguments.G && Array.isArray(rawData)) { + if (Array.isArray(rawData)) { const pairs = getParamPairs(rawData) - const newQueries = getQueries(pairs as [string, string][]) - queries = [...queries, ...newQueries.queries] - danglingParams = [...danglingParams, ...newQueries.danglingParams] + + if (parsedArguments.G) { + const newQueries = getQueries(pairs as [string, string][]) + queries = [...queries, ...newQueries.queries] + danglingParams = [...danglingParams, ...newQueries.danglingParams] + hasBodyBeenParsed = true + } else if (rawContentType.includes("application/x-www-form-urlencoded")) { + body = pairs?.map((p) => p.join(": ")).join("\n") || null + contentType = "application/x-www-form-urlencoded" + hasBodyBeenParsed = true + } else { + rawData = rawData.join("") + } } - const urlString = concatParams(urlObject?.origin, danglingParams) || "" + + const urlString = concatParams(urlObject, danglingParams) || "" let multipartUploads: Record = pipe( parsedArguments, @@ -73,16 +86,7 @@ export const parseCurlCommand = (curlCommand: string) => { ) } - const method = getMethod(parsedArguments) - let body: string | null = "" - let contentType: HoppRESTReqBody["contentType"] = null - - // just in case - if (Array.isArray(rawData)) rawData = rawData.join("") - - // if -F is not present, look for content type header - // -G is used to send --data as get params - if (rawContentType !== "multipart/form-data" && !parsedArguments.G) { + if (!hasBodyBeenParsed && typeof rawData === "string") { const tempBody = pipe( O.Do, @@ -203,10 +207,17 @@ function preProcessCurlCommand(curlCommand: string) { // replace string for insomnia for (const r in replaceables) { - curlCommand = curlCommand.replace( - RegExp(` ${r}(["' ])`), - ` ${replaceables[r]}$1` - ) + if (r.includes("data") || r.includes("form") || r.includes("header")) { + curlCommand = curlCommand.replaceAll( + RegExp(`[ \t]${r}(["' ])`, "g"), + ` ${replaceables[r]}$1` + ) + } else { + curlCommand = curlCommand.replace( + RegExp(`[ \t]${r}(["' ])`), + ` ${replaceables[r]}$1` + ) + } } // yargs parses -XPOST as separate arguments. just prescreen for it. @@ -280,7 +291,7 @@ function parseURL(parsedArguments: parser.Arguments) { return pipe( parsedArguments?._[1], O.fromNullable, - O.map((u) => u.replace(/["']/g, "")), + O.map((u) => u.toString().replace(/["']/g, "")), O.map((u) => u.trim()), O.chain((u) => pipe( @@ -346,7 +357,7 @@ function getQueries( const params = [] for (const q of iter) { - if (q[1] === "") { + if (!q[1]) { danglingParams.push(q[0]) continue } @@ -374,13 +385,13 @@ function getQueries( * @param params params without values * @returns origin string concatenated with dngling paramas */ -function concatParams(origin: string | undefined, params: string[]) { +function concatParams(urlObject: URL | undefined, params: string[]) { return pipe( O.Do, O.bind("originString", () => pipe( - origin, + urlObject?.origin, O.fromNullable, O.filter((h) => h !== "") ) @@ -392,8 +403,8 @@ function concatParams(origin: string | undefined, params: string[]) { O.fromNullable, O.filter((dp) => dp.length > 0), O.map(stringArrayJoin("&")), - O.map((h) => originString + "?" + h), - O.getOrElse(() => originString) + O.map((h) => originString + (urlObject?.pathname || "") + "?" + h), + O.getOrElse(() => originString + (urlObject?.pathname || "")) ) ), @@ -449,11 +460,8 @@ function getMethod(parsedArguments: parser.Arguments): string { () => { if (parsedArguments.T) return "put" else if (parsedArguments.I || parsedArguments.head) return "head" - else if ( - parsedArguments.d || - (parsedArguments.F && !(parsedArguments.G || parsedArguments.get)) - ) - return "post" + else if (parsedArguments.G) return "get" + else if (parsedArguments.d || parsedArguments.F) return "post" else return "get" }, (method) => method[0] @@ -616,6 +624,8 @@ export function requestToHoppRequest(parsedCurl: CurlParserRequest) { parsedCurl.hoppHeaders.filter( (header) => header.key !== "Authorization" && + header.key !== "content-type" && + header.key !== "Content-Type" && header.key !== "apikey" && header.key !== "api-key" ) || [] diff --git a/packages/hoppscotch-app/helpers/curl/index.ts b/packages/hoppscotch-app/helpers/curl/index.ts index 88c96d759..6cf96dd0e 100644 --- a/packages/hoppscotch-app/helpers/curl/index.ts +++ b/packages/hoppscotch-app/helpers/curl/index.ts @@ -5,6 +5,7 @@ import { HoppRESTAuth, } from "@hoppscotch/data" import { flow } from "fp-ts/function" +import cloneDeep from "lodash/cloneDeep" import { parseCurlCommand, requestToHoppRequest } from "./curlparser" export type CurlParserRequest = { @@ -25,5 +26,6 @@ export type CurlParserRequest = { export const parseCurlToHoppRESTReq = flow( parseCurlCommand, - requestToHoppRequest + requestToHoppRequest, + cloneDeep )