refactor: real-time system (#2228)

Co-authored-by: Andrew Bastin <andrewbastin.k@gmail.com>
Co-authored-by: liyasthomas <liyascthomas@gmail.com>

packages/hoppscotch-app/helpers/editor/codemirror.ts

@@ -28,8 +28,6 @@ import { javascriptLanguage } from "@codemirror/lang-javascript"
 import { xmlLanguage } from "@codemirror/lang-xml"
 import { jsonLanguage } from "@codemirror/lang-json"
 import { GQLLanguage } from "@hoppscotch/codemirror-lang-graphql"
-import { pipe } from "fp-ts/function"
-import * as O from "fp-ts/Option"
 import { StreamLanguage } from "@codemirror/stream-parser"
 import { html } from "@codemirror/legacy-modes/mode/xml"
 import { shell } from "@codemirror/legacy-modes/mode/shell"
@@ -96,8 +94,10 @@ const hoppCompleterExt = (completer: Completer): Extension => {
   })
 }
 
-const hoppLinterExt = (hoppLinter: LinterDefinition): Extension => {
+const hoppLinterExt = (hoppLinter: LinterDefinition | undefined): Extension => {
   return linter(async (view) => {
+    if (!hoppLinter) return []
+
     // Requires full document scan, hence expensive on big files, force disable on big files ?
     const linterResult = await hoppLinter(
       view.state.doc.toJSON().join(view.state.lineBreak)
@@ -119,16 +119,16 @@ const hoppLinterExt = (hoppLinter: LinterDefinition): Extension => {
 }
 
 const hoppLang = (
-  language: Language,
+  language: Language | undefined,
   linter?: LinterDefinition | undefined,
   completer?: Completer | undefined
-) => {
+): Extension | LanguageSupport => {
   const exts: Extension[] = []
 
-  if (linter) exts.push(hoppLinterExt(linter))
+  exts.push(hoppLinterExt(linter))
   if (completer) exts.push(hoppCompleterExt(completer))
 
-  return new LanguageSupport(language, exts)
+  return language ? new LanguageSupport(language, exts) : exts
 }
 
 const getLanguage = (langMime: string): Language | null => {
@@ -156,12 +156,7 @@ const getEditorLanguage = (
   langMime: string,
   linter: LinterDefinition | undefined,
   completer: Completer | undefined
-): Extension =>
-  pipe(
-    O.fromNullable(getLanguage(langMime)),
-    O.map((lang) => hoppLang(lang, linter, completer)),
-    O.getOrElseW(() => [])
-  )
+): Extension => hoppLang(getLanguage(langMime) ?? undefined, linter, completer)
 
 export function useCodemirror(
   el: Ref<any | null>,

packages/hoppscotch-app/helpers/realtime/MQTTConnection.ts

@@ -0,0 +1,223 @@
+import Paho, { ConnectionOptions } from "paho-mqtt"
+import { BehaviorSubject, Subject } from "rxjs"
+import { logHoppRequestRunToAnalytics } from "../fb/analytics"
+
+export type MQTTMessage = { topic: string; message: string }
+export type MQTTError =
+  | { type: "CONNECTION_NOT_ESTABLISHED"; value: unknown }
+  | { type: "CONNECTION_LOST" }
+  | { type: "CONNECTION_FAILED" }
+  | { type: "SUBSCRIPTION_FAILED"; topic: string }
+  | { type: "PUBLISH_ERROR"; topic: string; message: string }
+
+export type MQTTEvent = { time: number } & (
+  | { type: "CONNECTING" }
+  | { type: "CONNECTED" }
+  | { type: "MESSAGE_SENT"; message: MQTTMessage }
+  | { type: "SUBSCRIBED"; topic: string }
+  | { type: "SUBSCRIPTION_FAILED"; topic: string }
+  | { type: "MESSAGE_RECEIVED"; message: MQTTMessage }
+  | { type: "DISCONNECTED"; manual: boolean }
+  | { type: "ERROR"; error: MQTTError }
+)
+
+export type ConnectionState = "CONNECTING" | "CONNECTED" | "DISCONNECTED"
+
+export class MQTTConnection {
+  subscriptionState$ = new BehaviorSubject<boolean>(false)
+  connectionState$ = new BehaviorSubject<ConnectionState>("DISCONNECTED")
+  event$: Subject<MQTTEvent> = new Subject()
+
+  private mqttClient: Paho.Client | undefined
+  private manualDisconnect = false
+
+  private addEvent(event: MQTTEvent) {
+    this.event$.next(event)
+  }
+
+  connect(url: string, username: string, password: string) {
+    try {
+      this.connectionState$.next("CONNECTING")
+
+      this.addEvent({
+        time: Date.now(),
+        type: "CONNECTING",
+      })
+
+      const parseUrl = new URL(url)
+      const { hostname, pathname, port } = parseUrl
+      this.mqttClient = new Paho.Client(
+        `${hostname + (pathname !== "/" ? pathname : "")}`,
+        port !== "" ? Number(port) : 8081,
+        "hoppscotch"
+      )
+      const connectOptions: ConnectionOptions = {
+        onSuccess: this.onConnectionSuccess.bind(this),
+        onFailure: this.onConnectionFailure.bind(this),
+        useSSL: parseUrl.protocol !== "ws:",
+      }
+      if (username !== "") {
+        connectOptions.userName = username
+      }
+      if (password !== "") {
+        connectOptions.password = password
+      }
+      this.mqttClient.connect(connectOptions)
+      this.mqttClient.onConnectionLost = this.onConnectionLost.bind(this)
+      this.mqttClient.onMessageArrived = this.onMessageArrived.bind(this)
+    } catch (e) {
+      this.handleError(e)
+    }
+
+    logHoppRequestRunToAnalytics({
+      platform: "mqtt",
+    })
+  }
+
+  onConnectionFailure() {
+    this.connectionState$.next("DISCONNECTED")
+    this.addEvent({
+      time: Date.now(),
+      type: "ERROR",
+      error: {
+        type: "CONNECTION_FAILED",
+      },
+    })
+  }
+
+  onConnectionSuccess() {
+    this.connectionState$.next("CONNECTED")
+    this.addEvent({
+      type: "CONNECTED",
+      time: Date.now(),
+    })
+  }
+
+  onConnectionLost() {
+    this.connectionState$.next("DISCONNECTED")
+    if (this.manualDisconnect) {
+      this.addEvent({
+        time: Date.now(),
+        type: "DISCONNECTED",
+        manual: this.manualDisconnect,
+      })
+    } else {
+      this.addEvent({
+        time: Date.now(),
+        type: "ERROR",
+        error: {
+          type: "CONNECTION_LOST",
+        },
+      })
+    }
+    this.manualDisconnect = false
+    this.subscriptionState$.next(false)
+  }
+
+  onMessageArrived({
+    payloadString: message,
+    destinationName: topic,
+  }: {
+    payloadString: string
+    destinationName: string
+  }) {
+    this.addEvent({
+      time: Date.now(),
+      type: "MESSAGE_RECEIVED",
+      message: {
+        topic,
+        message,
+      },
+    })
+  }
+
+  private handleError(error: unknown) {
+    this.disconnect()
+    this.addEvent({
+      time: Date.now(),
+      type: "ERROR",
+      error: {
+        type: "CONNECTION_NOT_ESTABLISHED",
+        value: error,
+      },
+    })
+  }
+
+  publish(topic: string, message: string) {
+    if (this.connectionState$.value === "DISCONNECTED") return
+
+    try {
+      // it was publish
+      this.mqttClient?.send(topic, message, 0, false)
+      this.addEvent({
+        time: Date.now(),
+        type: "MESSAGE_SENT",
+        message: {
+          topic,
+          message,
+        },
+      })
+    } catch (e) {
+      this.addEvent({
+        time: Date.now(),
+        type: "ERROR",
+        error: {
+          type: "PUBLISH_ERROR",
+          topic,
+          message,
+        },
+      })
+    }
+  }
+
+  subscribe(topic: string) {
+    try {
+      this.mqttClient?.subscribe(topic, {
+        onSuccess: this.usubSuccess.bind(this, topic),
+        onFailure: this.usubFailure.bind(this, topic),
+      })
+    } catch (e) {
+      this.addEvent({
+        time: Date.now(),
+        type: "ERROR",
+        error: {
+          type: "SUBSCRIPTION_FAILED",
+          topic,
+        },
+      })
+    }
+  }
+
+  usubSuccess(topic: string) {
+    this.subscriptionState$.next(!this.subscriptionState$.value)
+    this.addEvent({
+      time: Date.now(),
+      type: "SUBSCRIBED",
+      topic,
+    })
+  }
+
+  usubFailure(topic: string) {
+    this.addEvent({
+      time: Date.now(),
+      type: "ERROR",
+      error: {
+        type: "SUBSCRIPTION_FAILED",
+        topic,
+      },
+    })
+  }
+
+  unsubscribe(topic: string) {
+    this.mqttClient?.unsubscribe(topic, {
+      onSuccess: this.usubSuccess.bind(this, topic),
+      onFailure: this.usubFailure.bind(this, topic),
+    })
+  }
+
+  disconnect() {
+    this.manualDisconnect = true
+    this.mqttClient?.disconnect()
+    this.connectionState$.next("DISCONNECTED")
+  }
+}

packages/hoppscotch-app/helpers/realtime/SIOClients.ts

@@ -0,0 +1,84 @@
+import wildcard from "socketio-wildcard"
+import ClientV2 from "socket.io-client-v2"
+import { io as ClientV4, Socket as SocketV4 } from "socket.io-client-v4"
+import { io as ClientV3, Socket as SocketV3 } from "socket.io-client-v3"
+
+type Options = {
+  path: string
+  auth: {
+    token: string | undefined
+  }
+}
+
+type PossibleEvent =
+  | "connect"
+  | "connect_error"
+  | "reconnect_error"
+  | "error"
+  | "disconnect"
+  | "*"
+
+export interface SIOClient {
+  connect(url: string, opts?: Options): void
+  on(event: PossibleEvent, cb: (data: any) => void): void
+  emit(event: string, data: any, cb: (data: any) => void): void
+  close(): void
+}
+
+export class SIOClientV4 implements SIOClient {
+  private client: SocketV4 | undefined
+  connect(url: string, opts?: Options) {
+    this.client = ClientV4(url, opts)
+  }
+
+  on(event: PossibleEvent, cb: (data: any) => void) {
+    this.client?.on(event, cb)
+  }
+
+  emit(event: string, data: any, cb: (data: any) => void): void {
+    this.client?.emit(event, data, cb)
+  }
+
+  close(): void {
+    this.client?.close()
+  }
+}
+
+export class SIOClientV3 implements SIOClient {
+  private client: SocketV3 | undefined
+  connect(url: string, opts?: Options) {
+    this.client = ClientV3(url, opts)
+  }
+
+  on(event: PossibleEvent, cb: (data: any) => void): void {
+    this.client?.on(event, cb)
+  }
+
+  emit(event: string, data: any, cb: (data: any) => void): void {
+    this.client?.emit(event, data, cb)
+  }
+
+  close(): void {
+    this.client?.close()
+  }
+}
+
+export class SIOClientV2 implements SIOClient {
+  private client: any | undefined
+  connect(url: string, opts?: Options) {
+    this.client = new ClientV2(url, opts)
+    wildcard(ClientV2.Manager)(this.client)
+  }
+
+  on(event: PossibleEvent, cb: (data: any) => void): void {
+    this.client?.on(event, cb)
+  }
+
+  emit(event: string, data: any, cb: (data: any) => void): void {
+    this.client?.emit(event, data, cb)
+  }
+
+  close(): void {
+    this.client?.close()
+  }
+}

packages/hoppscotch-app/helpers/realtime/SIOConnection.ts

@@ -0,0 +1,163 @@
+import { BehaviorSubject, Subject } from "rxjs"
+import { logHoppRequestRunToAnalytics } from "../fb/analytics"
+import { SIOClientV2, SIOClientV3, SIOClientV4, SIOClient } from "./SIOClients"
+import { SIOClientVersion } from "~/newstore/SocketIOSession"
+
+export const SOCKET_CLIENTS = {
+  v2: SIOClientV2,
+  v3: SIOClientV3,
+  v4: SIOClientV4,
+} as const
+
+type SIOAuth = { type: "None" } | { type: "Bearer"; token: string }
+
+export type ConnectionOption = {
+  url: string
+  path: string
+  clientVersion: SIOClientVersion
+  auth: SIOAuth | undefined
+}
+
+export type SIOMessage = {
+  eventName: string
+  value: unknown
+}
+
+type SIOErrorType = "CONNECTION" | "RECONNECT_ERROR" | "UNKNOWN"
+export type SIOError = {
+  type: SIOErrorType
+  value: unknown
+}
+
+export type SIOEvent = { time: number } & (
+  | { type: "CONNECTING" }
+  | { type: "CONNECTED" }
+  | { type: "MESSAGE_SENT"; message: SIOMessage }
+  | { type: "MESSAGE_RECEIVED"; message: SIOMessage }
+  | { type: "DISCONNECTED"; manual: boolean }
+  | { type: "ERROR"; error: SIOError }
+)
+
+export type ConnectionState = "CONNECTING" | "CONNECTED" | "DISCONNECTED"
+
+export class SIOConnection {
+  connectionState$: BehaviorSubject<ConnectionState>
+  event$: Subject<SIOEvent> = new Subject()
+  socket: SIOClient | undefined
+  constructor() {
+    this.connectionState$ = new BehaviorSubject<ConnectionState>("DISCONNECTED")
+  }
+
+  private addEvent(event: SIOEvent) {
+    this.event$.next(event)
+  }
+
+  connect({ url, path, clientVersion, auth }: ConnectionOption) {
+    this.connectionState$.next("CONNECTING")
+    this.addEvent({
+      time: Date.now(),
+      type: "CONNECTING",
+    })
+    try {
+      this.socket = new SOCKET_CLIENTS[clientVersion]()
+
+      if (auth?.type === "Bearer") {
+        this.socket.connect(url, {
+          path,
+          auth: {
+            token: auth.token,
+          },
+        })
+      } else {
+        this.socket.connect(url)
+      }
+
+      this.socket.on("connect", () => {
+        this.connectionState$.next("CONNECTED")
+        this.addEvent({
+          type: "CONNECTED",
+          time: Date.now(),
+        })
+      })
+
+      this.socket.on("*", ({ data }: { data: string[] }) => {
+        const [eventName, message] = data
+        this.addEvent({
+          message: { eventName, value: message },
+          type: "MESSAGE_RECEIVED",
+          time: Date.now(),
+        })
+      })
+
+      this.socket.on("connect_error", (error: unknown) => {
+        this.handleError(error, "CONNECTION")
+      })
+
+      this.socket.on("reconnect_error", (error: unknown) => {
+        this.handleError(error, "RECONNECT_ERROR")
+      })
+
+      this.socket.on("error", (error: unknown) => {
+        this.handleError(error, "UNKNOWN")
+      })
+
+      this.socket.on("disconnect", () => {
+        this.connectionState$.next("DISCONNECTED")
+        this.addEvent({
+          type: "DISCONNECTED",
+          time: Date.now(),
+          manual: true,
+        })
+      })
+    } catch (error) {
+      this.handleError(error, "CONNECTION")
+    }
+
+    logHoppRequestRunToAnalytics({
+      platform: "socketio",
+    })
+  }
+
+  private handleError(error: unknown, type: SIOErrorType) {
+    this.disconnect()
+    this.addEvent({
+      time: Date.now(),
+      type: "ERROR",
+      error: {
+        type,
+        value: error,
+      },
+    })
+  }
+
+  sendMessage(event: { message: string; eventName: string }) {
+    if (this.connectionState$.value === "DISCONNECTED") return
+    const { message, eventName } = event
+
+    this.socket?.emit(eventName, message, (data) => {
+      // receive response from server
+      this.addEvent({
+        time: Date.now(),
+        type: "MESSAGE_RECEIVED",
+        message: {
+          eventName,
+          value: data,
+        },
+      })
+    })
+
+    this.addEvent({
+      time: Date.now(),
+      type: "MESSAGE_SENT",
+      message: {
+        eventName,
+        value: message,
+      },
+    })
+  }
+
+  disconnect() {
+    this.socket?.close()
+    this.connectionState$.next("DISCONNECTED")
+  }
+}

packages/hoppscotch-app/helpers/realtime/SSEConnection.ts

@@ -0,0 +1,86 @@
+import { BehaviorSubject, Subject } from "rxjs"
+import { logHoppRequestRunToAnalytics } from "../fb/analytics"
+
+export type SSEEvent = { time: number } & (
+  | { type: "STARTING" }
+  | { type: "STARTED" }
+  | { type: "MESSAGE_RECEIVED"; message: string }
+  | { type: "STOPPED"; manual: boolean }
+  | { type: "ERROR"; error: Event | null }
+)
+
+export type ConnectionState = "STARTING" | "STARTED" | "STOPPED"
+
+export class SSEConnection {
+  connectionState$: BehaviorSubject<ConnectionState>
+  event$: Subject<SSEEvent> = new Subject()
+  sse: EventSource | undefined
+  constructor() {
+    this.connectionState$ = new BehaviorSubject<ConnectionState>("STOPPED")
+  }
+
+  private addEvent(event: SSEEvent) {
+    this.event$.next(event)
+  }
+
+  start(url: string, eventType: string) {
+    this.connectionState$.next("STARTING")
+    this.addEvent({
+      time: Date.now(),
+      type: "STARTING",
+    })
+    if (typeof EventSource !== "undefined") {
+      try {
+        this.sse = new EventSource(url)
+        this.sse.onopen = () => {
+          this.connectionState$.next("STARTED")
+          this.addEvent({
+            type: "STARTED",
+            time: Date.now(),
+          })
+        }
+        this.sse.onerror = this.handleError
+        this.sse.addEventListener(eventType, ({ data }) => {
+          this.addEvent({
+            type: "MESSAGE_RECEIVED",
+            message: data,
+            time: Date.now(),
+          })
+        })
+      } catch (error) {
+        // A generic event type returned if anything goes wrong or browser doesn't support SSE
+        // https://developer.mozilla.org/en-US/docs/Web/API/EventSource/error_event#event_type
+        this.handleError(error as Event)
+      }
+    } else {
+      this.addEvent({
+        type: "ERROR",
+        time: Date.now(),
+        error: null,
+      })
+    }
+
+    logHoppRequestRunToAnalytics({
+      platform: "sse",
+    })
+  }
+
+  private handleError(error: Event) {
+    this.stop()
+    this.addEvent({
+      time: Date.now(),
+      type: "ERROR",
+      error,
+    })
+  }
+
+  stop() {
+    this.sse?.close()
+    this.connectionState$.next("STOPPED")
+    this.addEvent({
+      type: "STOPPED",
+      time: Date.now(),
+      manual: true,
+    })
+  }
+}

packages/hoppscotch-app/helpers/realtime/WSConnection.ts

@@ -0,0 +1,102 @@
+import { BehaviorSubject, Subject } from "rxjs"
+import { logHoppRequestRunToAnalytics } from "../fb/analytics"
+
+export type WSErrorMessage = SyntaxError | Event
+
+export type WSEvent = { time: number } & (
+  | { type: "CONNECTING" }
+  | { type: "CONNECTED" }
+  | { type: "MESSAGE_SENT"; message: string }
+  | { type: "MESSAGE_RECEIVED"; message: string }
+  | { type: "DISCONNECTED"; manual: boolean }
+  | { type: "ERROR"; error: WSErrorMessage }
+)
+
+export type ConnectionState = "CONNECTING" | "CONNECTED" | "DISCONNECTED"
+
+export class WSConnection {
+  connectionState$: BehaviorSubject<ConnectionState>
+  event$: Subject<WSEvent> = new Subject()
+  socket: WebSocket | undefined
+
+  constructor() {
+    this.connectionState$ = new BehaviorSubject<ConnectionState>("DISCONNECTED")
+  }
+
+  private addEvent(event: WSEvent) {
+    this.event$.next(event)
+  }
+
+  connect(url: string, protocols: string[]) {
+    try {
+      this.connectionState$.next("CONNECTING")
+      this.socket = new WebSocket(url, protocols)
+
+      this.addEvent({
+        time: Date.now(),
+        type: "CONNECTING",
+      })
+
+      this.socket.onopen = () => {
+        this.connectionState$.next("CONNECTED")
+        this.addEvent({
+          type: "CONNECTED",
+          time: Date.now(),
+        })
+      }
+
+      this.socket.onerror = (error) => {
+        this.handleError(error)
+      }
+
+      this.socket.onclose = () => {
+        this.connectionState$.next("DISCONNECTED")
+        this.addEvent({
+          type: "DISCONNECTED",
+          time: Date.now(),
+          manual: true,
+        })
+      }
+
+      this.socket.onmessage = ({ data }) => {
+        this.addEvent({
+          time: Date.now(),
+          type: "MESSAGE_RECEIVED",
+          message: data,
+        })
+      }
+    } catch (error) {
+      // We will have SyntaxError if anything goes wrong with WebSocket constructor
+      // See https://developer.mozilla.org/en-US/docs/Web/API/WebSocket/WebSocket#exceptions
+      this.handleError(error as SyntaxError)
+    }
+
+    logHoppRequestRunToAnalytics({
+      platform: "wss",
+    })
+  }
+
+  private handleError(error: WSErrorMessage) {
+    this.disconnect()
+    this.addEvent({
+      time: Date.now(),
+      type: "ERROR",
+      error,
+    })
+  }
+
+  sendMessage(event: { message: string; eventName: string }) {
+    if (this.connectionState$.value === "DISCONNECTED") return
+    const { message } = event
+    this.socket?.send(message)
+    this.addEvent({
+      time: Date.now(),
+      type: "MESSAGE_SENT",
+      message,
+    })
+  }
+
+  disconnect() {
+    this.socket?.close()
+  }
+}

packages/hoppscotch-app/helpers/types/HoppRealtimeLog.ts

@@ -1,8 +1,9 @@
 export type HoppRealtimeLogLine = {
+  prefix?: string
   payload: string
   source: string
   color?: string
-  ts: string
+  ts: number | undefined
 }
 
 export type HoppRealtimeLog = HoppRealtimeLogLine[]