From d80c44c46b951de28164a048de10292d33869bf7 Mon Sep 17 00:00:00 2001 From: Jing T Date: Fri, 1 Sep 2023 17:27:48 -0400 Subject: [PATCH] Rework Studio/ZAP integration: use notification, not polling In the past, ZAP periodically ping Studio server for UC component states. Instead, ZAP will only issue 1 GET for global UC states / naming. For UC component state changes, ZAP subscribe to Studio server via WebSockets. BUG: ZAPP-1251 --- .../ide-integration/studio-rest-api.ts | 307 ++++++++++++++---- src-electron/ide-integration/studio-types.ts | 22 ++ src-electron/rest/file-ops.js | 46 ++- src-electron/util/studio-util.ts | 18 + src-shared/db-enum.js | 2 +- src/App.vue | 13 +- src/store/zap/actions.js | 13 + src/store/zap/mutations.js | 6 + src/util/common-mixin.js | 26 +- 9 files changed, 371 insertions(+), 82 deletions(-) create mode 100644 src-electron/ide-integration/studio-types.ts create mode 100644 src-electron/util/studio-util.ts diff --git a/src-electron/ide-integration/studio-rest-api.ts b/src-electron/ide-integration/studio-rest-api.ts index 239217d116..42ae6290e5 100644 --- a/src-electron/ide-integration/studio-rest-api.ts +++ b/src-electron/ide-integration/studio-rest-api.ts @@ -21,28 +21,41 @@ */ // dirty flag reporting interval -const UC_COMPONENT_STATE_REPORTING_INTERVAL_ID = 6000 -import axios, { AxiosPromise, AxiosResponse } from 'axios' +import axios, { AxiosResponse } from 'axios' import * as env from '../util/env' import * as dbTypes from '../../src-shared/types/db-types' import * as querySession from '../db/query-session.js' const queryNotification = require('../db/query-session-notification.js') const wsServer = require('../server/ws-server.js') -const dbEnum = require('../../src-shared/db-enum.js') +import * as dbEnum from '../../src-shared/db-enum.js' import * as ucTypes from '../../src-shared/types/uc-component-types' import * as dbMappingTypes from '../types/db-mapping-types' -import * as http from 'http-status-codes' +import { StatusCodes } from 'http-status-codes' import zcl from './zcl.js' +import WebSocket from 'ws' +import { + StudioRestAPI, + StudioWsConnection, + StudioProjectPath, + StudioQueryParams, + StudioWsAPI, + StudioWsMessage, +} from './studio-types' +import { projectName } from '../../src-electron/util/studio-util' const localhost = 'http://127.0.0.1:' -const resGetProjectInfo = '/rest/clic/components/all/project/' -const resAddComponent = '/rest/clic/component/add/project/' -const resRemoveComponent = '/rest/clic/component/remove/project/' +const wsLocalhost = 'ws://127.0.0.1:' -let ucComponentStateReportId: NodeJS.Timeout +// a periodic heartbeat for checking in on Studio server to maintain WS connections +let heartbeatId: NodeJS.Timeout +const heartbeatDelay = 6000 let studioHttpPort: number +let studioWsConnections: StudioWsConnection = {} -function projectPath(db: dbTypes.DbType, sessionId: number) { +async function projectPath( + db: dbTypes.DbType, + sessionId: number +): Promise { return querySession.getSessionKeyValue( db, sessionId, @@ -66,22 +79,42 @@ async function integrationEnabled(db: dbTypes.DbType, sessionId: number) { } /** - * Extract project name from the Studio project path - * @param {} db - * @param {*} sessionId - * @returns '' if retrival failed + * Studio REST API path helper/generator + * @param api + * @param path + * @param queryParams + * @returns */ -function projectName(studioProjectPath: string) { - const prefix = '_2F' - if (studioProjectPath && studioProjectPath.includes(prefix)) { - return studioProjectPath.substr( - studioProjectPath.lastIndexOf(prefix) + prefix.length - ) +function restApiUrl( + api: StudioRestAPI, + path: StudioProjectPath, + queryParams: StudioQueryParams = {} +) { + let base = localhost + studioHttpPort + api + path + let params = Object.entries(queryParams) + if (params.length) { + let queries = new URLSearchParams() + params.forEach(([key, value]) => { + queries.set(key, value) + }) + + return `${base}?${queries.toString()}` } else { - return '' + return base } } +/** + * Studio WebSocket API path helper/generator + * @param api + * @param path + * @param queryParams + * @returns + */ +function wsApiUrl(api: StudioWsAPI, path: StudioProjectPath) { + return wsLocalhost + studioHttpPort + api + path +} + /** * Send HTTP GET request to Studio Jetty server for project information. * @param {} db @@ -93,24 +126,32 @@ async function getProjectInfo( sessionId: number ): Promise<{ data: string[] - status?: http.StatusCodes + status?: StatusCodes }> { let project = await projectPath(db, sessionId) + let studioIntegration = await integrationEnabled(db, sessionId) + if (project) { let name = projectName(project) - let path = localhost + studioHttpPort + resGetProjectInfo + project - env.logDebug(`StudioUC(${name}): GET: ${path}`) - return axios - .get(path) - .then((resp) => { - env.logDebug(`StudioUC(${name}): RESP: ${resp.status}`) - return resp - }) - .catch((err) => { - return { data: [] } - }) + if (studioIntegration) { + let path = restApiUrl(StudioRestAPI.GetProjectInfo, project) + env.logInfo(`StudioUC(${name}): GET: ${path}`) + return axios + .get(path) + .then((resp) => { + env.logInfo(`StudioUC(${name}): RESP: ${resp.status}`) + return resp + }) + .catch((err) => { + env.logInfo(`StudioUC(${name}): ERR: ${err.message}`) + return { data: [] } + }) + } else { + env.logInfo(`StudioUC(${name}): Studio integration is not enabled!`) + return { data: [] } + } } else { - env.logDebug( + env.logInfo( `StudioUC(): Invalid Studio project path specified via project info API!` ) return { data: [] } @@ -199,7 +240,7 @@ async function updateComponentByComponentIds( AxiosResponse | ucTypes.UcComponentUpdateResponseWrapper >[] = [] let project = await projectPath(db, sessionId) - let name = await projectName(project) + let name = projectName(project) if (Object.keys(componentIds).length) { promises = componentIds.map((componentId) => @@ -224,10 +265,15 @@ function httpPostComponentUpdate( componentId: string, add: boolean ) { - let operation = add ? resAddComponent : resRemoveComponent + let operation = add + ? StudioRestAPI.AddComponent + : StudioRestAPI.RemoveComponent let operationText = add ? 'add' : 'remove' + let name = projectName(project) + let path = restApiUrl(operation, project) + env.logInfo(`StudioUC(${name}): POST: ${path}, ${componentId}`) return axios - .post(localhost + studioHttpPort + operation + project, { + .post(path, { componentId: componentId, }) .then((res) => { @@ -251,16 +297,44 @@ function httpPostComponentUpdate( } else { // Actual fail. return { - status: http.StatusCodes.NOT_FOUND, + status: StatusCodes.NOT_FOUND, id: componentId, - data: `StudioUC(${projectName( - project - )}): Failed to ${operationText} component(${componentId})`, + data: `StudioUC(${name}): Failed to ${operationText} component(${componentId})`, } } }) } +/** + * Handles WebSocket messages from Studio server + * @param db + * @param session + * @param message + */ +async function wsMessageHandler( + db: dbTypes.DbType, + session: any, + message: string +) { + let { sessionId } = session + let name = projectName(await projectPath(db, sessionId)) + try { + let resp = JSON.parse(message) + if (resp.msgType == 'updateComponents') { + env.logInfo( + `StudioUC(${name}): Received WebSocket message: ${JSON.stringify( + resp.delta + )}` + ) + sendSelectedUcComponents(db, session, JSON.parse(resp.tree)) + } + } catch (error) { + env.logError( + `StudioUC(${name}): Failed to process WebSocket notification message.` + ) + } +} + /** * Start the dirty flag reporting interval. * @@ -269,33 +343,150 @@ function initIdeIntegration(db: dbTypes.DbType, studioPort: number) { studioHttpPort = studioPort if (studioPort) { - ucComponentStateReportId = setInterval(() => { - sendUcComponentStateReport(db) - }, UC_COMPONENT_STATE_REPORTING_INTERVAL_ID) + heartbeatId = setInterval(async () => { + let sessions = await querySession.getAllSessions(db) + for (const session of sessions) { + await verifyWsConnection(db, session, wsMessageHandler) + } + }, heartbeatDelay) + } +} + +/** + * Check WebSocket connections between backend and Studio jetty server. + * If project is opened, verify connection is open. + * If project is closed, close ws connection as well. + * + * @param db + * @param sessionId + */ +async function verifyWsConnection( + db: dbTypes.DbType, + session: any, + messageHandler: StudioWsMessage +) { + try { + let { sessionId } = session + let path = await projectPath(db, sessionId) + if (path) { + if (await isProjectActive(path)) { + await wsConnect(db, session, path, messageHandler) + } else { + wsDisconnect(db, session, path) + } + } + } catch (error: any) { + env.logInfo(error.toString()) } } +/** + * Utility function for making websocket connection to Studio server + * @param sessionId + * @param path + * @returns + */ +async function wsConnect( + db: dbTypes.DbType, + session: any, + path: StudioProjectPath, + handler: StudioWsMessage +) { + let { sessionId } = session + let ws = studioWsConnections[sessionId] + if (ws && ws.readyState == WebSocket.OPEN) { + return ws + } else { + ws?.terminate() + + let wsPath = wsApiUrl(StudioWsAPI.WsServerNotification, path) + let name = projectName(path) + ws = new WebSocket(wsPath) + env.logInfo(`StudioUC(${name}): WS connecting to ${wsPath}`) + + ws.on('error', function () { + studioWsConnections[sessionId] = null + return null + }) + + ws.on('open', function () { + studioWsConnections[sessionId] = ws + env.logInfo(`StudioUC(${name}): WS connected.`) + return ws + }) + + ws.on('message', function (data) { + handler(db, session, data.toString()) + }) + } +} + +async function wsDisconnect( + db: dbTypes.DbType, + session: any, + path: StudioProjectPath +) { + let { sessionId } = session + if (studioWsConnections[sessionId]) { + env.logInfo(`StudioUC(${projectName(path)}): WS disconnected.`) + studioWsConnections[sessionId]?.close() + studioWsConnections[sessionId] = null + } +} + +/** + * Check if a specific Studio project (.slcp) file has been opened or not. + * + * Context: To get proper WebSocket notification for change in project states, + * that specific project needs to be opened already. Otherwise, no notification + * will happen. + * + * DependsComponent API used as a quick way to check if the project is opened or not + * If project is open/valid, the API will respond with "Component not found in project" + * Otherwise, "Project does not exists" + * + * @param path + */ +async function isProjectActive(path: StudioProjectPath): Promise { + if (!path) { + return false + } + + let url = restApiUrl(StudioRestAPI.DependsComponent, path) + return axios + .get(url) + .then((resp) => { + return true + }) + .catch((err) => { + let { response } = err + if (response.status == StatusCodes.BAD_REQUEST && response.data) { + return !response.data.includes('Project does not exists') + } + + return false + }) +} + /** * Clears up the reporting interval. */ function deinitIdeIntegration() { - if (ucComponentStateReportId) clearInterval(ucComponentStateReportId) + if (heartbeatId) clearInterval(heartbeatId) } -async function sendUcComponentStateReport(db: dbTypes.DbType) { - let sessions = await querySession.getAllSessions(db) - for (const session of sessions) { - let socket = wsServer.clientSocket(session.sessionKey) - let studioIntegration = await integrationEnabled(db, session.sessionId) - if (socket && studioIntegration) { - getProjectInfo(db, session.sessionId).then((resp) => { - if (resp.status == http.StatusCodes.OK) - wsServer.sendWebSocketMessage(socket, { - category: dbEnum.wsCategory.ucComponentStateReport, - payload: resp.data, - }) - }) - } +async function sendSelectedUcComponents( + db: dbTypes.DbType, + session: any, + ucComponentStates: string +) { + let socket = wsServer.clientSocket(session.sessionKey) + let studioIntegration = await integrationEnabled(db, session.sessionId) + if (socket && studioIntegration) { + wsServer.sendWebSocketMessage(socket, { + category: dbEnum.wsCategory.updateSelectedUcComponents, + payload: ucComponentStates, + }) } } diff --git a/src-electron/ide-integration/studio-types.ts b/src-electron/ide-integration/studio-types.ts new file mode 100644 index 0000000000..0cef740776 --- /dev/null +++ b/src-electron/ide-integration/studio-types.ts @@ -0,0 +1,22 @@ +import WebSocket from 'ws' +import { DbType } from '../../src-shared/types/db-types' + +export enum StudioRestAPI { + GetProjectInfo = '/rest/clic/components/all/project/', + AddComponent = '/rest/clic/component/add/project/', + RemoveComponent = '/rest/clic/component/remove/project/', + DependsComponent = '/rest/clic/component/depends/project/', +} + +export enum StudioWsAPI { + WsServerNotification = '/ws/clic/server/notifications/project/', +} + +export type StudioProjectPath = string +export type StudioQueryParams = { [key: string]: string } +export type StudioWsMessage = ( + db: DbType, + session: any, + message: string +) => void +export type StudioWsConnection = { [key: number]: WebSocket | null } diff --git a/src-electron/rest/file-ops.js b/src-electron/rest/file-ops.js index 5c316f99e2..92ba9127b2 100644 --- a/src-electron/rest/file-ops.js +++ b/src-electron/rest/file-ops.js @@ -29,9 +29,9 @@ const path = require('path') const { StatusCodes } = require('http-status-codes') const querySession = require('../db/query-session.js') const queryNotification = require('../db/query-session-notification.js') -const querystring = require('querystring') const dbEnum = require('../../src-shared/db-enum.js') const studio = require('../ide-integration/studio-rest-api') +import { projectName } from '../util/studio-util' /** * HTTP POST: IDE open @@ -49,22 +49,25 @@ function httpPostFileOpen(db) { zapFilePath = file ideProjectPath = query.get('studioProject') } - let name = '' if (zapFilePath) { + let p if (studio.integrationEnabled(db, req.zapSessionId)) { - name = path.posix.dirname( + p = path.posix.dirname( path.posix.dirname(path.posix.dirname(zapFilePath)) ) } else { - name = path.posix.basename(zapFilePath) + p = path.posix.basename(zapFilePath) } + let name = projectName(p) env.logInfo(`Loading project(${name})`) try { // set path before importDataFromFile() to avoid triggering DIRTY flag if (ideProjectPath) { - env.logInfo(`IDE: setting project path(${name}) to ${ideProjectPath}`) + env.logInfo( + `StudioUC(${name}): Setting project path to ${ideProjectPath}` + ) } // store studio project path await querySession.updateSessionKeyValue( @@ -96,15 +99,33 @@ function httpPostFileOpen(db) { message: e.message, stack: e.stack, } - studio.sendSessionCreationErrorStatus(db, errMsg.message, req.zapSessionId) + studio.sendSessionCreationErrorStatus( + db, + errMsg.message, + req.zapSessionId + ) env.logError(e.message) - queryNotification.setNotification(db, 'ERROR', errMsg.message, req.zapSessionId, 1, 0) + queryNotification.setNotification( + db, + 'ERROR', + errMsg.message, + req.zapSessionId, + 1, + 0 + ) res.status(StatusCodes.INTERNAL_SERVER_ERROR).json(errMsg) } } else { let msg = `Opening/Loading project: Missing zap file path.` env.logWarning(msg) - queryNotification.setNotification(db, 'WARNING', errMsg.message, req.zapSessionId, 2, 0) + queryNotification.setNotification( + db, + 'WARNING', + errMsg.message, + req.zapSessionId, + 2, + 0 + ) res.status(StatusCodes.BAD_REQUEST).send({ error: msg }) } } @@ -150,7 +171,14 @@ function httpPostFileSave(db) { } catch (err) { let msg = `Unable to save project.` env.logError(msg, err) - queryNotification.setNotification(db, 'ERROR', msg, req.zapSessionId, 1, 0) + queryNotification.setNotification( + db, + 'ERROR', + msg, + req.zapSessionId, + 1, + 0 + ) res.status(StatusCodes.INTERNAL_SERVER_ERROR).json(err) } } else { diff --git a/src-electron/util/studio-util.ts b/src-electron/util/studio-util.ts new file mode 100644 index 0000000000..e779f4184e --- /dev/null +++ b/src-electron/util/studio-util.ts @@ -0,0 +1,18 @@ +import path from 'path' +import { StudioProjectPath } from '../ide-integration/studio-types' + +/** + * Extract project name from the Studio project path + * @param {} db + * @param {*} sessionId + * @returns '' if parsing fails + */ +export function projectName(studioProjectPath: StudioProjectPath) { + // undo the manual trickery from the Studio side. + try { + let p = path.parse(decodeURIComponent(studioProjectPath.replace(/_/g, '%'))) + return p.name + } catch (error) { + return '' + } +} diff --git a/src-shared/db-enum.js b/src-shared/db-enum.js index 75d15316e8..0da9ef80ac 100644 --- a/src-shared/db-enum.js +++ b/src-shared/db-enum.js @@ -107,7 +107,7 @@ exports.wsCategory = { notificationInfo: 'notificationInfo', sessionCreationError: 'sessionCreationError', componentUpdateStatus: 'componentUpdateStatus', - ucComponentStateReport: 'ucComponentStateReport', + updateSelectedUcComponents: 'updateSelectedUcComponents', init: 'init', tick: 'tick', } diff --git a/src/App.vue b/src/App.vue index ac625c7a7f..17f06527d5 100644 --- a/src/App.vue +++ b/src/App.vue @@ -205,9 +205,16 @@ export default defineComponent({ this.$q.loading.hide() }) - this.$onWebSocket(dbEnum.wsCategory.ucComponentStateReport, (resp) => { - this.$store.dispatch('zap/updateUcComponentState', resp) - }) + // load initial UC component state + this.$store.dispatch(`zap/loadUcComponentState`) + + // handles UC component state change events + this.$onWebSocket( + dbEnum.wsCategory.updateSelectedUcComponents, + (resp) => { + this.$store.dispatch('zap/updateSelectedUcComponentState', resp) + } + ) }, addClassToBody() { if (this.uiThemeCategory === 'zigbee') { diff --git a/src/store/zap/actions.js b/src/store/zap/actions.js index 7c7dee052d..1f252d1272 100644 --- a/src/store/zap/actions.js +++ b/src/store/zap/actions.js @@ -772,6 +772,11 @@ export function clearLastSelectedDomain(context) { context.commit('clearLastSelectedDomain') } +export async function loadUcComponentState(context) { + let resp = await axiosRequests.$serverGet(restApi.uc.componentTree) + updateUcComponentState(context, resp.data) +} + export function updateUcComponentState(context, projectInfo) { let ucComponents = Util.getUcComponents(projectInfo) let selectedUcComponents = Util.getSelectedUcComponents(ucComponents) @@ -781,6 +786,14 @@ export function updateUcComponentState(context, projectInfo) { }) } +export function updateSelectedUcComponentState(context, projectInfo) { + let ucComponents = Util.getUcComponents(projectInfo) + let selectedUcComponents = Util.getSelectedUcComponents(ucComponents) + context.commit('updateSelectedUcComponentState', { + selectedUcComponents, + }) +} + export function loadZclClusterToUcComponentDependencyMap(context) { axiosRequests .$serverGet(`/zclExtension/cluster/component`) diff --git a/src/store/zap/mutations.js b/src/store/zap/mutations.js index 3dd8cf64bf..c015763eb2 100644 --- a/src/store/zap/mutations.js +++ b/src/store/zap/mutations.js @@ -540,6 +540,12 @@ export function updateUcComponentState(state, data) { } } +export function updateSelectedUcComponentState(state, data) { + if (data != null) { + vue3Set(state.studio, 'selectedUcComponents', data.selectedUcComponents) + } +} + export function loadZclClusterToUcComponentDependencyMap(state, map) { if (map != null) vue3Set(state.studio, 'zclSdkExtClusterToUcComponentMap', map) diff --git a/src/util/common-mixin.js b/src/util/common-mixin.js index 2706a83d3c..ea3e207fd1 100644 --- a/src/util/common-mixin.js +++ b/src/util/common-mixin.js @@ -204,28 +204,32 @@ export default { * @param {*} params */ missingUcComponentDependencies(cluster) { + let requiredComponentIdList = [] + let roles = [] let hasClient = this.selectionClients.includes(cluster.id) - let hasServer = this.selectionServers.includes(cluster.id) - - let requiredList = [] if (hasClient) { - let compList = this.ucComponentRequiredByCluster(cluster, 'client') - requiredList = requiredList.concat( - compList.map((x) => this.sdkExtUcComponentId(x)) - ) + roles.push('client') } + let hasServer = this.selectionServers.includes(cluster.id) if (hasServer) { - let compList = this.ucComponentRequiredByCluster(cluster, 'server') - requiredList = requiredList.concat( - compList.map((x) => this.sdkExtUcComponentId(x)) + roles.push('server') + } + + for (const role of roles) { + let components = this.ucComponentRequiredByCluster(cluster, role) + requiredComponentIdList.push( + ...components.map((c) => this.sdkExtUcComponentId(c)) ) } let selectedUcComponentIds = Util.getClusterIdsByUcComponents( this.$store.state.zap.studio.selectedUcComponents ) - return requiredList.filter((id) => !selectedUcComponentIds.includes(id)) + + return requiredComponentIdList.filter( + (id) => !selectedUcComponentIds.includes(id) + ) }, ucComponentRequiredByCluster(cluster, role) {