add timing and rate limits to prometheus

skadefro · skadefro · commit 021dcd89413d · 2020-11-22T14:07:01.000+01:00
diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts
@@ -69,13 +69,16 @@ export class Message {
                 // console.log("SOCKET_NO_RATE_LIMIT consumedPoints: " + res.consumedPoints + " remainingPoints: " + res.remainingPoints);
             } catch (error) {
                 if (error.consumedPoints) {
+                    WebSocketServer.websocket_rate_limit.inc();
+                    WebSocketServer.websocket_rate_limit.labels(command).inc();
                     if ((error.consumedPoints % 100) == 0) cli._logger.debug("[" + username + "/" + cli.clientagent + "/" + cli.id + "] SOCKET_RATE_LIMIT consumedPoints: " + error.consumedPoints + " remainingPoints: " + error.remainingPoints + " msBeforeNext: " + error.msBeforeNext);
                     setTimeout(() => { this.Process(cli); }, 250);
                 }
                 return;
             }
 
             if (!NoderedUtil.IsNullEmpty(this.replyto)) {
+                const end = WebSocketServer.websocket_messages.startTimer();
                 const qmsg: QueuedMessage = cli.messageQueue[this.replyto];
                 if (!NoderedUtil.IsNullUndefinded(qmsg)) {
                     try {
@@ -86,8 +89,10 @@ export class Message {
                     if (!NoderedUtil.IsNullUndefinded(qmsg.cb)) { qmsg.cb(this); }
                     delete cli.messageQueue[this.id];
                 }
+                end({ command: command });
                 return;
             }
+            const end = WebSocketServer.websocket_messages.startTimer();
             switch (command) {
                 case "listcollections":
                     this.ListCollections(cli);
@@ -218,6 +223,7 @@ export class Message {
                     this.UnknownCommand(cli);
                     break;
             }
+            end({ command: command });
         } catch (error) {
             cli._logger.error(error.message ? error.message : error);
         }
diff --git a/OpenFlow/src/WebSocketServer.ts b/OpenFlow/src/WebSocketServer.ts
@@ -37,6 +37,17 @@ export class WebSocketServer {
         help: 'Total number of queues messages',
         labelNames: ["queuename"]
     })
+    public static websocket_rate_limit = new client.Counter({
+        name: 'openflow_websocket_rate_limit_count',
+        help: 'Total number of rate limited messages',
+        labelNames: ["command"]
+    })
+    public static websocket_messages = new client.Histogram({
+        name: 'openflow_websocket_messages_duration_seconds',
+        help: 'Duration for handling websocket requests in microseconds',
+        labelNames: ['command'],
+        buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10]
+    })
 
     static configure(logger: winston.Logger, server: http.Server, register: client.Registry): void {
         this._clients = [];
@@ -53,6 +64,9 @@ export class WebSocketServer {
         if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebSocketServer.websocket_incomming_stats);
         if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebSocketServer.websocket_queue_count);
         if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebSocketServer.websocket_queue_message_count);
+        if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebSocketServer.websocket_rate_limit);
+        if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebSocketServer.websocket_messages);
+
         setInterval(this.pingClients, 10000);
     }
     private static async pingClients(): Promise<void> {
diff --git a/OpenFlowNodeRED/package.json b/OpenFlowNodeRED/package.json
@@ -1,6 +1,6 @@
 {
   "name": "openflow-nodered",
-  "version": "1.1.91",
+  "version": "1.1.92",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.1.91
+1.1.92
diff --git a/docker-compose-toolbox.yml b/docker-compose-toolbox.yml
@@ -45,7 +45,7 @@ services:
       - "traefik.http.routers.web.rule=Host(`toolbox.openrpa.dk`)"
       - "traefik.http.routers.web.entrypoints=web"
       - "traefik.frontend.passHostHeader=true"
-    image: "cloudhack/openflow:1.1.91"
+    image: "cloudhack/openflow:1.1.92"
     container_name: "web"
     environment:
       - update_acl_based_on_groups=true
@@ -82,7 +82,7 @@ services:
       - "traefik.http.routers.nodered.rule=Host(`nodered1.toolbox.openrpa.dk`)"
       - "traefik.http.routers.nodered.entrypoints=web"
       - "traefik.http.services.nodered.loadbalancer.server.port=1880"
-    image: "cloudhack/openflownodered:1.1.91"
+    image: "cloudhack/openflownodered:1.1.92"
     container_name: "nodered"
     environment:
       #      - nodered_id=1
diff --git a/docker-compose-traefik-letsencrypt.yml b/docker-compose-traefik-letsencrypt.yml
@@ -69,7 +69,7 @@ services:
       - "traefik.http.routers.web.entrypoints=web,websecure"
       - "traefik.frontend.passHostHeader=true"
       - "traefik.http.routers.web.tls.certresolver=myresolver"
-    image: "cloudhack/openflow:1.1.91"
+    image: "cloudhack/openflow:1.1.92"
     container_name: "web"
     environment:
       - update_acl_based_on_groups=true
@@ -107,7 +107,7 @@ services:
       - "traefik.http.routers.nodered.entrypoints=web,websecure"
       - "traefik.http.services.nodered.loadbalancer.server.port=1880"
       - "traefik.http.routers.nodered.tls.certresolver=myresolver"
-    image: "cloudhack/openflownodered:1.1.91"
+    image: "cloudhack/openflownodered:1.1.92"
     container_name: "nodered"
     environment:
       #      - nodered_id=1
diff --git a/docker-compose-traefik.yml b/docker-compose-traefik.yml
@@ -45,7 +45,7 @@ services:
       - "traefik.http.routers.web.rule=Host(`localhost.openrpa.dk`)"
       - "traefik.http.routers.web.entrypoints=web"
       - "traefik.frontend.passHostHeader=true"
-    image: "cloudhack/openflow:1.1.91"
+    image: "cloudhack/openflow:1.1.92"
     container_name: "web"
     environment:
       - update_acl_based_on_groups=true
@@ -82,7 +82,7 @@ services:
       - "traefik.http.routers.nodered.rule=Host(`nodered1.localhost.openrpa.dk`)"
       - "traefik.http.routers.nodered.entrypoints=web"
       - "traefik.http.services.nodered.loadbalancer.server.port=1880"
-    image: "cloudhack/openflownodered:1.1.91"
+    image: "cloudhack/openflownodered:1.1.92"
     container_name: "nodered"
     environment:
       #      - nodered_id=1
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -18,7 +18,7 @@ services:
       - "5672:5672"
       - "15672:15672"
   web:
-    image: "cloudhack/openflow:1.1.91"
+    image: "cloudhack/openflow:1.1.92"
     environment:
       - update_acl_based_on_groups=true
       - multi_tenant=false
@@ -52,7 +52,7 @@ services:
       - "80:80"
       - "5858:5858"
   nodered:
-    image: "cloudhack/openflownodered:1.1.91"
+    image: "cloudhack/openflownodered:1.1.92"
     environment:
       #      - nodered_id=1
       - nodered_sa=nodered1
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "openiap",
-  "version": "1.1.91",
+  "version": "1.1.92",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {

Original file line number	Diff line number	Diff line change
`@@ -69,13 +69,16 @@ export class Message {`
`69`	`69`	`// console.log("SOCKET_NO_RATE_LIMIT consumedPoints: " + res.consumedPoints + " remainingPoints: " + res.remainingPoints);`
`70`	`70`	`} catch (error) {`
`71`	`71`	`if (error.consumedPoints) {`
	`72`	`+ WebSocketServer.websocket_rate_limit.inc();`
	`73`	`+ WebSocketServer.websocket_rate_limit.labels(command).inc();`
`72`	`74`	`if ((error.consumedPoints % 100) == 0) cli._logger.debug("[" + username + "/" + cli.clientagent + "/" + cli.id + "] SOCKET_RATE_LIMIT consumedPoints: " + error.consumedPoints + " remainingPoints: " + error.remainingPoints + " msBeforeNext: " + error.msBeforeNext);`
`73`	`75`	`setTimeout(() => { this.Process(cli); }, 250);`
`74`	`76`	`}`
`75`	`77`	`return;`
`76`	`78`	`}`
`77`	`79`
`78`	`80`	`if (!NoderedUtil.IsNullEmpty(this.replyto)) {`
	`81`	`+ const end = WebSocketServer.websocket_messages.startTimer();`
`79`	`82`	`const qmsg: QueuedMessage = cli.messageQueue[this.replyto];`
`80`	`83`	`if (!NoderedUtil.IsNullUndefinded(qmsg)) {`
`81`	`84`	`try {`
`@@ -86,8 +89,10 @@ export class Message {`
`86`	`89`	`if (!NoderedUtil.IsNullUndefinded(qmsg.cb)) { qmsg.cb(this); }`
`87`	`90`	`delete cli.messageQueue[this.id];`
`88`	`91`	`}`
	`92`	`+ end({ command: command });`
`89`	`93`	`return;`
`90`	`94`	`}`
	`95`	`+ const end = WebSocketServer.websocket_messages.startTimer();`
`91`	`96`	`switch (command) {`
`92`	`97`	`case "listcollections":`
`93`	`98`	`this.ListCollections(cli);`
`@@ -218,6 +223,7 @@ export class Message {`
`218`	`223`	`this.UnknownCommand(cli);`
`219`	`224`	`break;`
`220`	`225`	`}`
	`226`	`+ end({ command: command });`
`221`	`227`	`} catch (error) {`
`222`	`228`	`cli._logger.error(error.message ? error.message : error);`
`223`	`229`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "openflow-nodered",`
`3`		`- "version": "1.1.91",`
	`3`	`+ "version": "1.1.92",`
`4`	`4`	`"description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "openiap",`
`3`		`- "version": "1.1.91",`
	`3`	`+ "version": "1.1.92",`
`4`	`4`	`"description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`