Add nodered node timing metrics

skadefro · skadefro · commit 2cd54fcdc9bd · 2020-11-15T21:30:58.000+01:00
diff --git a/OpenFlowNodeRED/package.json b/OpenFlowNodeRED/package.json
@@ -1,6 +1,6 @@
 {
   "name": "openflow-nodered",
-  "version": "1.1.81",
+  "version": "1.1.82",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {
diff --git a/OpenFlowNodeRED/src/Config.ts b/OpenFlowNodeRED/src/Config.ts
@@ -62,6 +62,7 @@ export class Config {
         Config.tls_passphrase = Config.getEnv("tls_passphrase", "");
 
         Config.amqp_message_ttl = parseInt(Config.getEnv("amqp_message_ttl", "20000"));
+        Config.prometheus_max_node_time_seconds = parseInt(Config.getEnv("prometheus_max_node_time_seconds", "300"));
     }
     public static version: string = Config.getversion();
     public static logpath: string = Config.getEnv("logpath", __dirname);
@@ -119,6 +120,8 @@ export class Config {
     // public static amqp_dlrk_prefix: string = Config.getEnv("amqp_dlrk_prefix", "dlx.");
     // public static amqp_dlq_prefix: string = Config.getEnv("amqp_dlq_prefix", "dlq.");
     public static amqp_message_ttl: number = parseInt(Config.getEnv("amqp_message_ttl", "20000"));
+    public static prometheus_max_node_time_seconds: number = parseInt(Config.getEnv("prometheus_max_node_time_seconds", "300"));
+
 
 
     public static baseurl(): string {
diff --git a/OpenFlowNodeRED/src/WebServer.ts b/OpenFlowNodeRED/src/WebServer.ts
@@ -27,6 +27,19 @@ export class WebServer {
     private static _logger: winston.Logger;
     private static app: express.Express = null;
 
+    public static openflow_nodered_node_count = new client.Counter({
+        name: 'openflow_nodered_node_count',
+        help: 'Total number of node calls',
+        labelNames: ["nodetype"]
+    })
+    public static openflow_nodered_node_duration = new client.Histogram({
+        name: 'openflow_nodered_node_duration',
+        help: 'Duration of each node call',
+        labelNames: ["nodetype"],
+        buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10]
+    })
+
+    public static log_messages: any = {};
     private static settings: nodered_settings = null;
     static async configure(logger: winston.Logger, socket: WebSocketClient): Promise<http.Server> {
         this._logger = logger;
@@ -49,6 +62,10 @@ export class WebServer {
                 register.setDefaultLabels(defaultLabels);
                 client.collectDefaultMetrics({ register })
 
+                if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebServer.openflow_nodered_node_count);
+                if (!NoderedUtil.IsNullUndefinded(register)) register.registerMetric(WebServer.openflow_nodered_node_duration);
+
+
                 const metricsMiddleware = promBundle({ includeMethod: true, includePath: true, promRegistry: register, autoregister: true });
                 this.app.use(metricsMiddleware);
                 // this.app.use(morgan('combined', { stream: (winston.stream as any).write }));
@@ -121,6 +138,50 @@ export class WebServer {
                 else {
                     this.settings.uiPort = Config.port;
                 }
+                this.settings.logging.customLogger = {
+                    level: 'debug',
+                    metrics: true,
+                    handler: function (settings) {
+                        // 
+                        // Return the logging function
+                        // return function (msg) {
+                        //     console.log(msg.timestamp, msg.event);
+                        // }
+                        return function (msg) {
+                            if (!NoderedUtil.IsNullEmpty(msg.msgid) && msg.event.startsWith("node.")) {
+                                msg.event = msg.event.substring(5);
+                                if (msg.event.endsWith(".receive")) {
+                                    msg.event = msg.event.substring(0, msg.event.length - 8);
+                                    msg.end = WebServer.openflow_nodered_node_duration.startTimer()
+                                    WebServer.openflow_nodered_node_count.labels(msg.event).inc();
+                                    WebServer.log_messages[msg.msgid] = msg;
+                                }
+                                if (msg.event.endsWith(".send")) {
+                                    msg.event = msg.event.substring(0, msg.event.length - 5);
+                                    const startmessage = WebServer.log_messages[msg.msgid];
+                                    if (!NoderedUtil.IsNullUndefinded(startmessage)) {
+                                        startmessage.end({ nodetype: startmessage.event });
+                                        delete WebServer.log_messages[msg.msgid];
+                                    }
+                                }
+                                const keys = Object.keys(WebServer.log_messages);
+                                keys.forEach(key => {
+                                    const meg = WebServer.log_messages[key];
+                                    var from = new Date(msg.timestamp);
+                                    const now = new Date();
+                                    const seconds = (now.getTime() - from.getTime()) / 1000;
+                                    if (seconds > Config.prometheus_max_node_time_seconds) {
+                                        console.log("Deleting message " + key + " that is more " + seconds + " seconds old");
+                                        delete WebServer.log_messages[key];
+                                    }
+                                });
+                                // } else {
+                                //     console.log(msg);
+                            }
+                            // console.log(msg.timestamp, msg.event);
+                        }
+                    }
+                }
 
 
 
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.1.81
+1.1.82
diff --git a/docker-compose-toolbox.yml b/docker-compose-toolbox.yml
@@ -45,7 +45,7 @@ services:
       - "traefik.http.routers.web.rule=Host(`toolbox.openrpa.dk`)"
       - "traefik.http.routers.web.entrypoints=web"
       - "traefik.frontend.passHostHeader=true"
-    image: "cloudhack/openflow:1.1.81"
+    image: "cloudhack/openflow:1.1.82"
     container_name: "web"
     environment:
       - update_acl_based_on_groups=true
@@ -82,7 +82,7 @@ services:
       - "traefik.http.routers.nodered.rule=Host(`nodered1.toolbox.openrpa.dk`)"
       - "traefik.http.routers.nodered.entrypoints=web"
       - "traefik.http.services.nodered.loadbalancer.server.port=1880"
-    image: "cloudhack/openflownodered:1.1.81"
+    image: "cloudhack/openflownodered:1.1.82"
     container_name: "nodered"
     environment:
       #      - nodered_id=1
diff --git a/docker-compose-traefik-letsencrypt.yml b/docker-compose-traefik-letsencrypt.yml
@@ -69,7 +69,7 @@ services:
       - "traefik.http.routers.web.entrypoints=web,websecure"
       - "traefik.frontend.passHostHeader=true"
       - "traefik.http.routers.web.tls.certresolver=myresolver"
-    image: "cloudhack/openflow:1.1.81"
+    image: "cloudhack/openflow:1.1.82"
     container_name: "web"
     environment:
       - update_acl_based_on_groups=true
@@ -107,7 +107,7 @@ services:
       - "traefik.http.routers.nodered.entrypoints=web,websecure"
       - "traefik.http.services.nodered.loadbalancer.server.port=1880"
       - "traefik.http.routers.nodered.tls.certresolver=myresolver"
-    image: "cloudhack/openflownodered:1.1.81"
+    image: "cloudhack/openflownodered:1.1.82"
     container_name: "nodered"
     environment:
       #      - nodered_id=1
diff --git a/docker-compose-traefik.yml b/docker-compose-traefik.yml
@@ -45,7 +45,7 @@ services:
       - "traefik.http.routers.web.rule=Host(`localhost.openrpa.dk`)"
       - "traefik.http.routers.web.entrypoints=web"
       - "traefik.frontend.passHostHeader=true"
-    image: "cloudhack/openflow:1.1.81"
+    image: "cloudhack/openflow:1.1.82"
     container_name: "web"
     environment:
       - update_acl_based_on_groups=true
@@ -82,7 +82,7 @@ services:
       - "traefik.http.routers.nodered.rule=Host(`nodered1.localhost.openrpa.dk`)"
       - "traefik.http.routers.nodered.entrypoints=web"
       - "traefik.http.services.nodered.loadbalancer.server.port=1880"
-    image: "cloudhack/openflownodered:1.1.81"
+    image: "cloudhack/openflownodered:1.1.82"
     container_name: "nodered"
     environment:
       #      - nodered_id=1
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -18,7 +18,7 @@ services:
       - "5672:5672"
       - "15672:15672"
   web:
-    image: "cloudhack/openflow:1.1.81"
+    image: "cloudhack/openflow:1.1.82"
     environment:
       - update_acl_based_on_groups=true
       - multi_tenant=false
@@ -52,7 +52,7 @@ services:
       - "80:80"
       - "5858:5858"
   nodered:
-    image: "cloudhack/openflownodered:1.1.81"
+    image: "cloudhack/openflownodered:1.1.82"
     environment:
       #      - nodered_id=1
       - nodered_sa=nodered1
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "openiap",
-  "version": "1.1.81",
+  "version": "1.1.82",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "openflow-nodered",`
`3`		`- "version": "1.1.81",`
	`3`	`+ "version": "1.1.82",`
`4`	`4`	`"description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "openiap",`
`3`		`- "version": "1.1.81",`
	`3`	`+ "version": "1.1.82",`
`4`	`4`	`"description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`