Improve clean acl speed with caching

skadefro · skadefro · commit aa675ec9002d · 2021-04-07T15:57:46.000+02:00
diff --git a/OpenFlow/src/Auth.ts b/OpenFlow/src/Auth.ts
@@ -24,33 +24,62 @@ export class Auth {
     }
 
     public static authorizationCache: HashTable<CachedUser> = {};
-    public static getUser(key: string): User {
-        var res: CachedUser = this.authorizationCache[key];
+    private static cacheTimer: NodeJS.Timeout;
+    public static getUser(key: string, type: string): User {
+        if (NoderedUtil.IsNullUndefinded(this.cacheTimer)) this.cacheTimer = setInterval(this.cleanCache, 60000)
+        var res: CachedUser = this.authorizationCache[key + type];
         if (res === null || res === undefined) return null;
         var begin: number = res.firstsignin.getTime();
         var end: number = new Date().getTime();
         var seconds = Math.round((end - begin) / 1000);
-        if (seconds < Config.api_credential_cache_seconds) {
+        let cache_seconds: number = Config.api_credential_cache_seconds;
+        if (type == "grafana") cache_seconds = Config.grafana_credential_cache_seconds;
+        if (type == "dashboard") cache_seconds = Config.dashboard_credential_cache_seconds;
+        if (type == "cleanacl") cache_seconds = Config.cleanacl_credential_cache_seconds;
+        if (seconds < cache_seconds) {
             // Logger.instanse.info("Return user " + res.user.username + " from cache");
             return res.user;
         }
-        this.RemoveUser(key);
+        this.RemoveUser(key, type);
         return null;
     }
-    public static async RemoveUser(key: string): Promise<void> {
+    private static async cleanCache() {
+        try {
+            if (this.authorizationCache == null) return;
+            const keys: string[] = Object.keys(this.authorizationCache);
+            for (let i = keys.length - 1; i >= 0; i--) {
+                let key: string = keys[i];
+                var res: CachedUser = this.authorizationCache[key];
+                if (res === null || res === undefined) continue;
+                var begin: number = res.firstsignin.getTime();
+                var end: number = new Date().getTime();
+                var seconds = Math.round((end - begin) / 1000);
+                let cache_seconds: number = Config.api_credential_cache_seconds;
+                if (res.type == "grafana") cache_seconds = Config.grafana_credential_cache_seconds;
+                if (res.type == "dashboard") cache_seconds = Config.dashboard_credential_cache_seconds;
+                if (res.type == "cleanacl") cache_seconds = Config.cleanacl_credential_cache_seconds;
+                if (seconds >= cache_seconds) {
+                    this.RemoveUser(key, res.type);
+                }
+            }
+        } catch (error) {
+            Logger.instanse.error(error)
+        }
+    }
+    public static async RemoveUser(key: string, type: string): Promise<void> {
         await semaphore.down();
-        if (!NoderedUtil.IsNullUndefinded(this.authorizationCache[key])) {
+        if (!NoderedUtil.IsNullUndefinded(this.authorizationCache[key + type])) {
             Logger.instanse.info("Delete user with key " + key + " from cache");
-            delete this.authorizationCache[key];
+            delete this.authorizationCache[key + type];
         }
         semaphore.up();
     }
-    public static async AddUser(user: User, key: string): Promise<void> {
+    public static async AddUser(user: User, key: string, type: string): Promise<void> {
         await semaphore.down();
-        if (NoderedUtil.IsNullUndefinded(this.authorizationCache[key])) {
-            Logger.instanse.info("Adding user " + user.username + " to cache with key " + key);
-            var cuser: CachedUser = new CachedUser(user, user._id);
-            this.authorizationCache[key] = cuser;
+        if (NoderedUtil.IsNullUndefinded(this.authorizationCache[key + type])) {
+            Logger.instanse.info("Adding user " + user.name + " to cache with key " + key);
+            var cuser: CachedUser = new CachedUser(user, user._id, type);
+            this.authorizationCache[key + type] = cuser;
         }
         semaphore.up();
     }
@@ -59,7 +88,8 @@ export class CachedUser {
     public firstsignin: Date;
     constructor(
         public user: User,
-        public _id: string
+        public _id: string,
+        public type: string
     ) {
         this.firstsignin = new Date();
     }
diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts
@@ -168,6 +168,9 @@ export class Config {
     public static tls_passphrase: string = Config.getEnv("tls_passphrase", "");
 
     public static api_credential_cache_seconds: number = parseInt(Config.getEnv("api_credential_cache_seconds", "60000"));
+    public static dashboard_credential_cache_seconds: number = parseInt(Config.getEnv("dashboard_credential_cache_seconds", "60000"));
+    public static grafana_credential_cache_seconds: number = parseInt(Config.getEnv("grafana_credential_cache_seconds", "60000"));
+    public static cleanacl_credential_cache_seconds: number = parseInt(Config.getEnv("grafana_credential_cache_seconds", "60000"));
     public static oauth_token_cache_seconds: number = parseInt(Config.getEnv("oauth_token_cache_seconds", "60000"));
     public static oauth_access_token_lifetime: number = parseInt(Config.getEnv("oauth_access_token_lifetime", "604800"));
     public static oauth_refresh_token_lifetime: number = parseInt(Config.getEnv("oauth_refresh_token_lifetime", "604800"));
diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts
@@ -10,6 +10,7 @@ import { OAuthProvider } from "./OAuthProvider";
 import { ValueRecorder, Counter } from "@opentelemetry/api-metrics"
 import { Span } from "@opentelemetry/api";
 import { Logger } from "./Logger";
+import { Auth } from "./Auth";
 // tslint:disable-next-line: typedef
 const safeObjectID = (s: string | number | ObjectID) => ObjectID.isValid(s) ? new ObjectID(s) : null;
 const isoDatePattern = new RegExp(/\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)/);
@@ -162,6 +163,21 @@ export class DatabaseConnection {
             Logger.otel.endSpan(span);
         }
     }
+    WellknownIdsArray: string[] = [
+        WellknownIds.root,
+        WellknownIds.admins,
+        WellknownIds.users,
+        WellknownIds.robots,
+        WellknownIds.nodered_users,
+        WellknownIds.nodered_admins,
+        WellknownIds.nodered_api_users,
+        WellknownIds.filestore_users,
+        WellknownIds.filestore_admins,
+        WellknownIds.robot_users,
+        WellknownIds.robot_admins,
+        WellknownIds.personal_nodered_users,
+        WellknownIds.robot_agent_users
+    ]
 
     async CleanACL<T extends Base>(item: T, user: TokenUser, parent: Span): Promise<T> {
         const span: Span = Logger.otel.startSubSpan("db.CleanACL", parent);
@@ -173,16 +189,26 @@ export class DatabaseConnection {
                         const b = new Binary(Buffer.from(ace.rights, "base64"), 0);
                         (ace.rights as any) = b;
                     }
-                    const ot_end = Logger.otel.startTimer();
-                    const mongodbspan: Span = Logger.otel.startSubSpan("mongodb.find", span);
-                    mongodbspan.setAttribute("collection", "users");
-                    const arr = await this.db.collection("users").find({ _id: ace._id }).project({ name: 1 }).limit(1).toArray();
-                    mongodbspan.setAttribute("results", arr.length);
-                    Logger.otel.endSpan(mongodbspan);
-                    Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_query, { collection: "users" });
-                    if (arr.length == 0) {
-                        item._acl.splice(i, 1);
-                    } else { ace.name = arr[0].name; }
+                    if (this.WellknownIdsArray.indexOf(ace._id) == -1) {
+                        let user = await Auth.getUser(ace._id, "cleanacl");
+                        if (user == null || user != null) {
+                            const ot_end = Logger.otel.startTimer();
+                            const mongodbspan: Span = Logger.otel.startSubSpan("mongodb.find", span);
+                            mongodbspan.setAttribute("collection", "users");
+                            mongodbspan.setAttribute("query", JSON.stringify({ _id: ace._id }));
+                            const arr = await this.db.collection("users").find({ _id: ace._id }).project({ name: 1 }).limit(1).toArray();
+                            mongodbspan.setAttribute("results", arr.length);
+                            Logger.otel.endSpan(mongodbspan);
+                            Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_query, { collection: "users" });
+                            if (arr.length > 0) {
+                                user = arr[0];
+                                await Auth.AddUser(user, ace._id, "cleanacl");
+                            }
+                        }
+                        if (user == null) {
+                            item._acl.splice(i, 1);
+                        } else { ace.name = user.name; }
+                    }
                 }
             }
             if (Config.force_add_admins) {
diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts
@@ -201,7 +201,7 @@ export class LoginProvider {
                 if (!NoderedUtil.IsNullEmpty(authorization) && authorization.indexOf(" ") > 1 &&
                     (authorization.toLocaleLowerCase().startsWith("bearer") || authorization.toLocaleLowerCase().startsWith("jwt"))) {
                     const token = authorization.split(" ")[1];
-                    let user: User = Auth.getUser(token);
+                    let user: User = Auth.getUser(token, "dashboard");
                     let tuser: TokenUser;
                     if (user == null) {
                         try {
@@ -220,7 +220,7 @@ export class LoginProvider {
                     if (user != null) {
                         const allowed = user.roles.filter(x => x.name == "dashboardusers" || x.name == "admins");
                         if (allowed.length > 0) {
-                            await Auth.AddUser(user, token);
+                            await Auth.AddUser(user, token, "dashboard");
                             // Logger.instanse.info("dashboardauth: Authorized " + user.username + " for " + req.url);
                             return res.send({
                                 status: "success",
@@ -242,13 +242,13 @@ export class LoginProvider {
                 const [login, password] = Buffer.from(b64auth, "base64").toString().split(':')
                 if (login && password) {
                     span.setAttribute("username", login);
-                    let user: User = Auth.getUser(login + ":" + password);
+                    let user: User = Auth.getUser(login + ":" + password, "dashboard");
                     if (user == null) user = await Auth.ValidateByPassword(login, password, span);
                     if (user != null) {
                         const allowed = user.roles.filter(x => x.name == "dashboardusers" || x.name == "admins");
                         if (allowed.length > 0) {
                             // Logger.instanse.info("dashboardauth: Authorized " + user.username + " for " + req.url);
-                            Auth.AddUser(user, login + ":" + password);
+                            Auth.AddUser(user, login + ":" + password, "dashboard");
                             return res.send({
                                 status: "success",
                                 display_status: "Success",
diff --git a/OpenFlowNodeRED/package.json b/OpenFlowNodeRED/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openiap/nodered",
-  "version": "1.2.65",
+  "version": "1.2.66",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.2.65
+1.2.66
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openiap/openflow",
-  "version": "1.2.65",
+  "version": "1.2.66",
   "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",
   "main": "index.js",
   "scripts": {

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@openiap/nodered",`
`3`		`- "version": "1.2.65",`
	`3`	`+ "version": "1.2.66",`
`4`	`4`	`"description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@openiap/openflow",`
`3`		`- "version": "1.2.65",`
	`3`	`+ "version": "1.2.66",`
`4`	`4`	`"description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"scripts": {`