Skip to content

Commit 8a17525

Browse files
skadefroskadefro
committed
optimize search speed with _acl
1 parent 8dd7d32 commit 8a17525

3 files changed

Lines changed: 70 additions & 25 deletions

File tree

‎OpenFlow/src/DatabaseConnection.ts‎

Lines changed: 59 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2243,22 +2243,22 @@ export class DatabaseConnection extends events.EventEmitter {
22432243
q[field] = {
22442244
$elemMatch: {
22452245
rights: { $bitsAllSet: bits },
2246-
deny: false,
22472246
$or: isme
22482247
}
22492248
};
2249+
// deny: false,
22502250
finalor.push(q);
2251-
if (field === "_acl") {
2252-
const q2 = {};
2253-
q2["value._acl"] = {
2254-
$elemMatch: {
2255-
rights: { $bitsAllSet: bits },
2256-
deny: false,
2257-
$or: isme
2258-
}
2259-
};
2260-
finalor.push(q2);
2261-
}
2251+
// if (field === "_acl") {
2252+
// const q2 = {};
2253+
// q2["value._acl"] = {
2254+
// $elemMatch: {
2255+
// rights: { $bitsAllSet: bits },
2256+
// deny: false,
2257+
// $or: isme
2258+
// }
2259+
// };
2260+
// finalor.push(q2);
2261+
// }
22622262
return { $or: finalor.concat() };
22632263
}
22642264
private async getbasequeryuserid(userid: string, field: string, bits: number[], parent: Span): Promise<Object> {
@@ -2738,25 +2738,39 @@ export class DatabaseConnection extends events.EventEmitter {
27382738
let collections = await DatabaseConnection.toArray(this.db.listCollections());
27392739
collections = collections.filter(x => x.name.indexOf("system.") === -1);
27402740

2741+
DatabaseConnection.timeseries_collections = [];
2742+
for (let i = 0; i < collections.length; i++) {
2743+
var collection = collections[i];
2744+
if (collection.type == "timeseries") {
2745+
DatabaseConnection.timeseries_collections = DatabaseConnection.timeseries_collections.filter(x => x != collection.name);
2746+
DatabaseConnection.timeseries_collections.push(collection.name);
2747+
}
2748+
if (collection.type != "collection" && collection.type != "timeseries") continue;
2749+
}
2750+
27412751
for (let i = 0; i < collections.length; i++) {
27422752
try {
27432753
const collection = collections[i];
27442754
if (collection.type != "collection") continue;
2755+
if (collection.name == "uploads.files" || collection.name == "uploads.chunks" || collection.name == "fs.chunks") continue;
27452756
const indexes = await this.db.collection(collection.name).indexes();
27462757
const indexnames = indexes.map(x => x.name);
27472758
if (collection.name.endsWith("_hist")) {
27482759
if (indexnames.indexOf("id_1__version_-1") === -1) {
27492760
await this.createIndex(collection.name, "id_1__version_-1", { "id": 1, "_version": -1 }, null, span)
27502761
}
2762+
if (indexnames.indexOf("_acl") === -1) {
2763+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2764+
}
27512765
} else {
27522766
switch (collection.name) {
27532767
case "fs.files":
27542768
if (indexnames.indexOf("metadata.workflow_1") === -1) {
27552769
await this.createIndex(collection.name, "metadata.workflow_1", { "metadata.workflow": 1 }, null, span)
27562770
}
2757-
// if (indexnames.indexOf("filename_text") === -1 && Config.create_text_index_for_names) {
2758-
// await this.createIndex(collection.name, "filename_text", { "filename": "text" }, {default_language: "none", language_override: "none"}, span)
2759-
// }
2771+
if (indexnames.indexOf("metadata._acl") === -1) {
2772+
await this.createIndex(collection.name, "metadata._acl", { "metadata._acl._id": 1, "metadata._acl.rights": 1, "metadata._acl.deny": 1 }, null, span)
2773+
}
27602774
break;
27612775
case "fs.chunks":
27622776
break;
@@ -2770,6 +2784,9 @@ export class DatabaseConnection extends events.EventEmitter {
27702784
if (indexnames.indexOf("queue_1") === -1) {
27712785
await this.createIndex(collection.name, "queue_1", { "queue": 1 }, null, span)
27722786
}
2787+
if (indexnames.indexOf("_acl") === -1) {
2788+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2789+
}
27732790
break;
27742791
case "openrpa_instances":
27752792
if (indexnames.indexOf("_created_1") === -1) {
@@ -2779,17 +2796,17 @@ export class DatabaseConnection extends events.EventEmitter {
27792796
await this.createIndex(collection.name, "_modified_1", { "_modified": 1 }, null, span)
27802797
}
27812798
if (indexnames.indexOf("InstanceId_1_WorkflowId_1") === -1) {
2782-
await this.createIndex(collection.name, "InstanceId_1_WorkflowId_1", { "WorkflowId": 1, "InstanceId": 1 }, null, span)
2799+
await this.createIndex(collection.name, "InstanceId_1_WorkflowId_1", { "WorkflowId": 1, "InstanceId": 1, "_acl.deny": 1 }, null, span)
27832800
}
27842801
if (indexnames.indexOf("state_1") === -1) {
27852802
await this.createIndex(collection.name, "state_1", { "state": 1 }, null, span)
27862803
}
27872804
if (indexnames.indexOf("fqdn_1") === -1) {
27882805
await this.createIndex(collection.name, "fqdn_1", { "fqdn": 1 }, null, span)
27892806
}
2790-
// if (indexnames.indexOf("name_text") === -1 && Config.create_text_index_for_names) {
2791-
// await this.createIndex(collection.name, "name_text", { "name": "text" }, {default_language: "none", language_override: "none"}, span)
2792-
// }
2807+
if (indexnames.indexOf("_acl") === -1) {
2808+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2809+
}
27932810
break;
27942811
case "audit":
27952812
if (indexnames.indexOf("_type_1") === -1) {
@@ -2810,9 +2827,9 @@ export class DatabaseConnection extends events.EventEmitter {
28102827
if (indexnames.indexOf("userid_1") === -1) {
28112828
await this.createIndex(collection.name, "userid_1", { "userid": 1 }, null, span)
28122829
}
2813-
// if (indexnames.indexOf("name_text") === -1 && Config.create_text_index_for_names) {
2814-
// await this.createIndex(collection.name, "name_text", { "name": "text" }, {default_language: "none", language_override: "none"}, span)
2815-
// }
2830+
if (indexnames.indexOf("_acl") === -1) {
2831+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2832+
}
28162833
break;
28172834
case "users":
28182835
if (indexnames.indexOf("workflowid_1") === -1) {
@@ -2843,6 +2860,9 @@ export class DatabaseConnection extends events.EventEmitter {
28432860
if (indexnames.indexOf("members._id_1") === -1) {
28442861
await this.createIndex(collection.name, "members._id_1", { "members._id": 1 }, null, span)
28452862
}
2863+
if (indexnames.indexOf("_acl") === -1) {
2864+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2865+
}
28462866
break;
28472867
case "openrpa":
28482868
if (indexnames.indexOf("_created_1") === -1) {
@@ -2854,6 +2874,9 @@ export class DatabaseConnection extends events.EventEmitter {
28542874
if (indexnames.indexOf("_type_projectid_name_1") === -1) {
28552875
await this.createIndex(collection.name, "_type_projectid_name_1", { _type: 1, "{projectid:-1,name:-1}": 1 }, null, span)
28562876
}
2877+
if (indexnames.indexOf("_acl") === -1) {
2878+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2879+
}
28572880
break;
28582881
case "dbusage":
28592882
if (indexnames.indexOf("_created_1") === -1) {
@@ -2865,6 +2888,9 @@ export class DatabaseConnection extends events.EventEmitter {
28652888
if (indexnames.indexOf("collection_1_timestamp_1") === -1) {
28662889
await this.createIndex(collection.name, "collection_1_timestamp_1", { _type: 1, "{collection:1,timestamp:1}": 1 }, null, span)
28672890
}
2891+
if (indexnames.indexOf("_acl") === -1) {
2892+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2893+
}
28682894
break;
28692895
default:
28702896
if (indexnames.indexOf("_type_1") === -1) {
@@ -2876,6 +2902,15 @@ export class DatabaseConnection extends events.EventEmitter {
28762902
if (indexnames.indexOf("_modified_1") === -1) {
28772903
await this.createIndex(collection.name, "_modified_1", { "_modified": 1 }, null, span)
28782904
}
2905+
if (DatabaseConnection.timeseries_collections.indexOf(collection.name) > -1) {
2906+
if (indexnames.indexOf("metadata._acl") === -1) {
2907+
await this.createIndex(collection.name, "metadata._acl", { "metadata._acl._id": 1, "metadata._acl.rights": 1, "metadata._acl.deny": 1 }, null, span)
2908+
}
2909+
} else {
2910+
if (indexnames.indexOf("_acl") === -1) {
2911+
await this.createIndex(collection.name, "_acl", { "_acl._id": 1, "_acl.rights": 1, "_acl.deny": 1 }, null, span)
2912+
}
2913+
}
28792914
break;
28802915
}
28812916
}
@@ -2888,6 +2923,8 @@ export class DatabaseConnection extends events.EventEmitter {
28882923
collections = await DatabaseConnection.toArray(this.db.listCollections());
28892924
collections = collections.filter(x => x.name.indexOf("system.") === -1);
28902925

2926+
DatabaseConnection.timeseries_collections = [];
2927+
DatabaseConnection.collections_with_text_index = [];
28912928
for (let i = 0; i < collections.length; i++) {
28922929
var collection = collections[i];
28932930
if (collection.type == "timeseries") {

‎OpenFlow/src/public/CommonControllers.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -587,9 +587,9 @@ export class entitiesCtrl<T> {
587587
let orderby = this.orderby;
588588
if (this.lastsearchstring !== this.searchstring) {
589589
this.models = [];
590-
this.orderby = {};
591-
orderby = {};
590+
this.orderby = null;
592591
}
592+
593593
this.lastsearchstring = this.searchstring;
594594
if (this.searchstring !== "" && this.searchstring != null) {
595595
if ((this.searchstring as string).indexOf("{") == 0) {
@@ -639,6 +639,15 @@ export class entitiesCtrl<T> {
639639

640640
}
641641
}
642+
if (this.WebSocketClientService.collections_with_text_index.indexOf(this.collection) == -1) {
643+
if (NoderedUtil.IsNullUndefinded(this.orderby) || Object.keys(this.orderby).length == 0) {
644+
orderby = { _created: -1 };
645+
}
646+
} else {
647+
orderby = null;
648+
}
649+
// orderby = null;
650+
642651
if (this.page == 0) {
643652
this.models = await NoderedUtil.Query(this.collection, query, this.baseprojection, orderby, this.pagesize, 0, null, basequeryas, null, 2);
644653
} else {

‎OpenFlow/src/public/Controllers.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2525,7 +2525,6 @@ export class UserCtrl extends entityCtrl<TokenUser> {
25252525
}
25262526
this.$location.path("/Users");
25272527
} catch (error) {
2528-
debugger;
25292528
this.errormessage = error.message ? error.message : error;
25302529
}
25312530
if (!this.$scope.$$phase) { this.$scope.$apply(); }

0 commit comments

Comments
 (0)