Skip to content

Commit 2f78a06

Browse files
skadefroskadefro
committed
Add support for auto create index doing startup
1 parent 329722e commit 2f78a06

5 files changed

Lines changed: 125 additions & 39 deletions

File tree

‎OpenFlow/src/Config.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export class Config {
3232
Config.stripe_api_secret = Config.getEnv("stripe_api_secret", "");
3333

3434
Config.supports_watch = Config.parseBoolean(Config.getEnv("supports_watch", "false"));
35+
Config.ensure_indexes = Config.parseBoolean(Config.getEnv("ensure_indexes", "true"));
3536

3637
Config.auto_create_users = Config.parseBoolean(Config.getEnv("auto_create_users", "false"));
3738
Config.auto_create_domains = Config.parseArray(Config.getEnv("auto_create_domains", ""));
@@ -112,6 +113,7 @@ export class Config {
112113
Config.prometheus_measure_nodeid = Config.parseBoolean(Config.getEnv("prometheus_measure_nodeid", "false"));
113114
Config.prometheus_measure_queued_messages = Config.parseBoolean(Config.getEnv("prometheus_measure_queued_messages", "false"));
114115
Config.prometheus_measure__mongodb_watch = Config.parseBoolean(Config.getEnv("prometheus_measure__mongodb_watch", "false"));
116+
Config.prometheus_measure_onlineuser = Config.parseBoolean(Config.getEnv("prometheus_measure_onlineuser", "false"));
115117
Config.validate_user_form = Config.getEnv("validate_user_form", "");
116118
}
117119
public static db: DatabaseConnection = null;
@@ -132,6 +134,7 @@ export class Config {
132134
public static stripe_api_secret: string = Config.getEnv("stripe_api_secret", "");
133135

134136
public static supports_watch: boolean = Config.parseBoolean(Config.getEnv("supports_watch", "false"));
137+
public static ensure_indexes: boolean = Config.parseBoolean(Config.getEnv("ensure_indexes", "true"));
135138

136139
public static auto_create_users: boolean = Config.parseBoolean(Config.getEnv("auto_create_users", "false"));
137140
public static auto_create_domains: string[] = Config.parseArray(Config.getEnv("auto_create_domains", ""));
@@ -215,6 +218,7 @@ export class Config {
215218
public static prometheus_measure_nodeid: boolean = Config.parseBoolean(Config.getEnv("prometheus_measure_nodeid", "false"));
216219
public static prometheus_measure_queued_messages: boolean = Config.parseBoolean(Config.getEnv("prometheus_measure_queued_messages", "false"));
217220
public static prometheus_measure__mongodb_watch: boolean = Config.parseBoolean(Config.getEnv("prometheus_measure__mongodb_watch", "false"));
221+
public static prometheus_measure_onlineuser: boolean = Config.parseBoolean(Config.getEnv("prometheus_measure_onlineuser", "false"));
218222
public static validate_user_form: string = Config.getEnv("validate_user_form", "");
219223

220224
public static baseurl(): string {

‎OpenFlow/src/DatabaseConnection.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,30 @@ export class DatabaseConnection {
822822
if (Config.log_inserts) this._logger.debug("[" + user.username + "][" + collectionname + "] inserted " + item.name);
823823
return item;
824824
}
825+
synRawUpdateOne(collection: string, query: any, updatedoc: any, measure: boolean, cb: any) {
826+
let end: any = null;
827+
if (measure) {
828+
DatabaseConnection.mongodb_update_count.labels(collection).inc();
829+
end = DatabaseConnection.mongodb_update.startTimer();
830+
}
831+
Config.db.db.collection(collection).updateOne(query, updatedoc).catch(err => {
832+
if (measure) end({ collection: collection });
833+
console.error(err);
834+
if (cb) cb(err, null);
835+
}).then((result) => {
836+
if (measure) end({ collection: collection });
837+
if (cb) cb(null, result);
838+
});
839+
}
840+
async rawUpdateOne(collection: string, query: any, updatedoc: any, measure: boolean) {
841+
let end: any = null;
842+
if (measure) {
843+
DatabaseConnection.mongodb_update_count.labels("users").inc();
844+
end = DatabaseConnection.mongodb_update.startTimer();
845+
}
846+
await Config.db.db.collection(collection).updateOne(query, updatedoc);
847+
if (measure) end({ collection: "users" });
848+
}
825849
/**
826850
* Update entity in database
827851
* @param {T} item Item to update
@@ -1805,4 +1829,76 @@ export class DatabaseConnection {
18051829
}
18061830

18071831

1832+
async createIndex(collectionname: string, name: string, keypath: any) {
1833+
return new Promise((resolve, reject) => {
1834+
this._logger.info("Adding index " + name + " to " + collectionname);
1835+
this.db.collection(collectionname).createIndex(keypath, (err, name) => {
1836+
if (err) {
1837+
reject(err);
1838+
return;
1839+
}
1840+
resolve(name);
1841+
})
1842+
});
1843+
}
1844+
async ensureindexes() {
1845+
if (!Config.ensure_indexes) return;
1846+
const collections = await DatabaseConnection.toArray(this.db.listCollections());
1847+
1848+
for (var i = 0; i < collections.length; i++) {
1849+
try {
1850+
const collection = collections[i];
1851+
if (collection.type != "collection") continue;
1852+
const indexes = await this.db.collection(collection.name).indexes();
1853+
const indexnames = indexes.map(x => x.name);
1854+
if (collection.name.endsWith("_hist")) {
1855+
if (indexnames.indexOf("id_1__version_-1") == -1) {
1856+
await this.createIndex(collection.name, "id_1__version_-1", { "id": 1, "_version": -1 })
1857+
}
1858+
} else {
1859+
switch (collection.name) {
1860+
case "fs.files":
1861+
if (indexnames.indexOf("metadata.workflow_1") == -1) {
1862+
await this.createIndex(collection.name, "metadata.workflow_1", { "metadata.workflow": 1 })
1863+
}
1864+
break;
1865+
case "fs.chunks":
1866+
break;
1867+
case "workflow":
1868+
if (indexnames.indexOf("queue_1") == -1) {
1869+
await this.createIndex(collection.name, "queue_1", { "queue": 1 })
1870+
}
1871+
break;
1872+
case "users":
1873+
if (indexnames.indexOf("workflowid_1") == -1) {
1874+
await this.createIndex(collection.name, "workflowid_1", { "workflowid": 1 })
1875+
}
1876+
if (indexnames.indexOf("_rpaheartbeat_1") == -1) {
1877+
await this.createIndex(collection.name, "_rpaheartbeat_1", { "_rpaheartbeat": 1 })
1878+
}
1879+
if (indexnames.indexOf("name_1") == -1) {
1880+
await this.createIndex(collection.name, "name_1", { "name": 1 })
1881+
}
1882+
if (indexnames.indexOf("_type_1") == -1) {
1883+
await this.createIndex(collection.name, "_type_1", { "_type": 1 })
1884+
}
1885+
if (indexnames.indexOf("_created_1") == -1) {
1886+
await this.createIndex(collection.name, "_created_1", { "_created": 1 })
1887+
}
1888+
break;
1889+
default:
1890+
if (indexnames.indexOf("_type_1") == -1) {
1891+
await this.createIndex(collection.name, "_type_1", { "_type": 1 })
1892+
}
1893+
if (indexnames.indexOf("_created_1") == -1) {
1894+
await this.createIndex(collection.name, "_created_1", { "_created": 1 })
1895+
}
1896+
break;
1897+
}
1898+
}
1899+
} catch (error) {
1900+
this._logger.error(error);
1901+
}
1902+
}
1903+
}
18081904
}

‎OpenFlow/src/WebSocketServer.ts‎

Lines changed: 18 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -191,55 +191,35 @@ export class WebSocketServer {
191191
}
192192
// Lets assume only robots register queues ( not true )
193193
if (cli.clientagent == "openrpa") {
194-
DatabaseConnection.mongodb_update_count.labels("users").inc();
195-
const end = DatabaseConnection.mongodb_update.startTimer();
196-
Config.db.db.collection("users").updateOne({ _id: cli.user._id },
197-
{ $set: { _rpaheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } }).catch((err) => {
198-
console.error(err);
199-
});
200-
end({ collection: "users" });
194+
Config.db.synRawUpdateOne("users", { _id: cli.user._id },
195+
{ $set: { _rpaheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } },
196+
Config.prometheus_measure_onlineuser, null);
201197
}
202198
if (cli.clientagent == "nodered") {
203-
DatabaseConnection.mongodb_update_count.labels("users").inc();
204-
const end = DatabaseConnection.mongodb_update.startTimer();
205-
Config.db.db.collection("users").updateOne({ _id: cli.user._id },
206-
{ $set: { _noderedheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } }).catch((err) => {
207-
console.error(err);
208-
});
209-
end({ collection: "users" });
199+
Config.db.synRawUpdateOne("users", { _id: cli.user._id },
200+
{ $set: { _noderedheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } },
201+
Config.prometheus_measure_onlineuser, null);
210202
}
211203
if (cli.clientagent == "webapp" || cli.clientagent == "aiotwebapp") {
212-
Config.db.db.collection("users").updateOne({ _id: cli.user._id },
213-
{ $set: { _webheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } }).catch((err) => {
214-
console.error(err);
215-
});
204+
Config.db.synRawUpdateOne("users", { _id: cli.user._id },
205+
{ $set: { _webheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } },
206+
Config.prometheus_measure_onlineuser, null);
216207
}
217208
if (cli.clientagent == "powershell") {
218-
DatabaseConnection.mongodb_update_count.labels("users").inc();
219-
const end = DatabaseConnection.mongodb_update.startTimer();
220-
Config.db.db.collection("users").updateOne({ _id: cli.user._id },
221-
{ $set: { _powershellheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } }).catch((err) => {
222-
console.error(err);
223-
});
224-
end({ collection: "users" });
209+
Config.db.synRawUpdateOne("users", { _id: cli.user._id },
210+
{ $set: { _powershellheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } },
211+
Config.prometheus_measure_onlineuser, null);
225212
}
226213
if (cli.clientagent == "mobileapp" || cli.clientagent == "aiotmobileapp") {
227-
DatabaseConnection.mongodb_update_count.labels("users").inc();
228-
const end = DatabaseConnection.mongodb_update.startTimer();
229-
Config.db.db.collection("users").updateOne({ _id: cli.user._id },
230-
{ $set: { _webheartbeat: new Date(new Date().toISOString()), _mobilheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } }).catch((err) => {
231-
console.error(err);
232-
});
233-
end({ collection: "users" });
214+
Config.db.synRawUpdateOne("users", { _id: cli.user._id },
215+
{ $set: { _webheartbeat: new Date(new Date().toISOString()), _mobilheartbeat: new Date(new Date().toISOString()), _heartbeat: new Date(new Date().toISOString()) } },
216+
Config.prometheus_measure_onlineuser, null);
234217
}
235218
else {
236219
// Should proberly turn this a little down, so we dont update all online users every 10th second
237-
DatabaseConnection.mongodb_update_count.labels("users").inc();
238-
const end = DatabaseConnection.mongodb_update.startTimer();
239-
Config.db.db.collection("users").updateOne({ _id: cli.user._id }, { $set: { _heartbeat: new Date(new Date().toISOString()) } }).catch((err) => {
240-
console.error(err);
241-
});
242-
end({ collection: "users" });
220+
Config.db.synRawUpdateOne("users", { _id: cli.user._id },
221+
{ $set: { _heartbeat: new Date(new Date().toISOString()) } },
222+
Config.prometheus_measure_onlineuser, null);
243223
}
244224
}
245225
} catch (error) {

‎OpenFlow/src/index.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,9 @@ async function initDatabase(): Promise<boolean> {
207207
Base.addRight(filestore_users, filestore_users._id, "filestore users", [Rights.read]);
208208
}
209209
await DBHelper.Save(filestore_users, jwt);
210+
211+
await Config.db.ensureindexes();
212+
210213
return true;
211214
} catch (error) {
212215
logger.error(error);
@@ -216,6 +219,9 @@ async function initDatabase(): Promise<boolean> {
216219

217220

218221

222+
223+
224+
219225
const unhandledRejection = require("unhandled-rejection");
220226
let rejectionEmitter = unhandledRejection({
221227
timeout: 20

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.1.174
1+
1.1.175

0 commit comments

Comments
 (0)