Skip to content

Commit 735fb17

Browse files
skadefroskadefro
committed
Work around on weird mongodb bug
1 parent 90c0578 commit 735fb17

1 file changed

Lines changed: 173 additions & 23 deletions

File tree

‎OpenFlow/src/Messages/Message.ts‎

Lines changed: 173 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3107,7 +3107,7 @@ export class Message {
31073107
if ((hasUser === null || hasUser === undefined)) {
31083108
Base.addRight(msg.metadata, WellknownIds.filestore_admins, "filestore admins", [Rights.full_control]);
31093109
}
3110-
msg.metadata = Config.db.ensureResource(msg.metadata);
3110+
msg.metadata = Config.db.ensureResource(msg.metadata, "fs.files");
31113111
if (!NoderedUtil.hasAuthorization(user, msg.metadata, Rights.create)) { throw new Error("Access denied, no authorization to save file"); }
31123112
msg.id = await this._SaveFile(readable, msg.filename, msg.mimeType, msg.metadata);
31133113
msg.result = await Config.db.getbyid(msg.id, "fs.files", msg.jwt, null);
@@ -3244,7 +3244,7 @@ export class Message {
32443244
Base.addRight(msg.metadata, WellknownIds.filestore_admins, "filestore admins", [Rights.full_control]);
32453245
if (!NoderedUtil.hasAuthorization(user, msg.metadata, Rights.update)) { throw new Error("Access denied, no authorization to update file"); }
32463246

3247-
msg.metadata = Config.db.ensureResource(msg.metadata);
3247+
msg.metadata = Config.db.ensureResource(msg.metadata, "fs.files");
32483248
const fsc = Config.db.db.collection("fs.files");
32493249
DatabaseConnection.traversejsonencode(msg.metadata);
32503250
const res = await fsc.updateOne(q, { $set: { metadata: msg.metadata } });
@@ -4424,6 +4424,143 @@ export class Message {
44244424
yesterday.setDate(yesterday.getDate() - 1);
44254425

44264426
try {
4427+
for (let i = 0; i < DatabaseConnection.collections_with_text_index.length; i++) {
4428+
let collectionname = DatabaseConnection.collections_with_text_index[i];
4429+
if (DatabaseConnection.timeseries_collections.indexOf(collectionname) > -1) continue;
4430+
if (DatabaseConnection.usemetadata(collectionname)) {
4431+
let exists = await Config.db.db.collection(collectionname).findOne({ "metadata._searchnames": { $exists: false } });
4432+
if (!NoderedUtil.IsNullUndefinded(exists)) {
4433+
Logger.instanse.info("[housekeeping][" + collectionname + "] Start creating metadata._searchnames for collection " + collectionname);
4434+
await Config.db.db.collection(collectionname).updateMany({ "metadata._searchnames": { $exists: false } },
4435+
[
4436+
{
4437+
"$set": {
4438+
"metadata._searchnames":
4439+
{
4440+
$split: [
4441+
{
4442+
$replaceAll: {
4443+
input:
4444+
{
4445+
$replaceAll: {
4446+
input:
4447+
{
4448+
$replaceAll: {
4449+
input:
4450+
{ $toLower: "$metadata.name" }
4451+
, find: ".", replacement: " "
4452+
}
4453+
}
4454+
, find: "-", replacement: " "
4455+
}
4456+
}
4457+
, find: "/", replacement: " "
4458+
}
4459+
}
4460+
, " "]
4461+
}
4462+
}
4463+
}
4464+
,
4465+
{
4466+
"$set": {
4467+
"_searchname":
4468+
{
4469+
$replaceAll: {
4470+
input:
4471+
{
4472+
$replaceAll: {
4473+
input:
4474+
{
4475+
$replaceAll: {
4476+
input:
4477+
{ $toLower: "$metadata.name" }
4478+
, find: ".", replacement: " "
4479+
}
4480+
}
4481+
, find: "-", replacement: " "
4482+
}
4483+
}
4484+
, find: "/", replacement: " "
4485+
}
4486+
}
4487+
}
4488+
}
4489+
,
4490+
{ "$set": { "metadata._searchnames": { $concatArrays: ["$metadata._searchnames", [{ $toLower: "$metadata.name" }]] } } }
4491+
]
4492+
)
4493+
Logger.instanse.info("[housekeeping][" + collectionname + "] Done creating _searchnames for collection " + collectionname);
4494+
}
4495+
} else {
4496+
let exists = await Config.db.db.collection(collectionname).findOne({ "_searchnames": { $exists: false } });
4497+
if (!NoderedUtil.IsNullUndefinded(exists)) {
4498+
Logger.instanse.info("[housekeeping][" + collectionname + "] Start creating _searchnames for collection " + collectionname);
4499+
await Config.db.db.collection(collectionname).updateMany({ "_searchnames": { $exists: false } },
4500+
[
4501+
{
4502+
"$set": {
4503+
"_searchnames":
4504+
{
4505+
$split: [
4506+
{
4507+
$replaceAll: {
4508+
input:
4509+
{
4510+
$replaceAll: {
4511+
input:
4512+
{
4513+
$replaceAll: {
4514+
input:
4515+
{ $toLower: "$name" }
4516+
, find: ".", replacement: " "
4517+
}
4518+
}
4519+
, find: "-", replacement: " "
4520+
}
4521+
}
4522+
, find: "/", replacement: " "
4523+
}
4524+
}
4525+
, " "]
4526+
}
4527+
}
4528+
}
4529+
,
4530+
{
4531+
"$set": {
4532+
"_searchname":
4533+
{
4534+
$replaceAll: {
4535+
input:
4536+
{
4537+
$replaceAll: {
4538+
input:
4539+
{
4540+
$replaceAll: {
4541+
input:
4542+
{ $toLower: "$name" }
4543+
, find: ".", replacement: " "
4544+
}
4545+
}
4546+
, find: "-", replacement: " "
4547+
}
4548+
}
4549+
, find: "/", replacement: " "
4550+
}
4551+
}
4552+
}
4553+
}
4554+
,
4555+
{ "$set": { "_searchnames": { $concatArrays: ["$_searchnames", [{ $toLower: "$name" }]] } } }
4556+
]
4557+
)
4558+
Logger.instanse.info("[housekeeping][" + collectionname + "] Done creating _searchnames for collection " + collectionname);
4559+
}
4560+
}
4561+
}
4562+
4563+
// skipCalculateSize = false;
44274564
if (!skipCalculateSize) {
44284565

44294566
const user = Crypt.rootUser();
@@ -4505,29 +4642,43 @@ export class Message {
45054642
Config.db.db.collection("dbusage").deleteMany({ timestamp: timestamp, collection: col.name });
45064643
let usage = 0;
45074644
if (items.length > 0) {
4508-
let bulkInsert = Config.db.db.collection("dbusage").initializeUnorderedBulkOp();
4509-
for (var _item of items) {
4510-
let item = Config.db.ensureResource(_item);
4511-
item = await Config.db.CleanACL(item, tuser, span);
4512-
delete item._id;
4513-
item.username = item.name;
4514-
item.name = item.name + " / " + col.name + " / " + this.formatBytes(_item.size);
4515-
item._type = "metered";
4516-
item._createdby = "root";
4517-
item._createdbyid = WellknownIds.root;
4518-
item._created = new Date(new Date().toISOString());
4519-
item._modifiedby = "root";
4520-
item._modifiedbyid = WellknownIds.root;
4521-
item._modified = item._created;
4522-
usage += item.size;
4523-
DatabaseConnection.traversejsonencode(item);
4524-
4525-
bulkInsert.insert(item);
4526-
}
4645+
// let bulkInsert = Config.db.db.collection("dbusage").initializeUnorderedBulkOp();
4646+
for (var i = 0; i < items.length; i++) {
4647+
try {
4648+
// sometimes the item is "weird", re-serializing it, cleans it, so it works again ... mongodb bug ???
4649+
let item = JSON.parse(JSON.stringify(items[i]));
4650+
item = Config.db.ensureResource(item, "dbusage");
4651+
item = await Config.db.CleanACL(item, tuser, "dbusage", span);
4652+
delete item._id;
4653+
item.username = item.name;
4654+
item.name = item.name + " / " + col.name + " / " + this.formatBytes(item.size);
4655+
item._type = "metered";
4656+
item._createdby = "root";
4657+
item._createdbyid = WellknownIds.root;
4658+
item._created = new Date(new Date().toISOString());
4659+
item._modifiedby = "root";
4660+
item._modifiedbyid = WellknownIds.root;
4661+
item._modified = item._created;
4662+
usage += item.size;
4663+
DatabaseConnection.traversejsonencode(item);
4664+
item.timestamp = new Date(timestamp.toISOString());
4665+
if (col.name == "cvr") {
4666+
delete item.timestamp;
4667+
}
4668+
await Config.db.db.collection("dbusage").insertOne(item);
4669+
if (col.name == "cvr") {
4670+
await Config.db.db.collection("dbusage").updateOne({ _id: item._id }, { $set: { "timestamp": new Date(timestamp.toISOString()) } });
4671+
}
4672+
// bulkInsert.insert(item);
4673+
} catch (error) {
4674+
Logger.instanse.error(error);
4675+
span?.recordException(error);
4676+
}
45274677

4678+
}
45284679
totalusage += usage;
45294680
try {
4530-
await bulkInsert.execute();
4681+
// await bulkInsert.execute();
45314682
if (items.length > 0) Logger.instanse.debug("[housekeeping][" + col.name + "][" + index + "/" + collections.length + "] add " + items.length + " items with a usage of " + this.formatBytes(usage));
45324683

45334684
} catch (error) {
@@ -4563,7 +4714,6 @@ export class Message {
45634714
"_id": "$userid",
45644715
"size": { "$sum": "$size" },
45654716
"count": { "$sum": 1 }
4566-
45674717
}
45684718
}
45694719
]// "items": { "$push": "$$ROOT" }

0 commit comments

Comments
 (0)