Skip to content

Commit b0c8145

Browse files
skadefroskadefro
committed
Push history, add delete history
1 parent 4d06bb6 commit b0c8145

10 files changed

Lines changed: 345 additions & 67 deletions

File tree

‎OpenFlow/src/DatabaseConnection.ts‎

Lines changed: 153 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -533,7 +533,12 @@ export class DatabaseConnection {
533533
try {
534534

535535
let result: T = await this.getbyid<T>(id, collectionname, jwt, span);
536-
if (result == null) return result;
536+
if (result == null) {
537+
const subbasehist = await this.query<any>({ id: id, item: { $exists: true, $ne: null } }, null, 1, 0, { _version: -1 }, collectionname + "_hist", jwt, undefined, undefined, span);
538+
if (subbasehist.length == 0) return null;
539+
result = subbasehist[0];
540+
result._version = version + 1;
541+
}
537542
if (result._version > version) {
538543
const rootjwt = Crypt.rootToken()
539544
// const baseversion = roundDown(version, Config.history_delta_count);
@@ -823,22 +828,49 @@ export class DatabaseConnection {
823828

824829
span.addEvent("encryptentity");
825830
item = this.encryptentity(item) as T;
826-
if (!item._id) { item._id = new ObjectID().toHexString(); }
827831

828832
if (collectionname === "users" && item._type === "user" && item.hasOwnProperty("newpassword")) {
829833
(item as any).passwordhash = await Crypt.hash((item as any).newpassword);
830834
delete (item as any).newpassword;
831835
}
832836
j = ((j as any) === 'true' || j === true);
833837
w = parseInt((w as any));
834-
835-
if (item.hasOwnProperty("_skiphistory")) {
836-
delete (item as any)._skiphistory;
837-
if (!Config.allow_skiphistory) {
838-
item._version = await this.SaveDiff(collectionname, null, item, span);
838+
item._version = 0;
839+
if (item._id != null) {
840+
const basehist = await this.query<any>({ id: item._id }, { _version: 1 }, 1, 0, { _version: -1 }, collectionname + "_hist", Crypt.rootToken(), undefined, undefined, span);
841+
if (basehist.length > 0) {
842+
item._version = basehist[0]._version;
843+
}
844+
if (basehist.length > 0) {
845+
const org = await this.GetDocumentVersion(collectionname, item._id, item._version, Crypt.rootToken(), span)
846+
if (org != null) {
847+
item._createdby = org._createdby;
848+
item._createdbyid = org._createdbyid;
849+
item._created = org._created;
850+
item._modifiedby = org._modifiedby;
851+
item._modifiedbyid = org._modifiedbyid;
852+
item._modified = org._modified;
853+
if (!item._created) item._created = new Date(new Date().toISOString());
854+
if (!item._createdby) item._createdby = user.name;
855+
if (!item._createdbyid) item._createdbyid = user._id;
856+
if (!item._modified) item._modified = new Date(new Date().toISOString());
857+
if (!item._modifiedby) item._modifiedby = user.name;
858+
if (!item._modifiedbyid) item._modifiedbyid = user._id;
859+
860+
if (item.hasOwnProperty("_skiphistory")) {
861+
delete (item as any)._skiphistory;
862+
if (!Config.allow_skiphistory) {
863+
item._version = await this.SaveDiff(collectionname, org, item, span);
864+
}
865+
} else {
866+
item._version = await this.SaveDiff(collectionname, org, item, span);
867+
}
868+
} else {
869+
item._version++;
870+
}
839871
}
840872
} else {
841-
item._version = await this.SaveDiff(collectionname, null, item, span);
873+
item._id = new ObjectID().toHexString();
842874
}
843875
span.addEvent("CleanACL");
844876
item = await this.CleanACL(item, user, span);
@@ -1432,10 +1464,39 @@ export class DatabaseConnection {
14321464
throw Error("item not found!");
14331465
}
14341466
}
1435-
if (Config.log_deletes) Logger.instanse.verbose("[" + user.username + "][" + collectionname + "] Deleting " + id + " in database");
1436-
const ot_end = Logger.otel.startTimer();
1437-
const res: DeleteWriteOpResultObject = await this.db.collection(collectionname).deleteOne(_query);
1438-
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_delete, { collection: collectionname });
1467+
// if (Config.log_deletes) Logger.instanse.verbose("[" + user.username + "][" + collectionname + "] Deleting " + id + " in database");
1468+
// const ot_end = Logger.otel.startTimer();
1469+
// const res: DeleteWriteOpResultObject = await this.db.collection(collectionname).deleteOne(_query);
1470+
// Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_delete, { collection: collectionname });
1471+
var docs = await this.db.collection(collectionname).find(_query).toArray();
1472+
for (var i = 0; i < docs.length; i++) {
1473+
var doc = docs[i];
1474+
doc._deleted = new Date(new Date().toISOString());
1475+
doc._deletedby = user.name;
1476+
doc._deletedbyid = user._id;
1477+
const fullhist = {
1478+
_acl: doc._acl,
1479+
_type: doc._type,
1480+
_modified: doc._modified,
1481+
_modifiedby: doc._modifiedby,
1482+
_modifiedbyid: doc._modifiedbyid,
1483+
_created: doc._modified,
1484+
_createdby: doc._modifiedby,
1485+
_createdbyid: doc._modifiedbyid,
1486+
_deleted: doc._deleted,
1487+
_deletedby: doc._deletedby,
1488+
_deletedbyid: doc._deletedbyid,
1489+
name: doc.name,
1490+
id: doc._id,
1491+
item: doc,
1492+
_version: doc._version,
1493+
reason: doc.reason
1494+
}
1495+
const ot_end = Logger.otel.startTimer();
1496+
await this.db.collection(collectionname + '_hist').insertOne(fullhist);
1497+
await this.db.collection(collectionname).deleteOne({ _id: doc._id });
1498+
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_delete, { collection: collectionname });
1499+
}
14391500
}
14401501

14411502
/**
@@ -1497,11 +1558,55 @@ export class DatabaseConnection {
14971558
if (Config.log_deletes) Logger.instanse.verbose("[" + user.username + "][" + collectionname + "] deleted " + arr.length + " items in database");
14981559
return arr.length;
14991560
} else {
1561+
// const ot_end = Logger.otel.startTimer();
1562+
// const res: DeleteWriteOpResultObject = await this.db.collection(collectionname).deleteMany(_query);
1563+
// Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_deletemany, { collection: collectionname });
1564+
var bulkInsert = this.db.collection(collectionname + "_hist").initializeUnorderedBulkOp();
1565+
var bulkRemove = this.db.collection(collectionname).initializeUnorderedBulkOp()
1566+
var x = 1000
1567+
var counter = 0
1568+
var date = new Date()
1569+
date.setMonth(date.getMonth() - 1)
1570+
1571+
var docs = await this.db.collection(collectionname).find(_query).toArray();
1572+
for (var i = 0; i < docs.length; i++) {
1573+
var doc = docs[i];
1574+
const fullhist = {
1575+
_acl: doc._acl,
1576+
_type: doc._type,
1577+
_modified: doc._modified,
1578+
_modifiedby: doc._modifiedby,
1579+
_modifiedbyid: doc._modifiedbyid,
1580+
_created: doc._modified,
1581+
_createdby: doc._modifiedby,
1582+
_createdbyid: doc._modifiedbyid,
1583+
_deleted: new Date(new Date().toISOString()),
1584+
_deletedby: user.name,
1585+
_deletedbyid: user._id,
1586+
name: doc.name,
1587+
id: doc._id,
1588+
item: doc,
1589+
_version: doc._version,
1590+
reason: doc.reason
1591+
}
1592+
bulkInsert.insert(fullhist);
1593+
bulkRemove.find({ _id: doc._id }).removeOne();
1594+
counter++
1595+
if (counter % x == 0) {
1596+
const ot_end = Logger.otel.startTimer();
1597+
bulkInsert.execute()
1598+
bulkRemove.execute()
1599+
bulkInsert = this.db.collection(collectionname + "_hist").initializeUnorderedBulkOp()
1600+
bulkRemove = this.db.collection(collectionname).initializeUnorderedBulkOp()
1601+
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_deletemany, { collection: collectionname });
1602+
}
1603+
}
15001604
const ot_end = Logger.otel.startTimer();
1501-
const res: DeleteWriteOpResultObject = await this.db.collection(collectionname).deleteMany(_query);
1605+
bulkInsert.execute()
1606+
bulkRemove.execute()
15021607
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_deletemany, { collection: collectionname });
1503-
if (Config.log_deletes) Logger.instanse.verbose("[" + user.username + "][" + collectionname + "] deleted " + res.deletedCount + " items in database");
1504-
return res.deletedCount;
1608+
if (Config.log_deletes) Logger.instanse.verbose("[" + user.username + "][" + collectionname + "] deleted " + counter + " items in database");
1609+
return counter;
15051610
}
15061611
}
15071612
/**
@@ -1886,26 +1991,46 @@ export class DatabaseConnection {
18861991
}
18871992
}
18881993
let delta: any = null;
1889-
// for backward comp, we cannot assume all objects have an history
1890-
// we create diff from version 0
1891-
// const delta_collections = Config.history_delta_collections.split(',');
1892-
// const full_collections = Config.history_full_collections.split(',');
1893-
// if (delta_collections.indexOf(collectionname) == -1 && full_collections.indexOf(collectionname) == -1) return 0;
18941994

18951995
item._version = _version;
18961996
delete item._modifiedby;
18971997
delete item._modifiedbyid;
18981998
delete item._modified;
18991999
delete item._updatereason;
19002000
delete item.lastseen;
1901-
1902-
// if (original != null && _version > 0 && delta_collections.indexOf(collectionname) > -1) {
1903-
if (original != null && _version > 0) {
1904-
this.visit(item, (obj, k) => {
1905-
if (typeof obj[k] === "function") {
1906-
delete obj[k];
1907-
}
2001+
this.visit(item, (obj, k) => {
2002+
if (typeof obj[k] === "function") {
2003+
delete obj[k];
2004+
}
2005+
});
2006+
if (original != null && original._version == 0) {
2007+
const fullhist = {
2008+
_acl: _acl,
2009+
_type: _type,
2010+
_modified: _modified,
2011+
_modifiedby: _modifiedby,
2012+
_modifiedbyid: _modifiedbyid,
2013+
_created: _modified,
2014+
_createdby: _modifiedby,
2015+
_createdbyid: _modifiedbyid,
2016+
name: original.name,
2017+
id: original._id,
2018+
item: original,
2019+
_version: 0,
2020+
reason: reason
2021+
}
2022+
const ot_end = Logger.otel.startTimer();
2023+
const mongodbspan: Span = Logger.otel.startSubSpan("mongodb.insertOne", span);
2024+
this.db.collection(collectionname + '_hist').insertOne(fullhist).then(() => {
2025+
Logger.otel.endSpan(mongodbspan);
2026+
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_insert, { collection: collectionname + '_hist' });
2027+
}).catch(err => {
2028+
mongodbspan.recordException(err);
2029+
Logger.otel.endSpan(mongodbspan);
2030+
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_insert, { collection: collectionname + '_hist' });
19082031
});
2032+
}
2033+
if (original != null && original._version > 0) {
19092034
delta = jsondiffpatch.diff(original, item);
19102035
if (delta == undefined || delta == null) return 0;
19112036
const keys = Object.keys(delta);
@@ -1941,33 +2066,8 @@ export class DatabaseConnection {
19412066
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_insert, { collection: collectionname + '_hist' });
19422067
});
19432068
}
1944-
} else {
1945-
const fullhist = {
1946-
_acl: _acl,
1947-
_type: _type,
1948-
_modified: _modified,
1949-
_modifiedby: _modifiedby,
1950-
_modifiedbyid: _modifiedbyid,
1951-
_created: _modified,
1952-
_createdby: _modifiedby,
1953-
_createdbyid: _modifiedbyid,
1954-
name: item.name,
1955-
id: item._id,
1956-
item: item,
1957-
_version: _version,
1958-
reason: reason
1959-
}
1960-
const ot_end = Logger.otel.startTimer();
1961-
const mongodbspan: Span = Logger.otel.startSubSpan("mongodb.insertOne", span);
1962-
this.db.collection(collectionname + '_hist').insertOne(fullhist).then(() => {
1963-
Logger.otel.endSpan(mongodbspan);
1964-
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_insert, { collection: collectionname + '_hist' });
1965-
}).catch(err => {
1966-
mongodbspan.recordException(err);
1967-
Logger.otel.endSpan(mongodbspan);
1968-
Logger.otel.endTimer(ot_end, DatabaseConnection.mongodb_insert, { collection: collectionname + '_hist' });
1969-
});
19702069
}
2070+
19712071
item._modifiedby = _modifiedby;
19722072
item._modifiedbyid = _modifiedbyid;
19732073
item._modified = _modified;

‎OpenFlow/src/public/CommonControllers.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ export class entitiesCtrl<T> {
312312
}
313313
}
314314
}
315-
private static parseJson(txt, reviver, context) {
315+
public static parseJson(txt, reviver, context) {
316316
context = context || 20
317317
try {
318318
return JSON.parse(txt, reviver)

0 commit comments

Comments
 (0)