From bd392e2fe25ffe64d4c056cf8b7ef26e51be8d68 Mon Sep 17 00:00:00 2001 From: pike Date: Sat, 9 Dec 2023 22:07:36 +0100 Subject: [PATCH 1/3] feat: Add json storage --- .env.dist | 1 + src/services/Storage.ts | 99 ++++++++++++++++++++++++++++++++--------- 2 files changed, 80 insertions(+), 20 deletions(-) diff --git a/.env.dist b/.env.dist index a044c88..d6b173d 100644 --- a/.env.dist +++ b/.env.dist @@ -8,6 +8,7 @@ FAIRPOST_LOGGER_LEVEL=trace FAIRPOST_LOGGER_CONSOLE=false FAIRPOST_STORAGE_SETTINGS=env FAIRPOST_STORAGE_AUTH=env +FAIRPOST_STORAGE_JSON=var/run/storage.json FAIRPOST_USER_AGENT=Fairpost 1.0 # feed settings diff --git a/src/services/Storage.ts b/src/services/Storage.ts index 76c50a1..6390366 100644 --- a/src/services/Storage.ts +++ b/src/services/Storage.ts @@ -1,23 +1,33 @@ +import * as fs from "fs"; +import * as path from "path"; + import Logger from "./Logger"; /** * Storage - minimalist singleton service * - * - sets and gets key / value pairs. + * - sets and gets key / value pairs, all string. * - uses two 'stores': * - 'settings' is typically what a user maintains, * - 'auth' is what fairpost maintains and may be * stored and encrypted somewhere else + * - uses two backends + * - 'env' is process.env (.env) + * - 'json' is plain json file * + * which store uses which backend should be + * set in the environment */ class Storage { static instance: Storage; + jsonData: { [key: string]: string } = {}; constructor() { if (Storage.instance) { throw new Error("Storage: call getInstance() instead"); } + this.loadJson(); } static getInstance(): Storage { @@ -28,21 +38,36 @@ class Storage { } public get(store: "settings" | "auth", key: string, def?: string): string { - let value = "" as string; const storage = store === "settings" ? process.env.FAIRPOST_STORAGE_SETTINGS - : process.env.FAIRPOST_STORAGE_AUTH; - if (storage === "env") { - value = process.env["FAIRPOST_" + key] ?? ""; - } else { - throw Logger.error("Storage " + storage + " not implemented"); + : (process.env.FAIRPOST_STORAGE_AUTH as "env" | "json"); + switch (storage) { + case "env": + return this.getEnv(key, def); + case "json": + return this.getJson(key, def); + default: + throw Logger.error("Storage " + storage + " not implemented"); } + } + + private getEnv(key: string, def?: string): string { + let value = process.env["FAIRPOST_" + key] ?? ""; if (!value) { if (def === undefined) { - throw Logger.error( - "Value " + key + " not found in store '" + store + "'", - ); + throw Logger.error("Storage.getEnv: Value " + key + " not found."); + } + value = def; + } + return value; + } + + private getJson(key: string, def?: string): string { + let value = this.jsonData[key] ?? ""; + if (!value) { + if (def === undefined) { + throw Logger.error("Storage.getJson: Value " + key + " not found."); } value = def; } @@ -53,21 +78,55 @@ class Storage { const storage = store === "settings" ? process.env.FAIRPOST_STORAGE_SETTINGS - : process.env.FAIRPOST_STORAGE_AUTH; + : (process.env.FAIRPOST_STORAGE_AUTH as "env" | "json"); + switch (storage) { + case "env": + return this.setEnv(key, value); + case "json": + return this.setJson(key, value); + default: + throw Logger.error("Storage " + storage + " not implemented"); + } + } + + private setEnv(key: string, value: string) { const ui = process.env.FAIRPOST_UI; - if (storage === "env") { - if (ui === "cli") { - console.log("Store this value in your .env file:"); - console.log(); - console.log("FAIRPOST_" + key + "=" + value); - console.log(); + if (ui === "cli") { + console.log("Store this value in your .env file:"); + console.log(); + console.log("FAIRPOST_" + key + "=" + value); + console.log(); + } else { + throw Logger.error("Storage.setEnv: UI " + ui + " not implemented"); + } + } + + private setJson(key: string, value: string) { + this.jsonData[key] = value; + this.saveJson(); + } + + private loadJson() { + const jsonFile = + process.env.FAIRPOST_STORAGE_JSONPATH || "var/run/storage.json"; + if (fs.existsSync(jsonFile)) { + const jsonData = JSON.parse(fs.readFileSync(jsonFile, "utf8")); + if (jsonData) { + this.jsonData = jsonData; } else { - throw Logger.error("UI " + ui + " not implemented"); + throw new Error("Storage.loadJson: cant parse " + jsonFile); } - } else { - throw Logger.error("Storage " + storage + " not implemented"); } } + + private saveJson() { + const jsonFile = + process.env.FAIRPOST_STORAGE_JSONPATH || "var/run/storage.json"; + if (!fs.existsSync(jsonFile)) { + fs.mkdirSync(path.dirname(jsonFile), { recursive: true }); + } + fs.writeFileSync(jsonFile, JSON.stringify(this.jsonData, null, "\t")); + } } export default Storage.getInstance(); From ac90f52854c119218eb002aa64cdf4e8d4a57d3c Mon Sep 17 00:00:00 2001 From: pike Date: Sat, 9 Dec 2023 22:44:11 +0100 Subject: [PATCH 2/3] feat: Add refresh-platform , plus impl for Reddit --- README.md | 2 + src/cli.ts | 12 ++++++ src/models/Feed.ts | 35 +++++++++++++-- src/models/Platform.ts | 68 ++++++++++++++++++------------ src/platforms/Reddit/Reddit.ts | 6 +++ src/platforms/Reddit/RedditAuth.ts | 23 ++++------ src/services/OAuth2Service.ts | 3 +- 7 files changed, 102 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 5edd4da..0732bbf 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,8 @@ fairpost.js setup-platform --platform=xxx fairpost.js setup-platforms [--platforms=xxx,xxx] fairpost.js test-platform --platform=xxx fairpost.js test-platforms [--platforms=xxx,xxx] +fairpost.js refresh-platform --platform=xxx +fairpost.js refresh-platforms [--platforms=xxx,xxx] fairpost.js get-platform --platform=xxx fairpost.js get-platforms [--platforms=xxx,xxx] fairpost.js get-folder --folder=xxx diff --git a/src/cli.ts b/src/cli.ts index ab0c8bb..cc5c8c7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -89,6 +89,16 @@ async function main() { report = "Result: \n" + JSON.stringify(result, null, "\t"); break; } + case "refresh-platform": { + result = await feed.refreshPlatform(PLATFORM); + report = "Result: \n" + JSON.stringify(result, null, "\t"); + break; + } + case "refresh-platforms": { + result = await feed.refreshPlatforms(PLATFORMS); + report = "Result: \n" + JSON.stringify(result, null, "\t"); + break; + } case "get-folder": { const folder = feed.getFolder(FOLDER); report += folder.report() + "\n"; @@ -221,6 +231,8 @@ async function main() { `${cmd} setup-platforms [--platforms=xxx,xxx]`, `${cmd} test-platform --platform=xxx`, `${cmd} test-platforms [--platforms=xxx,xxx]`, + `${cmd} refresh-platform --platform=xxx`, + `${cmd} refresh-platforms [--platforms=xxx,xxx]`, `${cmd} get-platform --platform=xxx`, `${cmd} get-platforms [--platforms=xxx,xxx]`, `${cmd} get-folder --folder=xxx`, diff --git a/src/models/Feed.ts b/src/models/Feed.ts index 4f1d67e..72278a4 100644 --- a/src/models/Feed.ts +++ b/src/models/Feed.ts @@ -82,7 +82,8 @@ export default class Feed { ): Promise<{ [id: string]: unknown }> { Logger.trace("Feed", "setupPlatforms", platformsIds); const results = {}; - for (const platformId of platformsIds) { + for (const platformId of platformsIds ?? + (Object.keys(this.platforms) as PlatformId[])) { results[platformId] = await this.setupPlatform(platformId); } return results; @@ -125,7 +126,7 @@ export default class Feed { } /** - * Test more platforms + * Test multiple platforms * @param platformsIds - the slugs of the platforms * @returns the test results indexed by platform ids */ @@ -134,12 +135,40 @@ export default class Feed { ): Promise<{ [id: string]: unknown }> { Logger.trace("Feed", "testPlatforms", platformsIds); const results = {}; - for (const platformId of platformsIds) { + for (const platformId of platformsIds ?? + (Object.keys(this.platforms) as PlatformId[])) { results[platformId] = await this.testPlatform(platformId); } return results; } + /** + * Refresh one platform + * @param platformId - the slug of the platform + * @returns the refresh result + */ + async refreshPlatform(platformId: PlatformId): Promise { + Logger.trace("Feed", "refreshPlatform", platformId); + return await this.getPlatform(platformId).refresh(); + } + + /** + * Refresh multiple platforms + * @param platformsIds - the slugs of the platforms + * @returns the refresh results indexed by platform ids + */ + async refreshPlatforms( + platformsIds?: PlatformId[], + ): Promise<{ [id: string]: boolean }> { + Logger.trace("Feed", "refreshPlatforms", platformsIds); + const results = {}; + for (const platformId of platformsIds ?? + (Object.keys(this.platforms) as PlatformId[])) { + results[platformId] = await this.refreshPlatform(platformId); + } + return results; + } + /** * Get all folders * @returns all folder in the feed diff --git a/src/models/Platform.ts b/src/models/Platform.ts index 5ef7e8b..70601f0 100644 --- a/src/models/Platform.ts +++ b/src/models/Platform.ts @@ -31,6 +31,46 @@ export default class Platform { return report; } + /** + * setup + * + * Set the platform up. Get the required keys and tokens. + * This may involve starting a webserver and/or communicating + * via the CLI. + * @returns - any object + */ + async setup() { + throw Logger.error( + "No setup implemented for " + + this.id + + ". Read the docs in the docs folder.", + ); + } + + /** + * test + * + * Test the platform installation. This should not post + * anything, but test access tokens et al. It can return + * anything. + * @returns - any object + */ + async test(): Promise { + return "No tests implemented for " + this.id; + } + + /** + * refresh + * + * Refresh the platform installation. This usually refreshes + * access tokens if required. It can throw errors + * @returns - true if refreshed + */ + async refresh(): Promise { + Logger.trace("Refresh not implemented for " + this.id); + return false; + } + /** * getAssetsFolderName * @returns the relative path to a folder used @@ -159,32 +199,4 @@ export default class Platform { post.save(); return false; } - - /** - * setup - * - * Set the platform up. Get the required keys and tokens. - * This may involve starting a webserver and/or communicating - * via the CLI. - * @returns - any object - */ - async setup() { - throw Logger.error( - "No setup implemented for " + - this.id + - ". Read the docs in the docs folder.", - ); - } - - /** - * test - * - * Test the platform installation. This should not post - * anything, but test access tokens et al. It can return - * anything. - * @returns - any object - */ - async test(): Promise { - return "No tests implemented for " + this.id; - } } diff --git a/src/platforms/Reddit/Reddit.ts b/src/platforms/Reddit/Reddit.ts index 08d8816..59dd726 100644 --- a/src/platforms/Reddit/Reddit.ts +++ b/src/platforms/Reddit/Reddit.ts @@ -45,6 +45,12 @@ export default class Reddit extends Platform { }; } + /** @inheritdoc */ + async refresh(): Promise { + await this.auth.refreshAccessToken(); + return true; + } + async preparePost(folder: Folder): Promise { const post = await super.preparePost(folder); if (post) { diff --git a/src/platforms/Reddit/RedditAuth.ts b/src/platforms/Reddit/RedditAuth.ts index 21255e5..7aaa9c9 100644 --- a/src/platforms/Reddit/RedditAuth.ts +++ b/src/platforms/Reddit/RedditAuth.ts @@ -14,19 +14,11 @@ export default class RedditAuth { Storage.set("auth", "REDDIT_REFRESH_TOKEN", tokens["refresh_token"]); } - public async getAccessToken(): Promise { - return await this.refreshAccessToken(); - } - /** - * Get Reddit Access token + * Refresh Reddit Access token * - * Reddits access token expire in 24 hours. Instead - * of using an access token from the Storage, the Reddit - * platform gets its token from here, which refreshes - * it if needed using the refresh_token - * - * ~~ TODO the api cant access these + * Reddits access token expire in 24 hours. + * Refresh this regularly. * @returns The access token */ public async refreshAccessToken(): Promise { @@ -35,15 +27,18 @@ export default class RedditAuth { } const result = await this.post("access_token", { grant_type: "refresh_token", - refresh_token: Storage.get("settings", "REDDIT_REFRESH_TOKEN"), + refresh_token: Storage.get("auth", "REDDIT_REFRESH_TOKEN"), }); if (!result["access_token"]) { const msg = "Remote response did not return a access_token"; throw Logger.error(msg, result); } - this.accessToken = result["access_token"]; - return this.accessToken; + const accessToken = result["access_token"]; + if (!accessToken) { + throw new Error("RedditAuth: refresh failed - no access token"); + } + Storage.set("auth", "REDDIT_ACCESS_TOKEN", accessToken); } protected async requestCode(): Promise { diff --git a/src/services/OAuth2Service.ts b/src/services/OAuth2Service.ts index 5b2e394..1d7d6c4 100644 --- a/src/services/OAuth2Service.ts +++ b/src/services/OAuth2Service.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as http from "http"; import * as url from "url"; + import Storage from "./Storage"; class DeferredResponseQuery { @@ -39,8 +40,6 @@ export default class OAuth2Service { * resolves the query passed. * @param serviceName - the name of the remote platform * @param serviceLink - the uri to the remote platform - * @param clientHost - the host name to serve the local page on - * @param clientPort - the port to serve the local page on * @returns a flat object of returned query */ From fb7ab65bbdcf6b80e76b78d6d1e4ea14e004f2d9 Mon Sep 17 00:00:00 2001 From: pike Date: Sat, 9 Dec 2023 22:46:26 +0100 Subject: [PATCH 3/3] feat: Default auth to json --- .env.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.dist b/.env.dist index d6b173d..5624ad0 100644 --- a/.env.dist +++ b/.env.dist @@ -7,7 +7,7 @@ FAIRPOST_LOGGER_CATEGORY=default FAIRPOST_LOGGER_LEVEL=trace FAIRPOST_LOGGER_CONSOLE=false FAIRPOST_STORAGE_SETTINGS=env -FAIRPOST_STORAGE_AUTH=env +FAIRPOST_STORAGE_AUTH=json FAIRPOST_STORAGE_JSON=var/run/storage.json FAIRPOST_USER_AGENT=Fairpost 1.0