diff --git a/README.md b/README.md index f6203a1..d6c2e49 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,16 @@ Fairpost helps you manage users social media feeds from a single entry point, using Node. It supports Facebook, Instagram, Reddit, Twitter, YouTube and LinkedIn. -A Feed is just a folder on disk, and all subfolders are Source Posts, -containing at least one text file (the post body) and -optionally images or video. The Source Post will be transformed -into real posts for each connected platform. +Fairpost behaves like a feed, not a calendar. +By default, there is only one scheduled post for each +platform. Once it is published, the next post may +be scheduled. + +A Feed is just a bunch of folders on disk, and all subfolders +in each folder are Source Posts, containing at least one text +file (the post body) and optionally images or video. The Source +Post will be transformed into destination posts for each +connected platform. Fairpost is *opinionated*, meaning, it will decide how a Source Post with contents can best be presented @@ -25,13 +31,13 @@ post on their behalf. This is usually done via an online (oauth) consent page in a webbrowser. Commonly, you would call this script every day or week -for every user. Fairpost can then automatically **prepare** the folders, -**schedule** the next post using a certain interval and -**publish** any post when it is due. All the user has to do is -add folders with content. +for every user. Fairpost can then automatically **prepare** the posts +from the sources, **schedule** the next post using a certain interval +and **publish** any post when it is due. All the user has to do is +add folders with content in the `incoming` folder. Or, if you prefer, you can manually publish one -specific folder as posts on all supported and enabled +specific source as posts on all supported and enabled platforms at once, or just one post on one platform, etcetera. @@ -74,12 +80,24 @@ nano users/foobar/var/lib/storage.json ``` ## Feed planning + +Inside each feed, there are folders for `incoming`, +`pending`, `active`, `done` and `archived` sources. +The location of a source depends on the statusses of +the posts inside that source. New posts should be +added to 'incoming'; from there, Fairpost will manage +the sources location with the commands below. + ### Prepare ``` fairpost.js prepare-posts ``` Sources need to be `prepared` (iow turned into posts) before they can be published to a platform. +`prepare-posts` will prepare allthe sources in the +`incoming` folder and on success, move the source +and its posts to the `pending` folder. + Each platform, as defined in src/platforms, will handle the folder contents by itself. It may decide to modify the media (eg, scale images) @@ -89,6 +107,7 @@ is youtube). Finally, it will add a json file describing the post for that platform in the folder. + ### Schedule ``` fairpost.js schedule-next-post @@ -101,12 +120,22 @@ By default the date will be `FAIRPOST_FEED_INTERVAL` days after the last post for that platform, or `now`, whichever is latest. +TODO +Without arguments, `schedule-next-post` will select the +next post for each platform from the `pending` or `active` folders. +If it wasn't already there, it will move the source and +its posts to the `active` folder. + ### Publish ``` fairpost.js publish-due-posts ``` This will publish any scheduled posts that are past their due date. +Without arguments, `publish-due-posts` will search for a +due post for each platform from the `active` folder. +Once all posts from a source are published, it will move +the source and its posts to the `done` folder. ## Other commands diff --git a/docs/NewPlatform.md b/docs/NewPlatform.md index e1ca6e5..90329ee 100644 --- a/docs/NewPlatform.md +++ b/docs/NewPlatform.md @@ -38,7 +38,7 @@ export default class FooBar extends Platform { const post = await super.preparePost(source); if (post) { // prepare your post here - post.save(); + await post.save(); } return post; } diff --git a/etc/skeleton/feed/README.md b/etc/skeleton/feed/README.md index ec28cb8..369a23d 100644 --- a/etc/skeleton/feed/README.md +++ b/etc/skeleton/feed/README.md @@ -3,5 +3,9 @@ This folder can actually be anywhere, the path can be defined in your `.env` file. -Post your source posts here, each in a separate folder. -Folder names starting with an underscore are ignored. \ No newline at end of file +Post your new source posts in the incoming folder, each in a separate folder. +Folder names starting with an underscore or dot are ignored. + +Fairpost will manage these sources while processing +the feed; they will move to other folders in the feed +depending on the posts statusses. \ No newline at end of file diff --git a/etc/skeleton/feed/active/README.md b/etc/skeleton/feed/active/README.md new file mode 100644 index 0000000..2922430 --- /dev/null +++ b/etc/skeleton/feed/active/README.md @@ -0,0 +1,7 @@ +# Fairpost Active sources + +A source is located here if all if its not incoming, +pending, done or archived. Most likely, some posts here +are scheduled. + +Fairpost will manage these sources; dont edit them here. \ No newline at end of file diff --git a/etc/skeleton/feed/archived/README.md b/etc/skeleton/feed/archived/README.md new file mode 100644 index 0000000..aea3562 --- /dev/null +++ b/etc/skeleton/feed/archived/README.md @@ -0,0 +1,6 @@ +# Fairpost Archived sources + +A source is located here if it is archived. Most likely, +all posts here are either published or canceled. + +Fairpost will manage these sources; dont edit them here. \ No newline at end of file diff --git a/etc/skeleton/feed/done/README.md b/etc/skeleton/feed/done/README.md new file mode 100644 index 0000000..1e1d17a --- /dev/null +++ b/etc/skeleton/feed/done/README.md @@ -0,0 +1,4 @@ +# Fairpost Done sources + +A source is located here if all if its posts are published or canceled. +Fairpost will manage these sources; dont edit them here. \ No newline at end of file diff --git a/etc/skeleton/feed/incoming/README.md b/etc/skeleton/feed/incoming/README.md new file mode 100644 index 0000000..1bc280c --- /dev/null +++ b/etc/skeleton/feed/incoming/README.md @@ -0,0 +1,9 @@ +# Fairpost Incoming sources + +Add your new sources here. + +Folder names starting with an underscore or dot are ignored. + +Fairpost will manage these sources while processing +the feed; they will move to other folders in the feed +depending on the posts statusses. \ No newline at end of file diff --git a/etc/skeleton/feed/pending/README.md b/etc/skeleton/feed/pending/README.md new file mode 100644 index 0000000..cf419f0 --- /dev/null +++ b/etc/skeleton/feed/pending/README.md @@ -0,0 +1,4 @@ +# Fairpost Pending sources + +A source is located here if all if its posts are unscheduled. +Fairpost will manage these sources; dont edit them here. \ No newline at end of file diff --git a/src/mappers/SourceMapper.ts b/src/mappers/SourceMapper.ts index a0d9454..c989346 100644 --- a/src/mappers/SourceMapper.ts +++ b/src/mappers/SourceMapper.ts @@ -36,6 +36,12 @@ export default class SourceMapper extends AbstractMapper { get: ["manageSources"], set: ["none"], }, + status: { + type: "string", + label: "Status", + get: ["manageSources"], + set: ["none"], + }, files: { type: "json", label: "Files", @@ -78,6 +84,9 @@ export default class SourceMapper extends AbstractMapper { case "path": dto[field] = this.source.path; break; + case "status": + dto[field] = this.source.status; + break; case "files": dto[field] = await this.source.getFiles(); break; diff --git a/src/models/Feed.ts b/src/models/Feed.ts index 1be0590..45573b9 100644 --- a/src/models/Feed.ts +++ b/src/models/Feed.ts @@ -8,7 +8,8 @@ import { basename } from "path"; * Feed - the sources handler of fairpost * * The feed is a container of sources. The sources - * path is set by USER_FEEDPATH. Every dir in there, + * path is set by USER_FEEDPATH. In it are subfolder + * for every source status. Every dir in those subfolders, * if not starting with _ or ., is a source. * * Every source can be prepared to become a post @@ -19,7 +20,9 @@ export default class Feed { path: string = ""; user: User; cache: { [id: string]: Source } = {}; - allCached: boolean = false; + allCached: { + [status in SourceStatus]?: boolean; + } = {}; mapper: FeedMapper; constructor(user: User) { @@ -29,6 +32,12 @@ export default class Feed { this.mapper = new FeedMapper(this); } + clearCache() { + this.user.log.trace("Feed", "clearCache"); + this.cache = {}; + this.allCached = {}; + } + /** * Get a report for this feed. This is * part of the user report, which is updated @@ -36,18 +45,18 @@ export default class Feed { * @returns a report for this feed */ async getReport() { - // TODO check cache first + // TODO check report cache first const sources = { [SourceStatus.UNKNOWN]: 0, [SourceStatus.INCOMING]: 0, - [SourceStatus.PREPARED]: 0, - [SourceStatus.PROCESSING]: 0, - [SourceStatus.PROCESSED]: 0, + [SourceStatus.PENDING]: 0, + [SourceStatus.ACTIVE]: 0, + [SourceStatus.DONE]: 0, [SourceStatus.ARCHIVED]: 0, }; - const allSources = await this.getAllSources(); + const allSources = await this.getSources(); for (const source of allSources) { - const status = await source.getStatus(); + const status = source.status; sources[status] = sources[status] + 1; } @@ -69,58 +78,100 @@ export default class Feed { /** * Get all sources + * @param sourceIds array of ids of source you want to get + * @param status optional status of the sources you want to get * @returns all source in the feed */ - async getAllSources(): Promise { - this.user.log.trace("Feed", "getAllSources"); - if (this.allCached) { - return Object.values(this.cache); - } - if (!(await this.user.files.exists(this.path))) { - this.user.log.info("creating dir " + this.path); - await this.user.files.mkdir(this.path); + async getSources( + sourceIds?: string[], + status?: SourceStatus, + ): Promise { + this.user.log.trace("Feed", "getSources", sourceIds ?? "", status ?? ""); + if (!sourceIds || !sourceIds.length) { + if (!status) { + // requesting all sources + if (!(await this.user.files.exists(this.path))) { + this.user.log.info("creating dir " + this.path); + await this.user.files.mkdir(this.path); + } + await Promise.all( + Object.values(SourceStatus).map((status) => + this.getSources([], status), + ), + ); + // should all be in the cache now + this.user.log.trace( + "found " + Object.keys(this.cache).length + " sources", + ); + return Object.values(this.cache); + } else { + // requesting sources with a specific status + if (this.allCached[status]) { + return Object.values(this.cache).filter( + (source) => source.status === status, + ); + } + const statusPath = this.path + "/" + status; + if (!(await this.user.files.exists(statusPath))) { + return []; + } + const sources: Source[] = []; + const files = this.user.files.list(statusPath).filter((entry) => { + if (entry.type === "file" || entry.isFile) return false; + const filename = basename(entry.path); + if (filename.startsWith("_")) return false; + if (filename.startsWith(".")) return false; + return true; + }); + for await (const file of files) { + const source = await Source.getSource(this, basename(file.path)); + this.cache[source.id] = source; + sources.push(source); + } + this.allCached[status] = true; + this.user.log.trace( + "found " + sources.length + " sources of status " + status, + ); + return sources; + } + } else { + // requesting sources with specific ids and optionally status + const sources: Source[] = []; + for (const sourceId of sourceIds) { + if (sourceId in this.cache) { + sources.push(this.cache[sourceId]); + } else { + const source = await Source.getSource(this, sourceId); + this.cache[source.id] = source; + sources.push(source); + } + } + this.user.log.trace("found " + sources.length + " sources"); + if (!status) { + return sources; + } + const filteredSources = sources.filter( + (source) => source.status === status, + ); + this.user.log.trace( + "found " + filteredSources.length + " sources of status " + status, + ); + return filteredSources; } - const files = this.user.files.list(this.path).filter((entry) => { - if (entry.type === "file" || entry.isFile) return false; - const filename = basename(entry.path); - if (filename.startsWith("_")) return false; - if (filename.startsWith(".")) return false; - return true; - }); - for await (const file of files) { - const source = await Source.getSource(this, basename(file.path)); - this.cache[source.id] = source; - } - this.allCached = true; - return Object.values(this.cache); } /** * Get one source - * @param path - path to a single source + * @param id - id of a single source * @returns the given source object */ - async getSource(path: string): Promise { - this.user.log.trace("Feed", "getSource", path); - const sourceId = this.getSourceId(path); - if (sourceId in this.cache) { - return this.cache[sourceId]; + async getSource(id: string): Promise { + this.user.log.trace("Feed", "getSource", id); + if (id in this.cache) { + return this.cache[id]; } - const source = await Source.getSource(this, path); + const source = await Source.getSource(this, id); this.cache[source.id] = source; return source; } - - /** - * Get multiple sources - * @param paths - paths to multiple sources - * @returns the given source objects - */ - async getSources(paths?: string[]): Promise { - this.user.log.trace("Feed", "getSources", paths); - if (!paths || !paths.length) { - return await this.getAllSources(); - } - return Promise.all(paths.map((path) => this.getSource(path))); - } } diff --git a/src/models/Platform.ts b/src/models/Platform.ts index 50ba3a0..89f0c80 100644 --- a/src/models/Platform.ts +++ b/src/models/Platform.ts @@ -1,7 +1,7 @@ import * as pluginClasses from "../plugins/index.ts"; import { PlatformId } from "../platforms/index.ts"; import PlatformMapper from "../mappers/PlatformMapper.ts"; -import { FieldMapping, PostStatus } from "../types/index.ts"; +import { FieldMapping, PostStatus, SourceStatus } from "../types/index.ts"; import Source from "./Source.ts"; import Plugin from "./Plugin.ts"; @@ -141,7 +141,7 @@ export default class Platform { */ async getPost(source: Source): Promise { - this.user.log.trace(this.id, "getPost", this.id, source.id); + this.user.log.trace(this.id, "getPost", source.id); const postId = this.getPostId(source); if (!(postId in this.cache)) { @@ -161,7 +161,7 @@ export default class Platform { this.user.log.trace(this.id, "getPosts"); const posts: Post[] = []; if (!sources) { - sources = await this.user.getFeed().getAllSources(); + sources = await this.user.getFeed().getSources(); } for (const source of sources) { try { @@ -177,13 +177,22 @@ export default class Platform { } /** - * Get last published post for a platform + * Get last published post for a platform from active or done sources + * Once a source is archived, posts there wil be ignored * @returns the above post or none */ async getLastPost(): Promise { this.user.log.trace(this.id, "getLastPost"); let lastPost: Post | undefined = undefined; - const posts = await this.getPosts(undefined, PostStatus.PUBLISHED); + const sources = [ + ...(await this.user.getFeed().getSources([], SourceStatus.DONE)), + ...(await this.user.getFeed().getSources([], SourceStatus.ACTIVE)), + ]; + if (!sources.length) { + this.user.log.trace(this.id, "getLastPost", "No active or done sources"); + return undefined; + } + const posts = await this.getPosts(sources, PostStatus.PUBLISHED); for (const post of posts) { if (post.published) { if ( @@ -210,7 +219,7 @@ export default class Platform { async getDuePost(sources: Source[]): Promise { const now = new Date(); for (const source of sources) { - const post = await this.getPost(source); + const post = await this.getPost(source); // TODO catch err if (post && post.status === PostStatus.SCHEDULED) { // some janitor checks if (!post.scheduled) { @@ -218,7 +227,7 @@ export default class Platform { "Found scheduled post without date. Unscheduling post.", post.id, ); - post.setStatus(PostStatus.UNSCHEDULED); + post.status = PostStatus.UNSCHEDULED; post.save(); continue; } @@ -227,7 +236,7 @@ export default class Platform { "Found scheduled post marked skip. Unscheduling post.", post.id, ); - post.setStatus(PostStatus.UNSCHEDULED); + post.status = PostStatus.UNSCHEDULED; post.save(); continue; } @@ -236,7 +245,7 @@ export default class Platform { "Found scheduled post previously published. Marking published.", post.id, ); - post.setStatus(PostStatus.PUBLISHED); + post.status = PostStatus.PUBLISHED; post.save(); continue; } @@ -295,8 +304,8 @@ export default class Platform { await post.prepare(false); } catch { post = await Post.getPost(this, source, false); - await post.prepare(true); this.cache[post.id] = post; + await post.prepare(true); } if (save) { await post.save(); @@ -340,27 +349,52 @@ export default class Platform { sources?: Source[], ): Promise { this.user.log.trace(this.id, "scheduleNextPost"); + + if (sources && !sources.length) { + this.user.log.trace(this.id, "scheduleNextPost", "No sources given"); + return undefined; + } if (!sources) { - sources = await this.user.getFeed().getAllSources(); + sources = [ + ...(await this.user.getFeed().getSources([], SourceStatus.ACTIVE)), + ...(await this.user.getFeed().getSources([], SourceStatus.PENDING)), + ]; + } + if (!sources.length) { + this.user.log.trace( + this.id, + "scheduleNextPost", + "No sources active or pending", + ); + return undefined; } - const scheduledPosts = await this.getPosts(sources, PostStatus.SCHEDULED); + const posts = await this.getPosts(sources); + + const scheduledPosts = posts.filter( + (post) => post.status === PostStatus.SCHEDULED, + ); if (scheduledPosts.length) { - this.user.log.trace(this.id, "scheduleNextPost", "Already scheduled"); + this.user.log.trace(this.id, "scheduleNextPost", "Already one scheduled"); return scheduledPosts[0]; } - const nextDate = date ? date : await this.getNextPostDate(); - for (const source of sources) { - const post = await this.getPost(source); - if ( - post && - post.valid && - !post.skip && - post.status === PostStatus.UNSCHEDULED - ) { - post.schedule(nextDate); - return post; - } + const candidatePosts = posts.filter( + (post) => + post.status === PostStatus.UNSCHEDULED && post.valid && !post.skip, + ); + + if (candidatePosts.length) { + const nextDate = date ? date : await this.getNextPostDate(); + const post = candidatePosts[0]; + this.user.log.trace( + this.id, + "scheduleNextPost", + "Scheduling post", + post.id, + ); + await post.schedule(nextDate); + return post; } + this.user.log.trace( this.id, "scheduleNextPost", diff --git a/src/models/Post.ts b/src/models/Post.ts index 770cc8b..21a24fc 100644 --- a/src/models/Post.ts +++ b/src/models/Post.ts @@ -40,6 +40,8 @@ export default class Post { remoteId?: string; mapper: PostMapper; + private originalStatus: PostStatus = PostStatus.UNKNOWN; + /** * Dont call the constructor yourself; * instead, call `await Post.getPost()` @@ -90,89 +92,72 @@ export default class Post { post.scheduled = post.scheduled ? new Date(post.scheduled) : undefined; post.published = post.published ? new Date(post.published) : undefined; post.ignoreFiles = post.ignoreFiles ?? []; + post.originalStatus = post.status; } return post; } /** - * Get the post status - * @returns the post status + * Save this post to disk */ - getStatus(): PostStatus { - return this.status; - } - /** - * Set the post status - this also updates the - * source status and the cached user report - * @param status - */ - async setStatus(status: PostStatus) { - this.platform.user.log.trace("Post", "setStatus", status); + async save() { + this.platform.user.log.trace("Post", this.id, "save"); - const originalPostStatus = this.status; - if (originalPostStatus !== status) { - // update the status - this.status = status; + // save the post json - // update the user report + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data = { ...this } as { [key: string]: any }; + delete data.source; + delete data.platform; + delete data.mapper; + await this.platform.user.files.write( + this.platform.getPostFilePath(this.source), + JSON.stringify(data, null, "\t"), + ); + + // update source status and the report if necessary + + if (this.originalStatus !== this.status) { + // update the source status if necessary + // note, this may *move* the source and all posts + const originalSourceStatus = this.source.status; + const newSourceStatus = await this.source.updateStatus(); + + // update the users report const report = await this.platform.user.getReport(); if (report.platforms[this.platform.id]) { - if (!report.platforms[this.platform.id]?.count[originalPostStatus]) { - report.platforms[this.platform.id]!.count[originalPostStatus] = 1; + if (!report.platforms[this.platform.id]?.count[this.originalStatus]) { + report.platforms[this.platform.id]!.count[this.originalStatus] = 1; } - if (!report.platforms[this.platform.id]?.count[status]) { - report.platforms[this.platform.id]!.count[status] = 0; + if (!report.platforms[this.platform.id]?.count[this.status]) { + report.platforms[this.platform.id]!.count[this.status] = 0; } - report.platforms[this.platform.id]!.count[originalPostStatus]!--; - report.platforms[this.platform.id]!.count[status]!++; - - // check if the source status is updated - // note, this may *move* the source and all posts - const orginalSourceStatus = await this.source.getStatus(); - const newSourceStatus = await this.source.updateStatus(); - - // update the user report if needed - if (orginalSourceStatus !== newSourceStatus) { - if (!report.feed.count[orginalSourceStatus]) { - report.feed.count[orginalSourceStatus] = 1; + report.platforms[this.platform.id]!.count[this.originalStatus]!--; + report.platforms[this.platform.id]!.count[this.status]!++; + + if (originalSourceStatus !== newSourceStatus) { + if (!report.feed.count[originalSourceStatus]) { + report.feed.count[originalSourceStatus] = 1; } if (!report.feed.count[newSourceStatus]) { report.feed.count[newSourceStatus] = 0; } - report.feed.count[orginalSourceStatus]!--; + report.feed.count[originalSourceStatus]!--; report.feed.count[newSourceStatus]!++; } - // save the report - await this.platform.user.putReport(report); this.platform.user.log.trace( "Post", - "setStatus", + this.id, + "save", "updated user report", ); } } } - /** - * Save this post to disk - */ - - async save() { - this.platform.user.log.trace("Post", "save"); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const data = { ...this } as { [key: string]: any }; - delete data.source; - delete data.platform; - delete data.mapper; - await this.platform.user.files.write( - this.platform.getPostFilePath(this.source), - JSON.stringify(data, null, "\t"), - ); - } - /** * Prepare this post * @@ -271,10 +256,10 @@ export default class Post { } if (this.status === PostStatus.UNKNOWN) { - await this.setStatus(PostStatus.UNSCHEDULED); + this.status = PostStatus.UNSCHEDULED; } if (this.status === PostStatus.FAILED) { - await this.setStatus(PostStatus.UNSCHEDULED); + this.status = PostStatus.UNSCHEDULED; } // done @@ -299,7 +284,7 @@ export default class Post { this.platform.user.log.warn("Rescheduling post"); } this.scheduled = date; - await this.setStatus(PostStatus.SCHEDULED); + this.status = PostStatus.SCHEDULED; await this.save(); } @@ -687,10 +672,10 @@ export default class Post { if (!result.error) { this.remoteId = remoteId; this.link = link; - await this.setStatus(PostStatus.PUBLISHED); + this.status = PostStatus.PUBLISHED; this.published = new Date(); } else { - await this.setStatus(PostStatus.FAILED); + this.status = PostStatus.FAILED; } } diff --git a/src/models/Source.ts b/src/models/Source.ts index b2f0d63..e9f9b55 100644 --- a/src/models/Source.ts +++ b/src/models/Source.ts @@ -1,6 +1,7 @@ import { basename, extname } from "path"; import sharp from "sharp"; +import { dirname } from "path"; import Feed from "./Feed.ts"; import { SourceStatus, @@ -27,6 +28,7 @@ export default class Source { feed: Feed; id: string; path: string; + status: SourceStatus; files?: FileInfo[]; mapper: SourceMapper; @@ -40,6 +42,7 @@ export default class Source { this.feed = feed; this.id = this.feed.getSourceId(path); this.path = path; + this.status = this.getStatus(); this.mapper = new SourceMapper(this); } @@ -48,41 +51,111 @@ export default class Source { * * get a new source and do some async checks. * @param feed - the feed this source belongs to - * @param path - the path within that feed + * @param id - the id of the source * @returns new source object */ - public static async getSource(feed: Feed, path: string): Promise { - if (!(await feed.user.files.isDir(feed.path + "/" + path))) { - throw feed.user.log.error("getSource", "Not a valid source: " + path); + public static async getSource(feed: Feed, id: string): Promise { + for (const status of Object.values(SourceStatus)) { + const sourcePath = feed.path + "/" + status + "/" + id; + if (await feed.user.files.isDir(sourcePath)) { + return new Source(feed, sourcePath); + } } - return new Source(feed, feed.path + "/" + path); + throw feed.user.log.error("getSource", "Not a valid source: " + id); } /** * Get the status of a source. * - * The status depends on the various statusses of the posts - * in the source. The path of the source depends on the status, - * and here we just check the path to see its current status. + * By definition, the status of a source is its location on disk. + * + * That location depends on the various statusses of the posts + * in the source. Here we just check the path to see its current status. * @returns {SourceStatus} - the status of the source */ - public async getStatus(): Promise { - // TODO + private getStatus(): SourceStatus { + const parent = dirname(this.path); + for (const status of Object.values(SourceStatus)) { + if (parent.endsWith(status)) { + return status; + } + } return SourceStatus.UNKNOWN; } /** - * Update the status of a source. + * Check/Update the status of a source. * * The status of the source depends on the various statusses - * of the posts in the source. Post.setStatus calls this method. + * of the posts in the source. Post.save() calls this method. * The path of the source depends on the status, so if * it is updated source may move to a new location. * @returns {SourceStatus} - the new status of the source */ public async updateStatus(): Promise { - // TODO - return SourceStatus.UNKNOWN; + this.feed.user.log.trace("Source", "updateStatus"); + + // check all posts to check their status + const orgStatus = this.status; + let newStatus: SourceStatus | undefined = undefined; + + if (this.status === SourceStatus.ARCHIVED) { + newStatus = SourceStatus.ARCHIVED; + } else { + const posts = await this.getPosts(); + if (posts.length === 0) { + newStatus = SourceStatus.INCOMING; + } else if ( + posts.every( + (post: Post) => + post.status === PostStatus.PUBLISHED || + post.status === PostStatus.CANCELED, + ) + ) { + newStatus = SourceStatus.DONE; + } else if ( + posts.every((post: Post) => post.status === PostStatus.UNSCHEDULED) + ) { + newStatus = SourceStatus.PENDING; + } else if ( + posts.every((post: Post) => post.status === PostStatus.UNKNOWN) + ) { + newStatus = SourceStatus.UNKNOWN; + } + if (newStatus === undefined) { + newStatus = SourceStatus.ACTIVE; + } + } + if (orgStatus === newStatus) { + this.feed.user.log.trace(this.id, "updateStatus", "no change"); + return this.status; + } + + // if our status changed, + // move this source to the new location + this.feed.user.log.trace(this.id, "updateStatus", orgStatus, newStatus); + const newPath = this.feed.path + "/" + newStatus + "/" + this.id; + if (await this.feed.user.files.exists(newPath)) { + this.feed.user.log.error( + this.id, + "updateStatus", + "source already exists: " + newPath, + ); + return this.status; + } + + // move directory + const log = await this.feed.user.files.moveDir(this.path, newPath); + for (const msg of log) { + this.feed.user.log.trace(msg); + } + + // update my status and clear feed cache + this.path = newPath; + this.status = newStatus; + this.feed.clearCache(); + + return this.status; } /** @@ -152,7 +225,7 @@ export default class Source { */ public async getPost(platform: Platform): Promise { - this.feed.user.log.trace(this.id, "getPost", this.id, platform.id); + this.feed.user.log.trace(this.id, "getPost", platform.id); return await platform.getPost(this); } diff --git a/src/models/User.ts b/src/models/User.ts index 8ddcc63..3b90c6e 100644 --- a/src/models/User.ts +++ b/src/models/User.ts @@ -123,7 +123,6 @@ export default class User { ); } const globalfs = new GlobalFs(); - const log = [] as string[]; if (!process.env.FAIRPOST_USER_HOMEDIR) { throw new Error("FAIRPOST_USER_HOMEDIR not set in env"); @@ -133,23 +132,10 @@ export default class User { if (await globalfs.exists(dst)) { throw new Error("Homedir already exists: " + dst); } - const listing = await globalfs.list(src, { deep: true }).toArray(); - for await (const entry of listing) { - if (entry.type === "directory" || entry.isDirectory) { - const entrydst = entry.path.replace("etc/skeleton", dst); - log.push("creating dir " + entrydst); - await globalfs.mkdir(entrydst); - } - } - for await (const entry of listing) { - if (entry.type === "file" || entry.isFile) { - const entrydst = entry.path.replace("etc/skeleton", dst); - log.push("copying file " + entry.path + " -> " + entrydst); - await globalfs.copy(entry.path, entrydst); - } - } + const log = await globalfs.copyDir(src, dst); const user = await User.getUser(newUserId); + user.data.set("settings", "FEED_PLATFORMS", ""); await user.data.save(); for (const msg of log) { @@ -257,7 +243,7 @@ export default class User { * @returns platforms given by ids */ getPlatforms(platformIds?: PlatformId[]): Platform[] { - this.log.trace("User", "getPlatforms", platformIds); + this.log.trace("User", "getPlatforms", platformIds ?? ""); if (this.platforms === undefined) { this.loadPlatforms(); } diff --git a/src/models/User/UserFiles.ts b/src/models/User/UserFiles.ts index 56b572c..a19ea20 100644 --- a/src/models/User/UserFiles.ts +++ b/src/models/User/UserFiles.ts @@ -82,10 +82,103 @@ export default class UserFiles { public async write(path: string, contents: FileContents): Promise { return await this.storage.write(path, contents); } - public async copy(src: string, dst: string): Promise { + public async copy( + src: string, + dst: string, + dontCheckType = false, + ): Promise { + if (!dontCheckType && (await this.isDir(src))) { + this.copyDir(src, dst); + } return await this.storage.copyFile(src, dst); } + public async copyDir( + sourceDir: string, + destinationDir: string, + ): Promise { + const log: string[] = []; + const createDirectoryPromises = []; + for await (const entry of this.list(sourceDir, { deep: true })) { + if (entry.type === "directory" || entry.isDirectory) { + const sourcePath = entry.path; + const relativePath = sourcePath + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("creating dir " + destinationPath); + createDirectoryPromises.push(this.mkdir(destinationPath)); + } + } + await Promise.all(createDirectoryPromises); + + const copyFilePromises = []; + for await (const entry of this.list(sourceDir, { deep: true })) { + if (entry.type === "file" || entry.isFile) { + const sourcePath = entry.path; + const relativePath = sourcePath + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("copying file " + entry.path + " -> " + destinationPath); + copyFilePromises.push(this.copy(entry.path, destinationPath, true)); + } + } + await Promise.all(copyFilePromises); + + return log; + } + + public async move( + src: string, + dst: string, + dontCheckType = false, + ): Promise { + if (!dontCheckType && (await this.isDir(src))) { + this.moveDir(src, dst); + } + return await this.storage.moveFile(src, dst); + } + + public async moveDir( + sourceDir: string, + destinationDir: string, + ): Promise { + const log: string[] = []; + + const createDirectoryPromises = []; + for await (const item of this.list(sourceDir, { deep: true })) { + if (item.isDirectory) { + const relativePath = item.path + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("creating dir " + destinationPath); + createDirectoryPromises.push( + this.storage.createDirectory(destinationPath), + ); + } + } + await Promise.all(createDirectoryPromises); + + const moveFilePromises = []; + for await (const item of this.list(sourceDir, { deep: true })) { + if (item.isFile) { + const relativePath = item.path + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("moving file " + item.path + "," + destinationPath); + moveFilePromises.push(this.move(item.path, destinationPath, true)); + } + } + await Promise.all(moveFilePromises); + + log.push("deleting dir " + sourceDir); + await this.storage.deleteDirectory(sourceDir); + return log; + } + public async getMimeType(path: string): Promise { return await this.storage.mimeType(path); } diff --git a/src/platforms/Facebook/Facebook.ts b/src/platforms/Facebook/Facebook.ts index 9cc3b1b..9e28615 100644 --- a/src/platforms/Facebook/Facebook.ts +++ b/src/platforms/Facebook/Facebook.ts @@ -91,7 +91,7 @@ export default class Facebook extends Platform { for (const plugin of plugins) { await plugin.process(post); } - post.save(); + await post.save(); } return post; } diff --git a/src/platforms/Instagram/Instagram.ts b/src/platforms/Instagram/Instagram.ts index c888293..10f2a8f 100644 --- a/src/platforms/Instagram/Instagram.ts +++ b/src/platforms/Instagram/Instagram.ts @@ -106,7 +106,7 @@ export default class Instagram extends Platform { } } - post.save(); + await post.save(); } return post; } diff --git a/src/platforms/LinkedIn/LinkedIn.ts b/src/platforms/LinkedIn/LinkedIn.ts index ac17e93..56996ce 100644 --- a/src/platforms/LinkedIn/LinkedIn.ts +++ b/src/platforms/LinkedIn/LinkedIn.ts @@ -92,7 +92,7 @@ export default class LinkedIn extends Platform { for (const plugin of plugins) { await plugin.process(post); } - post.save(); + await post.save(); } return post; } diff --git a/src/platforms/Reddit/Reddit.ts b/src/platforms/Reddit/Reddit.ts index c402dca..96fa067 100644 --- a/src/platforms/Reddit/Reddit.ts +++ b/src/platforms/Reddit/Reddit.ts @@ -141,7 +141,7 @@ export default class Reddit extends Platform { if (videoposter) { await post.addFile(videoposter); } - post.save(); + await post.save(); } return post; } diff --git a/src/platforms/Twitter/Twitter.ts b/src/platforms/Twitter/Twitter.ts index b0dfc98..773a0e8 100644 --- a/src/platforms/Twitter/Twitter.ts +++ b/src/platforms/Twitter/Twitter.ts @@ -134,7 +134,7 @@ export default class Twitter extends Platform { this.user.log.warn("Twitter post has no body"); post.valid = false; } - post.save(); + await post.save(); } return post; } diff --git a/src/platforms/YouTube/YouTube.ts b/src/platforms/YouTube/YouTube.ts index 12f6432..816ceb6 100644 --- a/src/platforms/YouTube/YouTube.ts +++ b/src/platforms/YouTube/YouTube.ts @@ -92,7 +92,7 @@ export default class YouTube extends Platform { for (const plugin of plugins) { await plugin.process(post); } - post.save(); + await post.save(); } return post; } diff --git a/src/services/Fairpost.ts b/src/services/Fairpost.ts index c018133..987bde6 100644 --- a/src/services/Fairpost.ts +++ b/src/services/Fairpost.ts @@ -15,6 +15,8 @@ import { PostDto, SourceDto, UserDto, + SourceStatus, + PostStatus, } from "../types/index.ts"; import Post from "../models/Post.ts"; @@ -355,10 +357,19 @@ class Fairpost { throw new Error("Missing permissions for command " + command); } if (!user) { - throw new Error("user is required for command " + command); + throw new Error("User is required for command " + command); + } + if ( + args.status && + !Object.values(SourceStatus).includes(args.status as SourceStatus) + ) { + throw new Error("Incorrect status " + args.status); } const feed = user.getFeed(); - const sources = await feed.getSources(args.sources); + const sources = await feed.getSources( + args.sources, + args.status as SourceStatus, + ); output = await Promise.all( sources.map((source) => source.mapper.getDto(operator)), ); @@ -396,8 +407,15 @@ class Fairpost { throw new Error("Missing permissions for command " + command); } if (!user) { - throw new Error("user is required for command " + command); + throw new Error("User is required for command " + command); } + if ( + args.status && + !Object.values(PostStatus).includes(args.status as PostStatus) + ) { + throw new Error("Incorrect status " + args.status); + } + if (!args.platforms && args.platform) { args.platforms = [args.platform]; } @@ -409,7 +427,9 @@ class Fairpost { const sources = await feed.getSources(args.sources); const posts = [] as Post[]; for (const platform of platforms) { - posts.push(...(await platform.getPosts(sources, args.status))); + posts.push( + ...(await platform.getPosts(sources, args.status as PostStatus)), + ); } output = await Promise.all( posts.map((p) => p.mapper.getDto(operator)), @@ -456,28 +476,35 @@ class Fairpost { args.sources = [args.source]; } const feed = user.getFeed(); - const sources = await feed.getSources(args.sources); - const platforms = user.getPlatforms(args.platforms); - output = {} as { [id in PlatformId]?: CombinedResult[] }; - for (const platform of platforms) { - for (const source of sources) { - if (!output[platform.id]) { - output[platform.id] = []; - } - try { - const post = await platform.preparePost(source); - (output[platform.id] as CombinedResult[]).push({ - success: true, - result: await post.mapper.getDto(operator), - }); - } catch (e) { - user.log.error("Fairpost", "preparePosts", e); - (output[platform.id] as CombinedResult[]).push({ - success: false, - message: e instanceof Error ? e.message : JSON.stringify(e), - }); + const sources = await feed.getSources( + args.sources, + SourceStatus.INCOMING, + ); + if (sources.length) { + const platforms = user.getPlatforms(args.platforms); + output = {} as { [id in PlatformId]?: CombinedResult[] }; + for (const platform of platforms) { + for (const source of sources) { + if (!output[platform.id]) { + output[platform.id] = []; + } + try { + const post = await platform.preparePost(source); + (output[platform.id] as CombinedResult[]).push({ + success: true, + result: await post.mapper.getDto(operator), + }); + } catch (e) { + user.log.error("Fairpost", "preparePosts", e); + (output[platform.id] as CombinedResult[]).push({ + success: false, + message: e instanceof Error ? e.message : JSON.stringify(e), + }); + } } } + } else { + output = { success: true, message: "No post left to prepare" }; } break; } @@ -642,9 +669,9 @@ class Fairpost { for (const platform of platforms) { try { const post = await platform.getPost(source); - await post.publish(!!args.dryrun); + const success = await post.publish(!!args.dryrun); output[platform.id] = { - success: await post.publish(!!args.dryrun), + success: success, dryrun: !!args.dryrun, result: post.link, }; @@ -674,7 +701,9 @@ class Fairpost { args.sources = [args.source]; } const feed = user.getFeed(); - const sources = await feed.getSources(args.sources); + const sources = args.sources?.length + ? await feed.getSources(args.sources) + : undefined; const platforms = user.getPlatforms(args.platforms); const posts = [] as Post[]; for (const platform of platforms) { @@ -697,7 +726,10 @@ class Fairpost { throw new Error("user is required for command " + command); } const feed = user.getFeed(); - const sources = await feed.getSources(args.sources); + const sources = await feed.getSources( + args.sources, + SourceStatus.ACTIVE, + ); const platforms = user.getPlatforms(args.platforms); output = {} as { [id in PlatformId]: CombinedResult }; for (const platform of platforms) { @@ -710,17 +742,20 @@ class Fairpost { output[platform.id] = { success: true, result: post.link, + dryrun: !!args.dryrun, }; } else { output[platform.id] = { success: true, message: "No posts due", + dryrun: !!args.dryrun, }; } } catch (e) { output[platform.id] = { success: false, message: e instanceof Error ? e.message : JSON.stringify(e), + dryrun: !!args.dryrun, }; } } diff --git a/src/services/GlobalFs.ts b/src/services/GlobalFs.ts index 842ca73..eaaf81b 100644 --- a/src/services/GlobalFs.ts +++ b/src/services/GlobalFs.ts @@ -72,10 +72,109 @@ export default class GlobalFs { public async write(path: string, contents: FileContents): Promise { return await this.storage.write(path, contents); } - public async copy(src: string, dst: string): Promise { + public async copy( + src: string, + dst: string, + dontCheckType = false, + ): Promise { + if (!dontCheckType && (await this.isDir(src))) { + this.copyDir(src, dst); + } return await this.storage.copyFile(src, dst); } + public async copyDir( + sourceDir: string, + destinationDir: string, + ): Promise { + const log: string[] = []; + const createDirectoryPromises = []; + for await (const entry of this.list(sourceDir, { deep: true })) { + if (entry.type === "directory" || entry.isDirectory) { + const sourcePath = entry.path; + const relativePath = sourcePath + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("creating dir " + destinationPath); + createDirectoryPromises.push(this.mkdir(destinationPath)); + } + } + await Promise.all(createDirectoryPromises); + + const copyFilePromises = []; + for await (const entry of this.list(sourceDir, { deep: true })) { + if (entry.type === "file" || entry.isFile) { + const sourcePath = entry.path; + const relativePath = sourcePath + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("copying file " + entry.path + " -> " + destinationPath); + copyFilePromises.push(this.copy(entry.path, destinationPath, true)); + } + } + await Promise.all(copyFilePromises); + + return log; + } + + public async move( + src: string, + dst: string, + dontCheckType = false, + ): Promise { + if (dontCheckType && (await this.isDir(src))) { + this.moveDir(src, dst); + } + return await this.storage.moveFile(src, dst); + } + + public async moveDir( + sourceDir: string, + destinationDir: string, + ): Promise { + const log: string[] = []; + + // List all items in the source directory recursively + const directoryListing = this.list(sourceDir, { deep: true }); + + // 1. Create destination directories in parallel + const createDirectoryPromises = []; + for await (const item of directoryListing) { + if (item.isDirectory) { + const relativePath = item.path + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("creating dir " + destinationPath); + createDirectoryPromises.push( + this.storage.createDirectory(destinationPath), + ); + } + } + await Promise.all(createDirectoryPromises); + + // 2. Move files in parallel + const moveFilePromises = []; + for await (const item of directoryListing) { + if (item.isFile) { + const relativePath = item.path + .slice(sourceDir.length) + .replace(/^\/+/, ""); + const destinationPath = `${destinationDir}/${relativePath}`; + log.push("moving file " + item.path + "," + destinationPath); + moveFilePromises.push(this.move(item.path, destinationPath, true)); + } + } + await Promise.all(moveFilePromises); + + // 3. Delete the source directory + log.push("deleting dir " + sourceDir); + await this.storage.deleteDirectory(sourceDir); + return log; + } + public async getMimeType(path: string): Promise { return await this.storage.mimeType(path); } diff --git a/src/types/CommandArguments.ts b/src/types/CommandArguments.ts index 92c2444..885b07d 100644 --- a/src/types/CommandArguments.ts +++ b/src/types/CommandArguments.ts @@ -1,5 +1,5 @@ import { PlatformId } from "../platforms/index.ts"; -import { PostStatus } from "./index.ts"; +import { PostStatus, SourceStatus } from "./index.ts"; /** * CommandArguments are the arguments that can be passed @@ -14,5 +14,5 @@ export default interface CommandArguments { sources?: string[]; source?: string; date?: Date; - status?: PostStatus; + status?: PostStatus | SourceStatus; } diff --git a/src/types/SourceDto.ts b/src/types/SourceDto.ts index bf15763..a3bfe3e 100644 --- a/src/types/SourceDto.ts +++ b/src/types/SourceDto.ts @@ -1,9 +1,10 @@ -import { type FileInfo } from "./index.ts"; +import { type FileInfo, SourceStatus } from "./index.ts"; export default interface SourceDto { model: string; id: string; user_id: string; feed_id?: string; path?: string; + status?: SourceStatus; files?: FileInfo[]; } diff --git a/src/types/SourceStatus.ts b/src/types/SourceStatus.ts index c72438a..f22b9bd 100644 --- a/src/types/SourceStatus.ts +++ b/src/types/SourceStatus.ts @@ -1,9 +1,9 @@ enum SourceStatus { - UNKNOWN = "unknown", + PENDING = "pending", // all posts unscheduled or unknown + ACTIVE = "active", // mixed post statuses + DONE = "done", // all posts published or canceled INCOMING = "incoming", // no posts - PREPARED = "prepared", // all posts unscheduled - PROCESSING = "processing", // mixed post statuses - PROCESSED = "processed", // all posts published, canceled or failed ARCHIVED = "archived", + UNKNOWN = "unknown", } export default SourceStatus;