feat: Better support for SourceStatus

README.md

@@ -7,10 +7,16 @@ Fairpost helps you manage users social media feeds from a single
 entry point, using Node. It supports Facebook, Instagram, 
 Reddit, Twitter, YouTube and LinkedIn.
 
-A Feed is just a folder on disk, and all subfolders are Source Posts, 
-containing at least one text file (the post body) and 
-optionally images or video. The Source Post will be transformed
-into real posts for each connected platform.
+Fairpost behaves like a feed, not a calendar.
+By default, there is only one scheduled post for each 
+platform. Once it is published, the next post may
+be scheduled. 
+
+A Feed is just a bunch of folders on disk, and all subfolders 
+in each folder are Source Posts, containing at least one text 
+file (the post body) and optionally images or video. The Source 
+Post will be transformed into destination posts for each 
+connected platform.
 
 Fairpost is *opinionated*, meaning, it will decide
 how a Source Post with contents can best be presented
@@ -25,13 +31,13 @@ post on their behalf. This is usually done via an
 online (oauth) consent page in a webbrowser.
 
 Commonly, you would call this script every day or week
-for every user. Fairpost can then automatically **prepare** the folders,
-**schedule** the next post using a certain interval and 
-**publish** any post when it is due. All the user has to do is 
-add folders with content.
+for every user. Fairpost can then automatically **prepare** the posts
+from the sources, **schedule** the next post using a certain interval 
+and **publish** any post when it is due. All the user has to do is 
+add folders with content in the `incoming` folder.
 
 Or, if you prefer, you can manually publish one
-specific folder as posts on all supported and enabled 
+specific source as posts on all supported and enabled 
 platforms at once, or just one post on one platform,
 etcetera.
 
@@ -74,12 +80,24 @@ nano users/foobar/var/lib/storage.json
 ```
 
 ## Feed planning
+
+Inside each feed, there are folders for `incoming`,
+`pending`, `active`, `done` and `archived` sources. 
+The location of a source depends on the statusses of 
+the posts inside that source. New posts should be 
+added to 'incoming'; from there, Fairpost will manage 
+the sources location with the commands below.
+
 ### Prepare
 ```
 fairpost.js prepare-posts
 ```
 Sources need to be `prepared` (iow turned into posts)
 before they can be published to a platform. 
+`prepare-posts` will prepare allthe sources in the 
+`incoming` folder and on success, move the source 
+and its posts to the `pending` folder.
+
 Each platform, as defined in src/platforms, will 
 handle the folder contents by itself. It may
 decide to modify the media (eg, scale images) 
@@ -89,6 +107,7 @@ is youtube). Finally, it will add a json file
 describing the post for that platform in the 
 folder.
 
+
 ### Schedule
 ```
 fairpost.js schedule-next-post
@@ -101,12 +120,22 @@ By default the date will be `FAIRPOST_FEED_INTERVAL` days
 after the last post for that platform, or `now`, whichever 
 is latest.
 
+TODO
+Without arguments, `schedule-next-post` will select the 
+next post for each platform from the `pending` or `active` folders.
+If it wasn't already there, it will move the source and 
+its posts to the `active` folder.
+
 ### Publish
 ```
 fairpost.js publish-due-posts
 ```
 This will publish any scheduled posts that are past their due date.
 
+Without arguments, `publish-due-posts` will search for a 
+due post for each platform from the `active` folder.
+Once all posts from a source are published, it will move 
+the source and its posts to the `done` folder.
 
 ## Other commands
 

docs/NewPlatform.md

@@ -38,7 +38,7 @@ export default class FooBar extends Platform {
         const post = await super.preparePost(source);
         if (post) {
             // prepare your post here
-            post.save();
+            await post.save();
         }
         return post;
     }

etc/skeleton/feed/README.md

@@ -3,5 +3,9 @@
 This folder can actually be anywhere, the path
 can be defined in your `.env` file.
 
-Post your source posts here, each in a separate folder.
-Folder names starting with an underscore are ignored.
+Post your new source posts in the incoming folder, each in a separate folder.
+Folder names starting with an underscore or dot are ignored.
+
+Fairpost will manage these sources while processing
+the feed; they will move to other folders in the feed
+depending on the posts statusses.

etc/skeleton/feed/active/README.md

@@ -0,0 +1,7 @@
+# Fairpost Active sources
+
+A source is located here if all if its not incoming,
+pending, done or archived. Most likely, some posts here
+are scheduled.
+
+Fairpost will manage these sources; dont edit them here.

etc/skeleton/feed/archived/README.md

@@ -0,0 +1,6 @@
+# Fairpost Archived sources
+
+A source is located here if it is archived. Most likely,
+all posts here are either published or canceled.
+
+Fairpost will manage these sources; dont edit them here.

etc/skeleton/feed/done/README.md

@@ -0,0 +1,4 @@
+# Fairpost Done sources
+
+A source is located here if all if its posts are published or canceled.
+Fairpost will manage these sources; dont edit them here.

etc/skeleton/feed/incoming/README.md

@@ -0,0 +1,9 @@
+# Fairpost Incoming sources
+
+Add your new sources here. 
+
+Folder names starting with an underscore or dot are ignored.
+
+Fairpost will manage these sources while processing
+the feed; they will move to other folders in the feed
+depending on the posts statusses.

etc/skeleton/feed/pending/README.md

@@ -0,0 +1,4 @@
+# Fairpost Pending sources
+
+A source is located here if all if its posts are unscheduled.
+Fairpost will manage these sources; dont edit them here.

src/mappers/SourceMapper.ts

@@ -36,6 +36,12 @@ export default class SourceMapper extends AbstractMapper<SourceDto> {
       get: ["manageSources"],
       set: ["none"],
     },
+    status: {
+      type: "string",
+      label: "Status",
+      get: ["manageSources"],
+      set: ["none"],
+    },
     files: {
       type: "json",
       label: "Files",
@@ -78,6 +84,9 @@ export default class SourceMapper extends AbstractMapper<SourceDto> {
         case "path":
           dto[field] = this.source.path;
           break;
+        case "status":
+          dto[field] = this.source.status;
+          break;
         case "files":
           dto[field] = await this.source.getFiles();
           break;

src/models/Feed.ts

@@ -8,7 +8,8 @@ import { basename } from "path";
  * Feed - the sources handler of fairpost
  *
  * The feed is a container of sources. The sources
- * path is set by USER_FEEDPATH. Every dir in there,
+ * path is set by USER_FEEDPATH. In it are subfolder
+ * for every source status. Every dir in those subfolders,
  * if not starting with _ or ., is a source.
  *
  * Every source can be prepared to become a post
@@ -19,7 +20,9 @@ export default class Feed {
   path: string = "";
   user: User;
   cache: { [id: string]: Source } = {};
-  allCached: boolean = false;
+  allCached: {
+    [status in SourceStatus]?: boolean;
+  } = {};
   mapper: FeedMapper;
 
   constructor(user: User) {
@@ -29,25 +32,31 @@ export default class Feed {
     this.mapper = new FeedMapper(this);
   }
 
+  clearCache() {
+    this.user.log.trace("Feed", "clearCache");
+    this.cache = {};
+    this.allCached = {};
+  }
+
   /**
    * Get a report for this feed. This is
    * part of the user report, which is updated
    * as posts are processed and then cached
    * @returns a report for this feed
    */
   async getReport() {
-    // TODO check cache first
+    // TODO check report cache first
     const sources = {
       [SourceStatus.UNKNOWN]: 0,
       [SourceStatus.INCOMING]: 0,
-      [SourceStatus.PREPARED]: 0,
-      [SourceStatus.PROCESSING]: 0,
-      [SourceStatus.PROCESSED]: 0,
+      [SourceStatus.PENDING]: 0,
+      [SourceStatus.ACTIVE]: 0,
+      [SourceStatus.DONE]: 0,
       [SourceStatus.ARCHIVED]: 0,
     };
-    const allSources = await this.getAllSources();
+    const allSources = await this.getSources();
     for (const source of allSources) {
-      const status = await source.getStatus();
+      const status = source.status;
       sources[status] = sources[status] + 1;
     }
 
@@ -69,58 +78,100 @@ export default class Feed {
 
   /**
    * Get all sources
+   * @param sourceIds array of ids of source you want to get
+   * @param status optional status of the sources you want to get
    * @returns all source in the feed
    */
-  async getAllSources(): Promise<Source[]> {
-    this.user.log.trace("Feed", "getAllSources");
-    if (this.allCached) {
-      return Object.values(this.cache);
-    }
-    if (!(await this.user.files.exists(this.path))) {
-      this.user.log.info("creating dir " + this.path);
-      await this.user.files.mkdir(this.path);
+  async getSources(
+    sourceIds?: string[],
+    status?: SourceStatus,
+  ): Promise<Source[]> {
+    this.user.log.trace("Feed", "getSources", sourceIds ?? "", status ?? "");
+    if (!sourceIds || !sourceIds.length) {
+      if (!status) {
+        // requesting all sources
+        if (!(await this.user.files.exists(this.path))) {
+          this.user.log.info("creating dir " + this.path);
+          await this.user.files.mkdir(this.path);
+        }
+        await Promise.all(
+          Object.values(SourceStatus).map((status) =>
+            this.getSources([], status),
+          ),
+        );
+        // should all be in the cache now
+        this.user.log.trace(
+          "found " + Object.keys(this.cache).length + " sources",
+        );
+        return Object.values(this.cache);
+      } else {
+        // requesting sources with a specific status
+        if (this.allCached[status]) {
+          return Object.values(this.cache).filter(
+            (source) => source.status === status,
+          );
+        }
+        const statusPath = this.path + "/" + status;
+        if (!(await this.user.files.exists(statusPath))) {
+          return [];
+        }
+        const sources: Source[] = [];
+        const files = this.user.files.list(statusPath).filter((entry) => {
+          if (entry.type === "file" || entry.isFile) return false;
+          const filename = basename(entry.path);
+          if (filename.startsWith("_")) return false;
+          if (filename.startsWith(".")) return false;
+          return true;
+        });
+        for await (const file of files) {
+          const source = await Source.getSource(this, basename(file.path));
+          this.cache[source.id] = source;
+          sources.push(source);
+        }
+        this.allCached[status] = true;
+        this.user.log.trace(
+          "found " + sources.length + " sources of status " + status,
+        );
+        return sources;
+      }
+    } else {
+      // requesting sources with specific ids and optionally status
+      const sources: Source[] = [];
+      for (const sourceId of sourceIds) {
+        if (sourceId in this.cache) {
+          sources.push(this.cache[sourceId]);
+        } else {
+          const source = await Source.getSource(this, sourceId);
+          this.cache[source.id] = source;
+          sources.push(source);
+        }
+      }
+      this.user.log.trace("found " + sources.length + " sources");
+      if (!status) {
+        return sources;
+      }
+      const filteredSources = sources.filter(
+        (source) => source.status === status,
+      );
+      this.user.log.trace(
+        "found " + filteredSources.length + " sources of status " + status,
+      );
+      return filteredSources;
     }
-    const files = this.user.files.list(this.path).filter((entry) => {
-      if (entry.type === "file" || entry.isFile) return false;
-      const filename = basename(entry.path);
-      if (filename.startsWith("_")) return false;
-      if (filename.startsWith(".")) return false;
-      return true;
-    });
-    for await (const file of files) {
-      const source = await Source.getSource(this, basename(file.path));
-      this.cache[source.id] = source;
-    }
-    this.allCached = true;
-    return Object.values(this.cache);
   }
 
   /**
    * Get one source
-   * @param path - path to a single source
+   * @param id - id of a single source
    * @returns the given source object
    */
-  async getSource(path: string): Promise<Source> {
-    this.user.log.trace("Feed", "getSource", path);
-    const sourceId = this.getSourceId(path);
-    if (sourceId in this.cache) {
-      return this.cache[sourceId];
+  async getSource(id: string): Promise<Source> {
+    this.user.log.trace("Feed", "getSource", id);
+    if (id in this.cache) {
+      return this.cache[id];
     }
-    const source = await Source.getSource(this, path);
+    const source = await Source.getSource(this, id);
     this.cache[source.id] = source;
     return source;
   }
-
-  /**
-   * Get multiple sources
-   * @param paths - paths to multiple sources
-   * @returns the given source objects
-   */
-  async getSources(paths?: string[]): Promise<Source[]> {
-    this.user.log.trace("Feed", "getSources", paths);
-    if (!paths || !paths.length) {
-      return await this.getAllSources();
-    }
-    return Promise.all(paths.map((path) => this.getSource(path)));
-  }
 }