bookmarked pagination, including multi-shard

introduces a new BookmarkedCollection module with behavior similar to
PaginatedCollection in the simple case.

the primary advantage is that assigning to current_page (e.g. from the
:page parameter to paginate) expects a bookmark token value and
automatically deserializes into current_bookmark. the library client can
then use current_bookmark to skip forward in the collection, rather than
using (current_page - 1) * per_page as the number of items to skip. the
client then calls set_next_bookmark on the pager if there's more
results, and it automatically derives the bookmark for the next page and
serializes it into next_page, for use by Api.paginate, etc.

in addition to the PaginatedCollection.build analog, you can simply wrap
an existing scope to change it from something that will paginate by page
number into something that will paginate by bookmark.

finally, the key reason to use bookmarked pagination is to enable
composition of collections. you can merge multiple collections into one
collection which when paginated will pull results from each
subcollection, in order, to produce the page of results. you can also
concatenate multiple collections into one collection which when
paginated will exhaust the collections in order with seamless transition
from one to the next when a page spans both.

with collection merging available, you can paginate an association where
you'd like to use with_each_shard. one collection is created per shard,
and then they are merged together. this process is automated for you in
the BookmarkedCollection.with_each_shard method.

fixes CNVS-1169

Change-Id: Ib998eee53c33604cb6f7e338153428a157928a6d
Reviewed-on: https://gerrit.instructure.com/16039
Tested-by: Jenkins <jenkins@instructure.com>
Reviewed-by: Jacob Fugal <jacob@instructure.com>
QA-Review: Clare Hetherington <clare@instructure.com>

Author: lukfugl

app/controllers/groups_controller.rb

@@ -146,11 +146,14 @@ def unassigned_members
   # @returns [Group]
   def index
     return context_index if @context
-    @groups = @current_user.try(:current_groups)
-    # can't use #try, cause it's not defined on the association class,
-    # so it will try to load the association and call it on the resulting array'
-    @groups = @groups.with_each_shard({ :include => :group_category, :order => "groups.id ASC" }) if @groups
-    @groups ||= []
+    if @current_user
+      @groups = BookmarkedCollection.with_each_shard(
+        Group::Bookmarker,
+        @current_user.current_groups,
+        :include => :group_category)
+    else
+      @groups = []
+    end
     respond_to do |format|
       format.html {}
       format.json do

app/models/group.rb

@@ -528,4 +528,22 @@ def default_collection_name
   def associated_shards
     [Shard.default]
   end
+
+  class Bookmarker
+    def self.bookmark_for(group)
+      group.id
+    end
+
+    def self.validate(bookmark)
+      bookmark.is_a?(Fixnum)
+    end
+
+    def self.restrict_scope(scope, pager)
+      if bookmark = pager.current_bookmark
+        comparison = (pager.include_bookmark ? 'groups.id >= ?' : 'groups.id > ?')
+        scope = scope.scoped(:conditions => [comparison, bookmark])
+      end
+      scope.scoped(:order => "groups.id ASC")
+    end
+  end
 end

app/models/page_view.rb

@@ -189,13 +189,25 @@ def update_without_callbacks
     { :conditions => {:user_id => users} }
   }
 
+  module CassandraBookmarker
+    def self.bookmark_for(item)
+      [item['bucket'], item['ordered_id']]
+    end
+
+    def self.validate(bookmark)
+      bookmark.is_a?(Array) &&
+      bookmark.size == 2
+    end
+  end
 
   # returns a collection with very limited functionality
   # basically, it responds to #paginate and returns a
   # WillPaginate::Collection-like object
   def self.for_user(user)
     if cassandra?
-      PaginatedCollection.build { |pager| page_view_history(user, pager) }
+      BookmarkedCollection.build(PageView::CassandraBookmarker) do |pager|
+        page_view_history(user, pager)
+      end
     else
       self.scoped(:conditions => { :user_id => user.id }, :order => 'created_at desc')
     end
@@ -205,43 +217,56 @@ def self.page_view_history(context, pager)
     context_type = context.class.name
     context_id = context.global_id
     scrollback_limit = Setting.get('page_views_scrollback_limit:users', 52.weeks.to_s).to_i.ago
-    results = []
 
-    if pager.current_page.to_s =~ %r{^(\d+):(\d+/\w+)$}
-      row_ts = $1.to_i
-      start_column = $2
+    # get the bucket to start at from the bookmark
+    if pager.current_bookmark
+      bucket, ordered_id = pager.current_bookmark
     else
       # page 1
-      row_ts = PageView.timeline_bucket_for_time(Time.now, context_type)
+      bucket = PageView.timeline_bucket_for_time(Time.now, context_type)
+      ordered_id = nil
     end
 
-    until pager.next_page || Time.at(row_ts) < scrollback_limit
-      limit = pager.per_page + 1 - results.size
+    # pull results from each bucket until the page is full or we hit the
+    # scrollback_limit
+    until pager.next_bookmark || Time.at(bucket) < scrollback_limit
+      # build up the query based on the context, bucket, and ordered_id. fetch
+      # one extra so we can tell if there are more pages
+      limit = pager.per_page + 1 - pager.size
       args = []
-      args << "#{context_type.underscore}_#{context_id}/#{row_ts}"
-      if start_column
-        ordered_id = "AND ordered_id <= ?"
-        args << start_column
+      args << "#{context_type.underscore}_#{context_id}/#{bucket}"
+      if ordered_id
+        ordered_id_clause = (pager.include_bookmark ? "AND ordered_id <= ?" : "AND ordered_id < ?")
+        args << ordered_id
       else
-        ordered_id = nil
+        ordered_id_clause = nil
       end
-      qs = "SELECT ordered_id, request_id FROM page_views_history_by_context where context_and_time_bucket = ? #{ordered_id} ORDER BY ordered_id DESC LIMIT #{limit}"
+      qs = "SELECT ordered_id, request_id FROM page_views_history_by_context where context_and_time_bucket = ? #{ordered_id_clause} ORDER BY ordered_id DESC LIMIT #{limit}"
+
+      # execute the query collecting the results. set the bookmark iff there
+      # was a result after the full page
       PageView.cassandra.execute(qs, *args).fetch do |row|
-        if results.size == pager.per_page
-          pager.next_page = "#{row_ts}:#{row['ordered_id']}"
+        if pager.size == pager.per_page
+          pager.has_more!
         else
-          results << row['request_id']
+          pager << row.to_hash.merge('bucket' => bucket)
         end
       end
 
-      start_column = nil
+      ordered_id = nil
       # possible optimization: query page_views_counters_by_context_and_day ,
       # and use it as a secondary index to skip days where the user didn't
       # have any page views
-      row_ts -= PageView.timeline_bucket_size(context_type)
+      bucket -= PageView.timeline_bucket_size(context_type)
     end
 
-    pager.replace(PageView.find(results).sort_by { |pv| results.index(pv.request_id) })
+    # now that the bookmark's been set, we only need to keep the request
+    # ids per item returned.
+    request_ids = pager.map{ |row| row['request_id'] }
+
+    # before returning, convert those request_ids into actual page views, but
+    # preserve the order
+    pager.replace(PageView.find(request_ids).sort_by { |pv| request_ids.index(pv.request_id) })
   end
 
   def self.cache_queue_name

lib/api.rb

@@ -181,15 +181,20 @@ def self.paginate(collection, controller, base_url, pagination_args = {})
     pagination_args.reverse_merge!({ :page => controller.params[:page], :per_page => per_page })
     collection = collection.paginate(pagination_args)
     return unless collection.respond_to?(:next_page)
-    total_pages = (pagination_args[:without_count] ? nil : collection.total_pages)
-    total_pages = nil if total_pages.to_i <= 1
+
+    first_page = collection.respond_to?(:first_page) && collection.first_page
+    first_page ||= 1
+
+    last_page = (pagination_args[:without_count] ? nil : collection.total_pages)
+    last_page = nil if last_page.to_i <= 1
+
     links = build_links(base_url, {
       :query_parameters => controller.request.query_parameters,
       :per_page => collection.per_page,
       :next => collection.next_page,
       :prev => collection.previous_page,
-      :first => 1,
-      :last => total_pages,
+      :first => first_page,
+      :last => last_page,
     })
     controller.response.headers["Link"] = links.join(',') if links.length > 0
     collection