Instrumentation for Incoming Mail Processing

Fixes CNVS-20436

This does two things:
1) Reports to StatsD each time a message
   is processed
2) Every 5 minutes, reports the unprocessed size
   of each incoming mailbox

Test Plan:
Regression test incoming mail processing. Will
want to check the basic cases of incoming messages
still being processed properly.

Smoke test the new Instrumentation delayed job.
1) Setup incoming mail processing by adding a 
   config/incoming_mail.yml with the right content
   - Wheeler can tell you what the file
     should contain. (not shown because passwords)

2) Start canvas jobs 
     `bundle exec script/delayed_job run`

3) Watch /error_reports for any new error reports.
   You should see none. 


Change-Id: Ia737906cedb67e183fec0f41571e002883c058c8
Reviewed-on: https://gerrit.instructure.com/57892
Reviewed-by: Joel Hough <joel@instructure.com>
Tested-by: Jenkins
QA-Review: Steven Shepherd <sshepherd@instructure.com>
Product-Review: Matthew Wheeler <mwheeler@instructure.com>

Author: mcwqy9

config/initializers/periodic_jobs.rb

@@ -80,6 +80,10 @@ def with_each_shard_by_database(klass, method, *args)
     end
   end
 
+  Delayed::Periodic.cron 'IncomingMailProcessor::Instrumentation#process', '*/5 * * * *' do
+    IncomingMailProcessor::Instrumentation.process
+  end
+
   Delayed::Periodic.cron 'ErrorReport.destroy_error_reports', '2-59/5 * * * *' do
     cutoff = Setting.get('error_reports_retain_for', 3.months.to_s).to_i
     if cutoff > 0

gems/incoming_mail_processor/incoming_mail_processor.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mail", "2.5.4"
   spec.add_dependency "html_text_helper"
   spec.add_dependency "utf8_cleaner"
+  spec.add_dependency "canvas_statsd"
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rspec", "2.99.0"

gems/incoming_mail_processor/lib/incoming_mail_processor.rb

@@ -1,6 +1,7 @@
 require "html_text_helper"
 require "mail"
 require "utf8_cleaner"
+require "canvas_statsd"
 
 module IncomingMailProcessor
   require "incoming_mail_processor/pop3_mailbox"
@@ -11,4 +12,5 @@ module IncomingMailProcessor
   require "incoming_mail_processor/incoming_message_processor"
   require "incoming_mail_processor/mailbox_account"
   require "incoming_mail_processor/settings"
+  require "incoming_mail_processor/instrumentation"
 end

gems/incoming_mail_processor/lib/incoming_mail_processor/directory_mailbox.rb

@@ -65,6 +65,11 @@ def move_message(filename, target_folder)
       move_file(folder, filename, target_folder)
     end
 
+    def unprocessed_message_count
+      # not implemented, and used only for performance monitoring.
+      nil
+    end
+
   private
     def folder_exists?(folder, subfolder = nil)
       to_check = subfolder ? File.join(folder, subfolder) : folder

gems/incoming_mail_processor/lib/incoming_mail_processor/imap_mailbox.rb

@@ -74,6 +74,13 @@ def move_message(message_id, target_folder)
       delete_message(message_id)
     end
 
+    def unprocessed_message_count
+      connect
+      @imap.select("INBOX")
+      length = @imap.search(["X-GM-RAW", "label:unread"]).length
+      disconnect
+      length
+    end
   end
 
 end

gems/incoming_mail_processor/lib/incoming_mail_processor/incoming_message_processor.rb

@@ -87,7 +87,9 @@ def process_single(incoming_message, tag, mailbox_account = IncomingMailProcesso
 
       body, html_body = extract_body(incoming_message)
 
-      @message_handler.handle(mailbox_account.address, body, html_body, incoming_message, tag)
+      handle = @message_handler.handle(mailbox_account.address, body, html_body, incoming_message, tag)
+      CanvasStatsd::Statsd.increment("incoming_mail_processor.incoming_message_processed")
+      handle
     end
 
     private

gems/incoming_mail_processor/lib/incoming_mail_processor/instrumentation.rb

@@ -0,0 +1,46 @@
+#
+# Copyright (C) 2011-2015 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+module IncomingMailProcessor
+  class Instrumentation
+    def self.process
+      unreads = []
+
+      mailbox_accounts.each do |a|
+        unreads << IncomingMailProcessor::IncomingMessageProcessor.create_mailbox(a).unprocessed_message_count
+      end
+
+      report_unreads(unreads)
+    end
+
+    def self.mailbox_accounts
+      IncomingMailProcessor::IncomingMessageProcessor.mailbox_accounts
+    end
+    private_class_method :mailbox_accounts
+
+    def self.report_unreads(unreads)
+      result = Hash[mailbox_accounts.map{|a| a.config[:username]}.zip(unreads)]
+      result.delete_if { |_k, v| v.nil? }
+      result.each_pair do |username, count|
+        name = "incoming_mail_processor." + CanvasStatsd::Statsd.escape(username)
+        CanvasStatsd::Statsd.gauge name, count
+      end
+    end
+    private_class_method :report_unreads
+  end
+end

gems/incoming_mail_processor/lib/incoming_mail_processor/pop3_mailbox.rb

@@ -66,6 +66,10 @@ def move_message(pop_message, target_folder)
       # pop can't do this -- just delete the message
       delete_message(pop_message)
     end
-  end
 
+    def unprocessed_message_count
+      # not implemented, and used only for performance monitoring.
+      nil
+    end
+  end
 end

gems/incoming_mail_processor/spec/incoming_mail_processor/directory_mailbox_spec.rb

@@ -89,6 +89,12 @@ def default_config
     end
   end
 
+  describe '#unprocessed_message_count' do
+    it "should return nil" do
+      @mailbox.unprocessed_message_count.should be_nil
+    end
+  end
+
   context "with simple foo file" do
 
     before do

gems/incoming_mail_processor/spec/incoming_mail_processor/imap_mailbox_spec.rb

@@ -87,6 +87,18 @@ class <<@imap_mock
     end
   end
 
+  describe '#unprocessed_message_count' do
+    it 'returns zero if there are no messages' do
+      @imap_mock.expects(:search).with(["X-GM-RAW", "label:unread"]).once.returns([])
+      @mailbox.unprocessed_message_count.should eql 0
+    end
+
+    it 'returns the number of messages if there are any' do
+      @imap_mock.expects(:search).with(["X-GM-RAW", "label:unread"]).once.returns([1,2,3,58,42])
+      @mailbox.unprocessed_message_count.should eql 5
+    end
+  end
+
   context "connected" do
     before do
       @mailbox.connect