don't update the service timeout again if we're over the limit

refs CNVS-8371

test plan:
* trigger 3 ldap timeouts, see test plan in c17f357
* you'll be blocked from further ldap attempts for 1 minute. after that
  minute passes, verify that canvas starts making ldap attempts again.

Change-Id: I8ac6b74bd75f6f3e18deaf3a1fabd01fb867c26e
Reviewed-on: https://gerrit.instructure.com/25047
Reviewed-by: Rob Orton <rob@instructure.com>
Tested-by: Brian Palmer <brianp@instructure.com>
QA-Review: August Thornton <august@instructure.com>
Product-Review: Brian Palmer <brianp@instructure.com>

Author: codekitchen

lib/canvas.rb

@@ -148,17 +148,20 @@ def self.timeout_protection(service_name, options={})
     end
 
     timeout = (Setting.get_cached("service_#{service_name}_timeout", nil) || options[:fallback_timeout_length] || Setting.get_cached("service_generic_timeout", 15.seconds.to_s)).to_f
-    Timeout.timeout(timeout) do
-      yield
-    end
-  rescue Timeout::Error => e
-    ErrorReport.log_exception(:service_timeout, e)
-    if Canvas.redis_enabled?
-      error_ttl = (Setting.get_cached("service_#{service_name}_error_ttl", nil) || Setting.get_cached("service_generic_error_ttl", 1.minute.to_s)).to_i
-      Canvas.redis.incrby(redis_key, 1)
-      Canvas.redis.expire(redis_key, error_ttl)
+
+    begin
+      Timeout.timeout(timeout) do
+        yield
+      end
+    rescue Timeout::Error => e
+      ErrorReport.log_exception(:service_timeout, e)
+      if Canvas.redis_enabled?
+        error_ttl = (Setting.get_cached("service_#{service_name}_error_ttl", nil) || Setting.get_cached("service_generic_error_ttl", 1.minute.to_s)).to_i
+        Canvas.redis.incrby(redis_key, 1)
+        Canvas.redis.expire(redis_key, error_ttl)
+      end
+      raise if options[:raise_on_timeout]
+      return nil
     end
-    raise if options[:raise_on_timeout]
-    return nil
   end
 end

spec/lib/canvas_spec.rb

@@ -75,6 +75,7 @@
       it "should raise on cutoff if raise_on_timeout option is specified" do
         Canvas.redis.set("service:timeouts:spec", 42)
         lambda { Canvas.timeout_protection("spec", raise_on_timeout: true) {} }.should raise_error(Timeout::Error)
+        Canvas.redis.get("service:timeouts:spec").should == "42"
       end
     end
   end