don't timeout the fleet when an agent fails

fixes CNVS-30198

TEST PLAN:
 1) kill a consul local agent
 2) the whole fleet should not start timing out to consul

Change-Id: Ic699339862ddfd17c1cb973d93b0091d452649a5
Reviewed-on: https://gerrit.instructure.com/83627
Reviewed-by: Cody Cutrer <cody@instructure.com>
Tested-by: Jenkins
QA-Review: Jeremy Putnam <jeremyp@instructure.com>
Product-Review: Ethan Vizitei <evizitei@instructure.com>

Author: evizitei

lib/canvas/dynamic_settings.rb

@@ -7,6 +7,7 @@ class ConsulError < StandardError
     end
 
     KV_NAMESPACE = "config/canvas".freeze
+    TIMEOUT_INTERVAL = 3.freeze
 
     class << self
       attr_accessor :config, :cache, :fallback_data
@@ -77,14 +78,16 @@ def init_values(hash)
             store_put("#{parent_key}/#{child_key}", value)
           end
         end
+      rescue Timeout::Error
+        return false
       end
 
       def store_get(key)
         begin
           # store all values that we get here to
           # kind-of recover in case of big failure
           @strategic_reserve ||= {}
-          consul_value = Canvas.timeout_protection('consul', {raise_on_timeout: true}) do
+          consul_value = Timeout.timeout(TIMEOUT_INTERVAL) do
             diplomat_get(key)
           end
 
@@ -106,7 +109,7 @@ def store_get(key)
       end
 
       def store_put(key, value)
-        Canvas.timeout_protection('consul') do
+        Timeout.timeout(TIMEOUT_INTERVAL) do
           Diplomat::Kv.put("#{KV_NAMESPACE}/#{key}", value)
         end
       end

spec/lib/canvas/dynamic_settings_spec.rb

@@ -249,18 +249,13 @@ def stub_consul_with(value)
         end
 
         it "returns old value during connection timeout" do
-          Canvas.stubs(:timeout_protection).with('consul', raise_on_timeout: true).
+
+          Timeout.stubs(:timeout).with(DynamicSettings::TIMEOUT_INTERVAL).
             raises(Timeout::Error, 'consul took too long')
           value = DynamicSettings.from_cache(parent_key, expires_in: 10.minutes)
           expect(value["app-host"]).to eq("rce.insops.com")
         end
 
-        it "uses cached values during TimeoutCutoff events" do
-          Canvas.stubs(:timeout_protection).with('consul', raise_on_timeout: true).
-            raises(TimeoutCutoff, 'consul took too long too many times')
-          value = DynamicSettings.from_cache(parent_key, expires_in: 10.minutes)
-          expect(value["app-host"]).to eq("rce.insops.com")
-        end
       end
     end