switch yaml engine to psych

closes #CNVS-27229

Change-Id: I257718e3e4603a0e8a25519ab10247f26d57394b
Reviewed-on: https://gerrit.instructure.com/77480
Tested-by: Jenkins
Reviewed-by: Cody Cutrer <cody@instructure.com>
Product-Review: James Williams  <jamesw@instructure.com>
QA-Review: James Williams  <jamesw@instructure.com>

Author: maneframe

app/models/submission_version.rb

@@ -43,7 +43,7 @@ def extract_version_attributes(version, options = {})
         begin
           return nil unless Submission.where(:id => version.versionable_id).exists?
           version.model
-        rescue ArgumentError
+        rescue Psych::SyntaxError
           return nil
         end
       else

db/migrate/20160210153643_begin_psych_migration.rb

@@ -1,10 +1,16 @@
 class BeginPsychMigration < ActiveRecord::Migration
-  tag :postdeploy
+  tag :predeploy
 
-  disable_ddl_transaction!
+  def runnable?
+    Shard.current.default?
+  end
 
   def up
-    DataFixup::PsychMigration.run if CANVAS_RAILS4_0 || !Rails.env.test?
+    if User.exists? # don't raise for a fresh install
+      raise "WARNING:\n
+        This migration needs to be run with the release/2016-04-23 version of canvas-lms to
+        change all yaml columns in the database to a Psych compatible format.\n"
+    end
   end
 
   def down

db/migrate/20160420152338_cleanse_the_syckness.rb

@@ -0,0 +1,10 @@
+class CleanseTheSyckness < ActiveRecord::Migration
+  tag :postdeploy
+
+  def up
+    DataFixup::SycknessCleanser.send_later_if_production_enqueue_args(:run, {:strand => Shard.current.database_server.id.to_s})
+  end
+
+  def down
+  end
+end

lib/canvas_yaml.rb

@@ -20,20 +20,15 @@
 # and before canvas-jobs
 
 # We need to make sure that safe_yaml is loaded *after* the YAML engine
-# is switched to Syck. Otherwise we
+# is switched to Psych. Otherwise we
 # won't have access to (safe|unsafe)_load.
 require 'yaml'
 
-require 'syck'
-YAML::ENGINE.yamler = 'syck' if defined?(YAML::ENGINE)
+require 'syck' # so we can undo all the things before something else requires it
+YAML::ENGINE.yamler = 'psych' if defined?(YAML::ENGINE)
 
 require 'safe_yaml'
 
-trusted_tags = SafeYAML::TRUSTED_TAGS.dup
-trusted_tags << 'tag:yaml.org,2002:merge'
-SafeYAML.send(:remove_const, :TRUSTED_TAGS)
-SafeYAML.const_set(:TRUSTED_TAGS, trusted_tags.freeze)
-
 module FixSafeYAMLNullMerge
   def merge_into_hash(hash, array)
     return unless array
@@ -49,25 +44,6 @@ def merge_into_hash(hash, array)
     # This tag whitelist is syck specific. We'll need to tweak it when we upgrade to psych.
     # See the tests in spec/lib/safe_yaml_spec.rb
     whitelisted_tags: %w[
-        tag:ruby.yaml.org,2002:symbol
-        tag:yaml.org,2002:float
-        tag:yaml.org,2002:float#exp
-        tag:yaml.org,2002:float#inf
-        tag:yaml.org,2002:str
-        tag:yaml.org,2002:timestamp
-        tag:yaml.org,2002:timestamp#iso8601
-        tag:yaml.org,2002:timestamp#spaced
-        tag:yaml.org,2002:map:HashWithIndifferentAccess
-        tag:yaml.org,2002:map:ActiveSupport::HashWithIndifferentAccess
-        tag:ruby.yaml.org,2002:object:Class
-        tag:ruby.yaml.org,2002:object:OpenStruct
-        tag:ruby.yaml.org,2002:object:Scribd::Document
-        tag:ruby.yaml.org,2002:object:Mime::Type
-        tag:ruby.yaml.org,2002:object:URI::HTTP
-        tag:ruby.yaml.org,2002:object:URI::HTTPS
-        tag:ruby.yaml.org,2002:object:OpenObject
-        tag:yaml.org,2002:map:WeakParameters
-      ] + %w[
         !ruby/symbol
         !binary
         !float
@@ -96,43 +72,6 @@ def merge_into_hash(hash, array)
 
 module Syckness
   TAG = "#GETDOWNWITHTHESYCKNESS\n"
-
-  module SyckDumper
-    def dump(*args)
-      Psych.dump(*args)
-    end
-  end
-
-  module PsychDumper
-    def dump(*args)
-      yaml = super
-      yaml += TAG if yaml.is_a?(String)
-      yaml
-    end
-  end
-
-  module SafeLoader
-    require "safe_yaml/psych_resolver"
-    require "safe_yaml/safe_to_ruby_visitor"
-
-    def load(yaml, *args)
-      if yaml.is_a?(String) && yaml.end_with?(TAG)
-        SafeYAML::PsychResolver.new.resolve_node(Psych.parse(yaml))
-      else
-        super
-      end
-    end
-  end
-
-  module UnsafeLoader
-    def unsafe_load(yaml, *args)
-      if yaml.is_a?(String) && yaml.end_with?(TAG)
-        Psych.load(yaml)
-      else
-        super
-      end
-    end
-  end
 end
 
 [Object, Hash, Struct, Array, Exception, String, Symbol, Range, Regexp, Time,
@@ -142,11 +81,6 @@ def unsafe_load(yaml, *args)
   end
 end
 
-Syck.singleton_class.prepend(Syckness::SyckDumper)
-Psych.singleton_class.prepend(Syckness::PsychDumper)
-SafeYAML.singleton_class.prepend(Syckness::SafeLoader)
-YAML.singleton_class.prepend(Syckness::UnsafeLoader)
-
 SafeYAML::PsychResolver.class_eval do
   attr_accessor :aliased_nodes
 end

lib/data_fixup/psych_migration.rb

@@ -0,0 +1,41 @@
+module DataFixup
+  module SycknessCleanser
+    def self.columns_hash
+      result = ActiveRecord::Base.all_models.map do |model|
+        next unless model.superclass == ActiveRecord::Base
+        next if model.name == 'RemoveQuizDataIds::QuizQuestionDataMigrationARShim'
+
+        attributes = model.serialized_attributes.select do |attr, coder|
+          coder.is_a?(ActiveRecord::Coders::YAMLColumn)
+        end
+        next if attributes.empty?
+        [model, attributes.keys]
+      end.compact.to_h
+      result[Version] = ['yaml']
+      result[Delayed::Backend::ActiveRecord::Job] = ['handler']
+      result[Delayed::Backend::ActiveRecord::Job::Failed] = ['handler']
+      result
+    end
+
+    def self.run
+      self.columns_hash.each do |model, columns|
+        update_sqls = {}
+        quoted = {}
+
+        columns.each do |column|
+          quoted[column] = model.connection.quote_column_name(column)
+          update_sqls[column] = "#{quoted[column]} = left(#{quoted[column]}, -#{Syckness::TAG.length})"
+        end
+        model.find_ids_in_ranges do |min_id, max_id|
+          columns.each do |column|
+            model.where(model.primary_key => min_id..max_id).where("#{quoted[column]} LIKE ?", "%#{Syckness::TAG}").update_all(update_sqls[column])
+          end
+          delay = Setting.get("syckness_sleeping_ms", nil)
+          if delay
+            sleep(delay.to_i.fdiv(1000)) # just in case the cleansing is a little too strong
+          end
+        end
+      end
+    end
+  end
+end