diff --git a/CHANGELOG.md b/CHANGELOG.md
index e2b926fe82..3571bf28e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -59,6 +59,7 @@ The file documents changes to the PHP_CodeSniffer project.
         - A descriptive error message is provided to allow users to fix their ruleset
     - Sniff properties set for complete standards/complete sniff categories will now only be set on sniffs which explicitly support the property
         - The property will be silently ignored for those sniffs which do not support the property
+    - Invalid sniff properties set for sniffs via inline annotations will result in an informative `Internal.PropertyDoesNotExist` errror on line 1 of the scanned file, but will not halt the execution of PHPCS
     - For sniff developers, it is strongly recommended for sniffs to explicitly declare any user-adjustable public properties
         - If dynamic properties need to be supported for a sniff, either declare the magic __set()/__get()/__isset()/__unset() methods on the sniff or let the sniff extend stdClass
         - Note: The #[\AllowDynamicProperties] attribute will have no effect for properties which are being set in rulesets.
diff --git a/src/Files/File.php b/src/Files/File.php
index 42e204a534..50a763dce0 100644
--- a/src/Files/File.php
+++ b/src/Files/File.php
@@ -343,6 +343,7 @@ public function process()
         $listenerIgnoreTo = [];
         $inTests          = defined('PHP_CODESNIFFER_IN_TESTS');
         $checkAnnotations = $this->config->annotations;
+        $annotationErrors = [];
 
         // Foreach of the listeners that have registered to listen for this
         // token, get them to process it.
@@ -411,7 +412,15 @@ public function process()
                                 'scope' => 'sniff',
                             ];
                             $listenerClass = $this->ruleset->sniffCodes[$listenerCode];
-                            $this->ruleset->setSniffProperty($listenerClass, $propertyCode, $settings);
+                            try {
+                                $this->ruleset->setSniffProperty($listenerClass, $propertyCode, $settings);
+                            } catch (RuntimeException $e) {
+                                // Non-existant property being set via an inline annotation.
+                                // This is typically a PHPCS test case file, but we can't throw an error on the annotation
+                                // line as it would get ignored. We also don't want this error to block
+                                // the scan of the current file, so collect these and throw later.
+                                $annotationErrors[] = 'Line '.$token['line'].': '.str_replace('Ruleset invalid. ', '', $e->getMessage());
+                            }
                         }
                     }
                 }//end if
@@ -536,6 +545,13 @@ public function process()
             }
         }
 
+        if ($annotationErrors !== []) {
+            $error  = 'Encountered invalid inline phpcs:set annotations. Found:'.PHP_EOL;
+            $error .= implode(PHP_EOL, $annotationErrors);
+
+            $this->addWarning($error, null, 'Internal.PropertyDoesNotExist');
+        }
+
         if (PHP_CODESNIFFER_VERBOSITY > 2) {
             echo "\t*** END TOKEN PROCESSING ***".PHP_EOL;
             echo "\t*** START SNIFF PROCESSING REPORT ***".PHP_EOL;
diff --git a/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.inc b/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.inc
index 3355c2d32c..e2ca72d1f1 100644
--- a/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.inc
+++ b/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.inc
@@ -1,2 +1,9 @@
 <?php
 $output = `ls -al`;
+
+// Testing an invalid phpcs:set annotations.
+// This test is unrelated to this sniff, but the issue needs a sniff to be tested.
+// phpcs:set Generic.PHP.BacktickOperator unknown 10
+
+// Make sure errors after an incorrect annotation are still being thrown.
+$output = `ls -al`;
diff --git a/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.php b/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.php
index 27c405a883..f3e8cfd537 100644
--- a/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.php
+++ b/src/Standards/Generic/Tests/PHP/BacktickOperatorUnitTest.php
@@ -25,7 +25,10 @@ class BacktickOperatorUnitTest extends AbstractSniffUnitTest
      */
     public function getErrorList()
     {
-        return [2 => 2];
+        return [
+            2 => 2,
+            9 => 2,
+        ];
 
     }//end getErrorList()
 
@@ -40,6 +43,7 @@ public function getErrorList()
      */
     public function getWarningList()
     {
+        // Warning about incorrect annotation will be shown on line 1 once PR #3915 would be merged.
         return [];
 
     }//end getWarningList()