Add new hook on_plugin_not_found for Verification Controller (#41)

* Add New hook on_plugin_not_found for Verification Controller

* Update Changelog

Author: shashankrnr32

Makefile

@@ -21,11 +21,15 @@ autogen:
 	ychaos manual --file docs/cli/manual.md > /dev/null
 
 # Builds documentation and serves on http://localhost:8000
-.PHONEY: doc
+.PHONY: doc
 doc:
 	chmod +x develop/doc.sh
 	./develop/doc.sh
 
-.PHONEY: docbuild
+.PHONY: docbuild
 docbuild:
 	mkdocs build
+
+.PHONY: schema
+schema:
+	python3 develop/autogen_schema.py

docs/releases.md

@@ -2,7 +2,12 @@
 
 ## Version 0.x.x
 
-## Next release (Version 0.1.0)
+## Next Release (0.2.0)
+
+1. Fix throwing error when a verification plugin implementation is not found or in development stage
+by [Shashank Sharma](https://github.com/shashankrnr32)
+
+## Version 0.1.0
 
 1. Add documentation to YChaos by [Shashank Sharma](https://github.com/shashankrnr32)
 

src/ychaos/cli/verify.py

@@ -8,7 +8,7 @@
 from ..core.verification.controller import VerificationController
 from ..core.verification.data import VerificationStateData
 from ..testplan import SystemState
-from ..testplan.verification import VerificationConfig
+from ..testplan.verification import VerificationConfig, VerificationType
 from . import YChaosCLIHook, YChaosTestplanInputSubCommand
 
 __all__ = ["Verify"]
@@ -125,6 +125,12 @@ def __call__(self, index: int, config: VerificationConfig):
                     f"Running [i]{self.state.value.lower()}[/i] state verification of type={config.type.value}[{index}]"
                 )
 
+        class OnPluginNotFoundHook(VerificationHook):
+            def __call__(self, index: int, plugin_type: VerificationType):
+                self.console.log(
+                    f"The verification plugin type=[i]{plugin_type.value}[/i][{index}] is not available for use."
+                )
+
         class OnEachPluginEndHook(VerificationHook):
             def __call__(
                 self,
@@ -150,6 +156,9 @@ def __call__(
         verification_controller.register_hook(
             "on_each_plugin_end", OnEachPluginEndHook(self.app, self.state)
         )
+        verification_controller.register_hook(
+            "on_plugin_not_found", OnPluginNotFoundHook(self.app, self.state)
+        )
 
         self.console.log(
             f"Starting [i]{self.state.value.lower()}[/i] state verification."

src/ychaos/core/verification/controller.py

@@ -76,6 +76,14 @@ def callable_hook(index: int, config: VerificationConfig, state_data: Verificati
             def callable_hook(verify_list: List[bool]): ...
         ```
 
+    === "on_plugin_not_found"
+        Hook that gets called when a plugin available in schema is not ready for usage/not implemented.
+        This case is possible for the plugins that are in Beta/development phase
+
+        ```python
+            def callable_hook(index:int, plugin_type: VerificationType): ...
+        ```
+
     ---
     Each of the hooks get called on a certain event. The caller can register as many hooks for a particular event,
     by calling the `register_hook(event_name, hook_method)` method. All the hooks are executed sequentially. The best example
@@ -86,6 +94,7 @@ def callable_hook(verify_list: List[bool]): ...
         "on_start",
         "on_each_plugin_start",
         "on_each_plugin_end",
+        "on_plugin_not_found",
         "on_end",
     )
 
@@ -148,9 +157,17 @@ def execute(self) -> bool:
                     msg=f"Starting {verification_plugin.type.value} verification"
                 )
                 plugin_class = VERIFICATION_PLUGIN_MAP.get(
-                    verification_plugin.type.value
+                    verification_plugin.type.value, None
                 )
-                assert plugin_class is not None
+
+                if plugin_class is None:
+                    # This can happen when a new plugin is not implemented yet, but is
+                    # available in the schema
+                    self.execute_hooks(
+                        "on_plugin_not_found", index, verification_plugin.type
+                    )
+                    continue
+
                 plugin = plugin_class(verification_plugin.config, data)
 
                 # Call all the hooks that were registered for `verification_plugin_start`.

src/ychaos/testplan/resources/schema.json

@@ -46,7 +46,8 @@
             "enum": [
                 "python_module",
                 "http_request",
-                "sdv4"
+                "sdv4",
+                "noop"
             ]
         },
         "VerificationConfig": {

src/ychaos/testplan/verification.py

@@ -141,6 +141,10 @@ class SDv4Verification(SchemaModel):
     job_timeout: PositiveInt = Field(default=3600, description="Job Timeout in seconds")
 
 
+class NoOpConfig(SchemaModel):
+    pass
+
+
 class VerificationType(AEnum):
     """
     Defines the Type of plugin to be used for verification.
@@ -153,6 +157,9 @@ class VerificationType(AEnum):
     HTTP_REQUEST = "http_request", SimpleNamespace(schema=HTTPRequestVerification)
     SDV4_VERIFICATION = "sdv4", SimpleNamespace(schema=SDv4Verification)
 
+    # For Testing purpose, cannot be used by users.
+    NOOP = "noop", SimpleNamespace(schema=NoOpConfig)
+
 
 class VerificationConfig(SchemaModel):
     """

tests/cli/test_verify.py

@@ -189,3 +189,24 @@ def test_verification_for_valid_test_plan_with_invalid_plugin_path_when_not_stri
         self.assertTrue(
             "The system is verified to be in steady state" in app.get_console_output()
         )
+
+    def test_verification_for_valid_testplan_with_unimplemented_plugin(self):
+        args = Namespace()
+        args.cls = self.cls
+
+        testplan_path = self.testplans_directory.joinpath("valid/testplan6.yaml")
+
+        # Required Arguments for VerificationCommand
+        args.testplan = str(testplan_path.resolve())
+        args.state = "steady"
+
+        # Create a Mocked CLI App
+        app = MockApp(args)
+        args.app = app
+
+        self.assertEqual(0, args.cls.main(args))
+
+        self.assertTrue(
+            "The verification plugin type=noop[0] is not available for use."
+            in app.get_console_output()
+        )

tests/core/verification/test_controller.py

@@ -8,6 +8,7 @@
 from ychaos.core.verification.controller import VerificationController
 from ychaos.testplan import SystemState
 from ychaos.testplan.schema import TestPlan
+from ychaos.testplan.verification import VerificationConfig, VerificationType
 
 
 class TestVerificationController(TestCase):
@@ -89,3 +90,33 @@ def test_verfication_controller_dump_json_data(self):
         self.assertListEqual(
             verification_data, verification_controller.get_encoded_verification_data()
         )
+
+    def test_verification_controller_for_plugin_not_found(self):
+        self.mock_testplan.verification[0] = VerificationConfig(
+            states=[
+                SystemState.STEADY,
+            ],
+            type=VerificationType.NOOP,
+            config=dict(),
+        )
+
+        verification_controller = VerificationController(
+            self.mock_testplan, SystemState.STEADY, list()
+        )
+
+        class MockOnPluginNotFoundHook:
+            def __init__(self):
+                self.test_var = False
+
+            def __call__(self, *args):
+                self.test_var = True
+
+        mock_on_plugin_not_found_hook = MockOnPluginNotFoundHook()
+        verification_controller.register_hook(
+            "on_plugin_not_found", mock_on_plugin_not_found_hook
+        )
+
+        is_verified = verification_controller.execute()
+        self.assertTrue(is_verified)
+
+        self.assertTrue(mock_on_plugin_not_found_hook.test_var)

tests/resources/testplans/valid/testplan6.yaml

@@ -0,0 +1,11 @@
+description: A valid mock testplan file with un-implemented verification plugin
+verification:
+    - states:
+          - CHAOS
+          - STEADY
+      type: noop
+      config: {}
+attack:
+    target_type: self
+    agents:
+        - type: no_op