Silence false positive UnconsumedParameterWarning (spotify#3235)

* Silence false positive UnconsumedParameterWarning

Do not emit UnconsumedParameterWarning when parameter name is written in camelCase or with dashes.

* Silence warnings for not consumed autoload_range and no_configure_logging parameters in core config.

---------

Co-authored-by: Dillon Stadther <dlstadther@gmail.com>

Author: starhel

luigi/configuration/cfg_parser.py

@@ -117,6 +117,7 @@ def before_write(self, parser, section, option, value):
 class LuigiConfigParser(BaseParser, ConfigParser):
     NO_DEFAULT = object()
     enabled = True
+    optionxform = str
     _instance = None
     _config_paths = [
         '/etc/luigi/client.cfg',  # Deprecated old-style global luigi config

luigi/interface.py

@@ -49,6 +49,10 @@ class core(task.Config):
     This is arguably a bit of a hack.
     '''
     use_cmdline_section = False
+    ignore_unconsumed = {
+        'autoload_range',
+        'no_configure_logging',
+    }
 
     local_scheduler = parameter.BoolParameter(
         default=False,

luigi/task.py

@@ -437,9 +437,11 @@ def list_to_tuple(x):
         if not hasattr(cls, "_unconsumed_params"):
             cls._unconsumed_params = set()
         if task_family in conf.sections():
+            ignore_unconsumed = getattr(cls, 'ignore_unconsumed', set())
             for key, value in conf[task_family].items():
+                key = key.replace('-', '_')
                 composite_key = f"{task_family}_{key}"
-                if key not in result and composite_key not in cls._unconsumed_params:
+                if key not in result and key not in ignore_unconsumed and composite_key not in cls._unconsumed_params:
                     warnings.warn(
                         "The configuration contains the parameter "
                         f"'{key}' with value '{value}' that is not consumed by the task "

test/task_test.py

@@ -218,6 +218,65 @@ class TaskB(luigi.Task):
                     f"the task '{task_name}'."
                 )
 
+    @with_config(
+        {
+            "TaskEdgeCase": {
+                "camelParam": "camelCase",
+                "underscore_param": "underscore",
+                "dash-param": "dash",
+            },
+        }
+    )
+    def test_unconsumed_params_edge_cases(self):
+        class TaskEdgeCase(luigi.Task):
+            camelParam = luigi.Parameter()
+            underscore_param = luigi.Parameter()
+            dash_param = luigi.Parameter()
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.filterwarnings(
+                action="ignore",
+                category=Warning,
+            )
+            warnings.simplefilter(
+                action="always",
+                category=luigi.parameter.UnconsumedParameterWarning,
+            )
+
+            task = TaskEdgeCase()
+            assert len(w) == 0
+            assert task.camelParam == "camelCase"
+            assert task.underscore_param == "underscore"
+            assert task.dash_param == "dash"
+
+    @with_config(
+        {
+            "TaskIgnoreUnconsumed": {
+                "a": "a",
+                "b": "b",
+                "c": "c",
+            },
+        }
+    )
+    def test_unconsumed_params_ignore_unconsumed(self):
+        class TaskIgnoreUnconsumed(luigi.Task):
+            ignore_unconsumed = {"b", "d"}
+
+            a = luigi.Parameter()
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.filterwarnings(
+                action="ignore",
+                category=Warning,
+            )
+            warnings.simplefilter(
+                action="always",
+                category=luigi.parameter.UnconsumedParameterWarning,
+            )
+
+            TaskIgnoreUnconsumed()
+            assert len(w) == 1
+
 
 class TaskFlattenOutputTest(unittest.TestCase):
     def test_single_task(self):