diff --git a/xarray/testing.py b/xarray/testing.py
index 59737e1d23e..481a23340fd 100644
--- a/xarray/testing.py
+++ b/xarray/testing.py
@@ -30,7 +30,8 @@ def wrapper(*args, **kwargs):
         __tracebackhide__ = True
 
         with warnings.catch_warnings():
-            warnings.simplefilter("always")
+            # only remove filters that would "error"
+            warnings.filters = [f for f in warnings.filters if f[0] != "error"]
 
             return func(*args, **kwargs)
 
diff --git a/xarray/tests/test_testing.py b/xarray/tests/test_testing.py
index 90e12292966..df78b876965 100644
--- a/xarray/tests/test_testing.py
+++ b/xarray/tests/test_testing.py
@@ -164,3 +164,16 @@ def __array__(self):
             getattr(xr.testing, func)(a, b)
 
         assert len(w) > 0
+
+        # ensure warnings still raise outside of assert_*
+        with pytest.raises(UserWarning):
+            warnings.warn("test")
+
+    # ensure warnings stay ignored in assert_*
+    with warnings.catch_warnings(record=True) as w:
+        # ignore warnings
+        warnings.filterwarnings("ignore")
+        with pytest.raises(AssertionError):
+            getattr(xr.testing, func)(a, b)
+
+        assert len(w) == 0