Use new sets everywhere

With OTP 28 sets switchd to using version 2 by default. That caused various
tests to fail, as Jan discovered in #5507

In the previous PR #5510 we switched few mango instances to use a helper
utility, but it was repeated a few times and Gábor made a good suggestion to
use utility functions instead, so that's what this PR does. But it does it for
the whole code base.

Once we're post OTP 28 we can remove the utility and just use plain set
functions.

Author: nickva

src/couch/src/couch_multidb_changes.erl

@@ -200,7 +200,7 @@ handle_info(_Msg, State) ->
 
 shards_db_fold(#row{dbname = <<"shards/", _/binary>> = DbName} = Row, {Add, Remove}) ->
     Fun = fun(Dbs) -> sets:add_element(DbName, Dbs) end,
-    Init = sets:from_list([DbName], [{version, 2}]),
+    Init = couch_util:set_from_list([DbName]),
     case Row#row.wait_shard_map of
         true ->
             {maps:update_with(mem3:dbname(DbName), Fun, Init, Add), Remove};
@@ -354,10 +354,10 @@ is_deleted(Change) ->
 local_shards(Db0) ->
     Db = mem3:dbname(Db0),
     try
-        sets:from_list([S || #shard{name = S} <- mem3:local_shards(Db)], [{version, 2}])
+        couch_util:set_from_list([S || #shard{name = S} <- mem3:local_shards(Db)])
     catch
         error:database_does_not_exist ->
-            sets:new([{version, 2}])
+            couch_util:new_set()
     end.
 
 notify_fold(DbName, {Server, DbSuffix, Count}) ->

src/couch/src/couch_util.erl

@@ -47,6 +47,7 @@
 -export([get_config_hash_algorithms/0]).
 -export([remove_sensitive_data/1]).
 -export([ejson_to_map/1]).
+-export([new_set/0, set_from_list/1]).
 
 -include_lib("couch/include/couch_db.hrl").
 
@@ -862,3 +863,16 @@ ejson_to_map({Val}) when is_list(Val) ->
     maps:from_list(lists:map(fun({K, V}) -> {K, ejson_to_map(V)} end, Val));
 ejson_to_map(Val) ->
     Val.
+
+% Set types were switched to version 2 by default in OTP 28, use this helper to
+% create version 2 sets in OTP versions < 28 as well to avoid having to deal
+% with a different ordered to_list result in tests and just ensure we always
+% use version 2, which is also more performant
+%
+% Remove after OTP >= 28 and switch to plain sets:new/0 and sets:from_list/1
+
+new_set() ->
+    sets:new([{version, 2}]).
+
+set_from_list(KVs) ->
+    sets:from_list(KVs, [{version, 2}]).

src/couch_prometheus/test/eunit/couch_prometheus_e2e_tests.erl

@@ -119,7 +119,7 @@ t_no_duplicate_metrics(Port) ->
     ?assertEqual(erlang:length(Diff), 0).
 
 get_duplicates(List) ->
-    List -- sets:to_list(sets:from_list(List)).
+    List -- sets:to_list(couch_util:set_from_list(List)).
 
 t_metric_updated(Port) ->
     % The passage of time should increment this metric

src/couch_replicator/src/couch_replicator_httpd_util.erl

@@ -49,8 +49,8 @@ parse_replication_state_filter(States) when is_list(States) ->
                 Msg1 = io_lib:format("States must be one or more of ~w", [AllStates]),
                 throw({query_parse_error, ?l2b(Msg1)})
         end,
-    AllSet = sets:from_list(AllStates),
-    StatesSet = sets:from_list(AtomStates),
+    AllSet = couch_util:set_from_list(AllStates),
+    StatesSet = couch_util:set_from_list(AtomStates),
     Diff = sets:to_list(sets:subtract(StatesSet, AllSet)),
     case Diff of
         [] ->

src/fabric/src/fabric_db_purged_infos.erl

@@ -29,7 +29,7 @@ go(DbName) ->
     Fun = fun handle_message/3,
     Acc0 = #pacc{
         counters = fabric_dict:init(Workers, nil),
-        replies = sets:new([{version, 2}]),
+        replies = couch_util:new_set(),
         ring_opts = [{any, Shards}]
     },
     try
@@ -66,7 +66,7 @@ handle_message({rexi_EXIT, Reason}, Shard, #pacc{} = Acc) ->
     end;
 handle_message({ok, Info}, #shard{} = Shard, #pacc{} = Acc) ->
     #pacc{counters = Counters, replies = Replies} = Acc,
-    InfoSet = sets:from_list(Info, [{version, 2}]),
+    InfoSet = couch_util:set_from_list(Info),
     Replies1 = sets:union(InfoSet, Replies),
     Counters1 = fabric_dict:erase(Shard, Counters),
     case fabric_dict:size(Counters1) =:= 0 of
@@ -92,7 +92,7 @@ make_shards() ->
 init_acc(Shards) ->
     #pacc{
         counters = fabric_dict:init(Shards, nil),
-        replies = sets:new([{version, 2}]),
+        replies = couch_util:new_set(),
         ring_opts = [{any, Shards}]
     }.
 

src/fabric/src/fabric_group_info.erl

@@ -26,7 +26,7 @@ go(DbName, #doc{id = DDocId}) ->
     Ushards = mem3:ushards(DbName),
     Workers = fabric_util:submit_jobs(Shards, group_info, [DDocId]),
     RexiMon = fabric_util:create_monitors(Shards),
-    USet = sets:from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]),
+    USet = couch_util:set_from_list([{Id, N} || #shard{name = Id, node = N} <- Ushards]),
     Acc = {fabric_dict:init(Workers, nil), [], USet},
     try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc) of
         {timeout, {WorkersDict, _, _}} ->

src/fabric/src/fabric_streams.erl

@@ -199,7 +199,7 @@ spawn_worker_cleaner(Coordinator, Workers, ClientReq) when
         undefined ->
             Pid = spawn(fun() ->
                 erlang:monitor(process, Coordinator),
-                NodeRefSet = set_from_list(shards_to_node_refs(Workers)),
+                NodeRefSet = couch_util:set_from_list(shards_to_node_refs(Workers)),
                 cleaner_loop(Coordinator, NodeRefSet, ClientReq)
             end),
             put(?WORKER_CLEANER, Pid),
@@ -228,9 +228,6 @@ add_worker_to_cleaner(CoordinatorPid, #shard{node = Node, ref = Ref}) ->
             ok
     end.
 
-set_from_list(List) when is_list(List) ->
-    sets:from_list(List, [{version, 2}]).
-
 shards_to_node_refs(Workers) when is_list(Workers) ->
     Fun = fun(#shard{node = Node, ref = Ref}) -> {Node, Ref} end,
     lists:map(Fun, Workers).

src/global_changes/src/global_changes_listener.erl

@@ -55,7 +55,7 @@ init(_) ->
     State = #state{
         update_db = UpdateDb,
         pending_update_count = 0,
-        pending_updates = sets:new([{version, 2}]),
+        pending_updates = couch_util:new_set(),
         max_event_delay = MaxEventDelay,
         dbname = global_changes_util:get_dbname()
     },
@@ -99,7 +99,7 @@ handle_cast({set_update_db, Boolean}, State0) ->
             {false, true} ->
                 State0#state{
                     update_db = Boolean,
-                    pending_updates = sets:new([{version, 2}]),
+                    pending_updates = couch_util:new_set(),
                     pending_update_count = 0,
                     last_update_time = undefined
                 };
@@ -141,7 +141,7 @@ maybe_send_updates(#state{update_db = true} = State) ->
                         0
                     ),
                     State1 = State#state{
-                        pending_updates = sets:new([{version, 2}]),
+                        pending_updates = couch_util:new_set(),
                         pending_update_count = 0,
                         last_update_time = undefined
                     },

src/global_changes/src/global_changes_server.erl

@@ -66,7 +66,7 @@ init([]) ->
     State = #state{
         update_db = UpdateDb,
         pending_update_count = 0,
-        pending_updates = sets:new([{version, 2}]),
+        pending_updates = couch_util:new_set(),
         max_write_delay = MaxWriteDelay,
         dbname = GlobalChangesDbName,
         handler_ref = erlang:monitor(process, Handler)
@@ -79,7 +79,7 @@ handle_call(_Msg, _From, State) ->
 handle_cast(_Msg, #state{update_db = false} = State) ->
     {noreply, State};
 handle_cast({update_docs, DocIds}, State) ->
-    Pending = sets:union(sets:from_list(DocIds), State#state.pending_updates),
+    Pending = sets:union(couch_util:set_from_list(DocIds), State#state.pending_updates),
     PendingCount = sets:size(Pending),
     couch_stats:update_gauge(
         [global_changes, server_pending_updates],
@@ -100,7 +100,7 @@ handle_cast({set_update_db, Boolean}, State0) ->
             {false, true} ->
                 State0#state{
                     update_db = Boolean,
-                    pending_updates = sets:new([{version, 2}]),
+                    pending_updates = couch_util:new_set(),
                     pending_update_count = 0
                 };
             _ ->
@@ -168,7 +168,7 @@ flush_updates(State) ->
         0
     ),
     {noreply, State#state{
-        pending_updates = sets:new([{version, 2}]),
+        pending_updates = couch_util:new_set(),
         pending_update_count = 0
     }}.
 

src/mango/src/mango_cursor.erl

@@ -287,9 +287,9 @@ extract_selector_hints(Selector) ->
     Modules = lists:flatmap(AsIndex, ?CURSOR_MODULES),
     Populate =
         fun({Module, IndexType}) ->
-            AllFields = set_from_list(mango_selector:fields(Selector)),
+            AllFields = couch_util:set_from_list(mango_selector:fields(Selector)),
             Normalize = fun(N) -> hd(string:split(N, ":")) end,
-            IndexableFields = set_from_list(
+            IndexableFields = couch_util:set_from_list(
                 lists:map(Normalize, Module:indexable_fields(Selector))
             ),
             UnindexableFields = sets:subtract(AllFields, IndexableFields),
@@ -442,7 +442,8 @@ maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc)
 % create a dummy cursor in absence of usable indexes, utilized by _explain
 create_cursor(Db, {[], Trace0}, Selector, Opts) ->
     Blank = #{
-        filtered_indexes => sets:new([{version, 2}]), indexes_of_type => sets:new([{version, 2}])
+        filtered_indexes => couch_util:new_set(),
+        indexes_of_type => couch_util:new_set()
     },
     Trace = maps:merge(Trace0, Blank),
     Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
@@ -463,9 +464,9 @@ create_cursor(Db, {[], Trace0}, Selector, Opts) ->
         execution_stats = Stats
     }};
 create_cursor(Db, {Indexes, Trace0}, Selector, Opts) ->
-    Trace1 = maps:merge(Trace0, #{filtered_indexes => set_from_list(Indexes)}),
+    Trace1 = maps:merge(Trace0, #{filtered_indexes => couch_util:set_from_list(Indexes)}),
     [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
-    Trace = maps:merge(Trace1, #{indexes_of_type => set_from_list(CursorModIndexes)}),
+    Trace = maps:merge(Trace1, #{indexes_of_type => couch_util:set_from_list(CursorModIndexes)}),
     CursorMod:create(Db, {CursorModIndexes, Trace}, Selector, Opts).
 
 group_indexes_by_type(Indexes) ->
@@ -585,9 +586,6 @@ ddoc_name(<<"_design/", Name/binary>>) ->
 ddoc_name(Name) ->
     Name.
 
-set_from_list(KVs) ->
-    sets:from_list(KVs, [{version, 2}]).
-
 -ifdef(TEST).
 -include_lib("couch/include/couch_eunit.hrl").
 
@@ -618,8 +616,8 @@ t_create_regular(_) ->
     Trace1 = #{},
     Trace2 =
         #{
-            filtered_indexes => set_from_list(FilteredIndexes),
-            indexes_of_type => set_from_list(IndexesOfType)
+            filtered_indexes => couch_util:set_from_list(FilteredIndexes),
+            indexes_of_type => couch_util:set_from_list(IndexesOfType)
         },
     Options = [{use_index, []}, {allow_fallback, true}],
     meck:expect(mango_selector, normalize, [selector], meck:val(normalized_selector)),
@@ -648,8 +646,8 @@ t_create_user_specified_index(_) ->
     Trace1 = #{},
     Trace2 =
         #{
-            filtered_indexes => set_from_list(FilteredIndexes),
-            indexes_of_type => set_from_list(IndexesOfType)
+            filtered_indexes => couch_util:set_from_list(FilteredIndexes),
+            indexes_of_type => couch_util:set_from_list(IndexesOfType)
         },
     Options = [{use_index, [<<"_design/view_idx2">>]}],
     meck:expect(mango_selector, normalize, [selector], meck:val(normalized_selector)),
@@ -677,8 +675,8 @@ t_create_invalid_user_specified_index(_) ->
     Trace1 = #{},
     Trace2 =
         #{
-            filtered_indexes => set_from_list(UsableIndexes),
-            indexes_of_type => set_from_list(IndexesOfType)
+            filtered_indexes => couch_util:set_from_list(UsableIndexes),
+            indexes_of_type => couch_util:set_from_list(IndexesOfType)
         },
     Options = [{use_index, [<<"foobar">>]}, {allow_fallback, true}],
     meck:expect(mango_selector, normalize, [selector], meck:val(normalized_selector)),
@@ -706,8 +704,8 @@ t_create_invalid_user_specified_index_no_fallback(_) ->
     Trace1 = #{},
     Trace2 =
         #{
-            filtered_indexes => set_from_list(UsableIndexes),
-            indexes_of_type => set_from_list(IndexesOfType)
+            filtered_indexes => couch_util:set_from_list(UsableIndexes),
+            indexes_of_type => couch_util:set_from_list(IndexesOfType)
         },
     UseIndex = [<<"design">>, <<"foobar">>],
     Options = [{use_index, UseIndex}, {allow_fallback, false}],
@@ -734,8 +732,8 @@ t_create_no_suitable_index_no_fallback(_) ->
     Trace1 = #{},
     Trace2 =
         #{
-            filtered_indexes => set_from_list(UsableIndexes),
-            indexes_of_type => set_from_list(IndexesOfType)
+            filtered_indexes => couch_util:set_from_list(UsableIndexes),
+            indexes_of_type => couch_util:set_from_list(IndexesOfType)
         },
     Options = [{use_index, []}, {allow_fallback, false}],
     meck:expect(mango_selector, normalize, [selector], meck:val(normalized_selector)),
@@ -782,7 +780,7 @@ extract_candidate_indexes_test_() ->
     }.
 
 t_extract_candidate_indexes_empty(_) ->
-    Indexes = sets:new([{version, 2}]),
+    Indexes = couch_util:new_set(),
     UsabilityMap = [],
     Trace =
         #{
@@ -804,7 +802,7 @@ t_extract_candidate_indexes_empty(_) ->
     ?assertEqual(Candidates, extract_candidate_indexes(Cursor)).
 
 t_extract_candidate_indexes_singleton(_) ->
-    Indexes = set_from_list([winner]),
+    Indexes = couch_util:set_from_list([winner]),
     UsabilityMap = [{winner, {true, #{reason => []}}}],
     Trace =
         #{
@@ -840,15 +838,17 @@ t_extract_candidate_indexes_user_specified(_) ->
         ],
     Trace =
         #{
-            all_indexes => set_from_list([
+            all_indexes => couch_util:set_from_list([
                 winner, Partial, Partitioned, NotUsable, Filtered, Unfavored
             ]),
-            global_indexes => set_from_list([winner, Partitioned, NotUsable, Filtered, Unfavored]),
-            partition_indexes => set_from_list([winner, NotUsable, Filtered, Unfavored]),
-            usable_indexes => set_from_list([winner, Filtered, Unfavored]),
+            global_indexes => couch_util:set_from_list([
+                winner, Partitioned, NotUsable, Filtered, Unfavored
+            ]),
+            partition_indexes => couch_util:set_from_list([winner, NotUsable, Filtered, Unfavored]),
+            usable_indexes => couch_util:set_from_list([winner, Filtered, Unfavored]),
             usability_map => UsabilityMap,
-            filtered_indexes => set_from_list([winner, Unfavored]),
-            indexes_of_type => set_from_list([winner])
+            filtered_indexes => couch_util:set_from_list([winner, Unfavored]),
+            indexes_of_type => couch_util:set_from_list([winner])
         },
     Cursor =
         #cursor{
@@ -940,7 +940,7 @@ t_extract_candidate_indexes_regular(_) ->
     ],
     Trace =
         #{
-            all_indexes => set_from_list([
+            all_indexes => couch_util:set_from_list([
                 winner,
                 Partial1,
                 Partial2,
@@ -953,7 +953,7 @@ t_extract_candidate_indexes_regular(_) ->
                 Usable2,
                 Usable3
             ]),
-            global_indexes => set_from_list([
+            global_indexes => couch_util:set_from_list([
                 winner,
                 Partitioned1,
                 Partitioned2,
@@ -964,17 +964,17 @@ t_extract_candidate_indexes_regular(_) ->
                 Usable2,
                 Usable3
             ]),
-            partition_indexes => set_from_list([
+            partition_indexes => couch_util:set_from_list([
                 winner, NotUsable, Unfavored1, Unfavored2, Usable1, Usable2, Usable3
             ]),
-            usable_indexes => set_from_list([
+            usable_indexes => couch_util:set_from_list([
                 winner, Unfavored1, Unfavored2, Usable1, Usable2, Usable3
             ]),
             usability_map => UsabilityMap,
-            filtered_indexes => set_from_list([
+            filtered_indexes => couch_util:set_from_list([
                 winner, Unfavored1, Unfavored2, Usable1, Usable2, Usable3
             ]),
-            indexes_of_type => set_from_list([winner, Usable1, Usable2, Usable3]),
+            indexes_of_type => couch_util:set_from_list([winner, Usable1, Usable2, Usable3]),
             sorted_index_ranges => SortedIndexRanges
         },
     Cursor =
@@ -1193,7 +1193,7 @@ explain_test_() ->
 
 t_explain_empty(_) ->
     Selector = {[]},
-    Indexes = sets:new([{version, 2}]),
+    Indexes = couch_util:new_set(),
     Trace =
         #{
             all_indexes => Indexes,
@@ -1241,7 +1241,7 @@ t_explain_regular(_) ->
         type = <<"special">>, name = index, def = all_docs, dbname = db, partitioned = partitioned
     },
     Selector = {[]},
-    Indexes = set_from_list([Index]),
+    Indexes = couch_util:set_from_list([Index]),
     Fields = some_fields,
     Trace =
         #{

src/mango/src/mango_cursor_nouveau.erl

@@ -108,7 +108,7 @@ execute(Cursor, UserFun, UserAcc) ->
         user_acc = UserAcc,
         fields = Cursor#cursor.fields,
         execution_stats = mango_execution_stats:log_start(Stats),
-        documents_seen = sets:new([{version, 2}])
+        documents_seen = couch_util:new_set()
     },
     try
         case Query of