diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 99f8ea9b468d..d1a489ebbbbb 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -1240,7 +1240,17 @@ handle_info(Done = #loader_done{worker_pid=WPid, table_name=Tab}, State0) -> DelState = State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)}, cast({disc_load, Tab, ram_only}), DelState; - {_, []} -> %% Table deleted or not loaded anywhere + {_, []} -> + case val({Tab, load_by_force}) of + true -> + %% Network load could have failed, and user forced the load, + %% retry loading again from disk + %% TODO: Can this go into infinite loop? + cast({disc_load, Tab, forced_by_user}); + _ -> + ok + end, + %% Table deleted or not loaded anywhere State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)} end end, diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile index e07f7063afff..733a9ca72ba6 100644 --- a/lib/mnesia/test/Makefile +++ b/lib/mnesia/test/Makefile @@ -56,7 +56,8 @@ MODULES= \ ext_test \ ext_test_server \ mnesia_index_plugin_test \ - mnesia_external_backend_test + mnesia_external_backend_test \ + gen_tcp_blocking_dist DocExamplesDir := ../doc/src/ @@ -77,7 +78,7 @@ ExampleModules = \ ERL_FILES= $(MODULES:%=%.erl) $(DocExampleModules:%=$(DocExamplesDir)/%.erl) $(ExampleModules:%=$(ExamplesDir)/%.erl) -HRL_FILES= mnesia_test_lib.hrl ext_test_server.hrl $(DocExamplesHrl:%=$(DocExamplesDir)/%) +HRL_FILES= mnesia_test_lib.hrl ext_test_server.hrl gen_tcp_blocking_dist.hrl $(DocExamplesHrl:%=$(DocExamplesDir)/%) TARGET_FILES= \ $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(DocExampleModules:%=$(EBIN)/%.$(EMULATOR)) $(ExampleModules:%=$(EBIN)/%.$(EMULATOR)) @@ -93,6 +94,7 @@ RELSYSDIR = $(RELEASE_PATH)/mnesia_test # FLAGS # ---------------------------------------------------- #ERL_COMPILE_FLAGS += +ERL_COMPILE_FLAGS += +nowarn_missing_spec_documented -Werror ERL_COMPILE_FLAGS := $(filter-out +deterministic,$(ERL_COMPILE_FLAGS)) EBIN = . diff --git a/lib/mnesia/test/ext_test.erl b/lib/mnesia/test/ext_test.erl index 7d5795551194..a4cc4238cfcd 100644 --- a/lib/mnesia/test/ext_test.erl +++ b/lib/mnesia/test/ext_test.erl @@ -121,7 +121,7 @@ delete_table(Alias, Tab) -> try error_if_not_initialized() of ok -> call({?FUNCTION_NAME, Alias, Tab}) - catch error : {backend_not_initialized, _} = Reason -> + catch error : {backend_not_initialized, _} = _Reason -> ok end. @@ -253,8 +253,8 @@ repair_continuation(Cont, Ms) -> call(Req) -> error_if_not_initialized(), case gen_server:call({global, mnesia_test_lib:get_ext_test_server_name()}, Req) of - #exception{c = Class, r = Reason, st = ST} = Ex -> - ?DBG("call ~p resulted in an exception: ~p~n", [Req, Ex]), + #exception{c = Class, r = Reason, st = ST} = _Ex -> + ?DBG("call ~p resulted in an exception: ~p~n", [Req, _Ex]), erlang:raise(Class, Reason, ST); Res -> Res diff --git a/lib/mnesia/test/ext_test_server.erl b/lib/mnesia/test/ext_test_server.erl index 22a1ec8f0aeb..d806a097345f 100644 --- a/lib/mnesia/test/ext_test_server.erl +++ b/lib/mnesia/test/ext_test_server.erl @@ -41,8 +41,8 @@ init(_) -> create_table(ext_ram_copies, Tab, Props, #state{tables = Tables} = State) when is_atom(Tab) -> case maps:get(Tab, Tables, undefined) of - #table{state = opened, tid = Tid} -> - ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tab), Tid]), + #table{state = opened, tid = _Tid} -> + ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tab), _Tid]), {ok, State}; _ -> ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p~n", [tab_to_list(Tab)]), @@ -62,10 +62,10 @@ create_table(ext_disc_only_copies, Tab, Props, #state{tables = Tables} = State) ?DBG("create_table Alias: ext_disc_only_copies after dets:open_file, Tab: ~p~n", [tab_to_list(Tab)]), {ok, State#state{tables = maps:put(Tab, #table{state = opened, tid = Tab}, Tables)}} end; -create_table(ext_ram_copies, Tag={Tab, index, {_Where, Type}}, _Opts, #state{tables = Tables} = State) -> +create_table(ext_ram_copies, Tag={_Tab, index, {_Where, Type}}, _Opts, #state{tables = Tables} = State) -> case maps:get(Tag, Tables, undefined) of - #table{state = opened, tid = Tid} -> - ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tag), Tid]), + #table{state = opened, tid = _Tid} -> + ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tag), _Tid]), {ok, State}; _ -> ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p~n", [tab_to_list(Tag)]), @@ -73,7 +73,7 @@ create_table(ext_ram_copies, Tag={Tab, index, {_Where, Type}}, _Opts, #state{tab ?DBG("create_table, Alias, ext_ram_copies, Tab: ~p(~p)~n", [tab_to_list(Tag), Tid]), {ok, State#state{tables = maps:put(Tag, #table{state = opened, tid = Tid}, Tables)}} end; -create_table(ext_disc_only_copies, Tag={Tab, index, {_Where, Type}}, _Opts, #state{tables = Tables} = State) -> +create_table(ext_disc_only_copies, Tag={_Tab, index, {_Where, Type}}, _Opts, #state{tables = Tables} = State) -> case maps:get(Tag, Tables, undefined) of #table{state = opened, tid = Tag} -> ?DBG("create_table, Alias: ext_disc_only_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tag), Tag]), @@ -85,10 +85,10 @@ create_table(ext_disc_only_copies, Tag={Tab, index, {_Where, Type}}, _Opts, #sta ?DBG("create_table Alias: ext_disc_only_copies after dets:open_file, Tab: ~p~n", [tab_to_list(Tag)]), {ok, State#state{tables = maps:put(Tag, #table{state = opened, tid = Tag}, Tables)}} end; -create_table(ext_ram_copies, Tag={_Tab, retainer, ChkPName}, _Opts, #state{tables = Tables} = State) -> +create_table(ext_ram_copies, Tag={_Tab, retainer, _ChkPName}, _Opts, #state{tables = Tables} = State) -> case maps:get(Tag, Tables, undefined) of - #table{state = opened, tid = Tid} -> - ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tag), Tid]), + #table{state = opened, tid = _Tid} -> + ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tag), _Tid]), {ok, State}; _ -> ?DBG("create_table, Alias: ext_ram_copies, Tab: ~p~n", [tab_to_list(Tag)]), @@ -96,7 +96,7 @@ create_table(ext_ram_copies, Tag={_Tab, retainer, ChkPName}, _Opts, #state{table ?DBG("create_table, Alias, ext_ram_copies, Tab: ~p(~p)~n", [tab_to_list(Tag), Tid]), {ok, State#state{tables = maps:put(Tag, #table{state = opened, tid = Tid}, Tables)}} end; -create_table(ext_disc_only_copies, Tag={_Tab, retainer, ChkPName}, _Opts, #state{tables = Tables} = State) -> +create_table(ext_disc_only_copies, Tag={_Tab, retainer, _ChkPName}, _Opts, #state{tables = Tables} = State) -> case maps:get(Tag, Tables, undefined) of #table{state = opened, tid = Tag} -> ?DBG("create_table, Alias: ext_disc_only_copies, Tab: ~p(~p) is already opened~n", [tab_to_list(Tag), Tag]), @@ -109,15 +109,15 @@ create_table(ext_disc_only_copies, Tag={_Tab, retainer, ChkPName}, _Opts, #state {ok, State#state{tables = maps:put(Tag, #table{state = opened, tid = Tag}, Tables)}} end. -receive_data(Data, ext_ram_copies, Name, Sender, {Name, Tab, Sender} = MnesiaState, State) -> +receive_data(Data, ext_ram_copies, Name, Sender, {Name, Tab, Sender} = _MnesiaState, State) -> ?DBG({Data, ext_ram_copies, Name, Sender, {Name, tab_to_list(Tab), Sender}}), true = ets:insert(tab_to_tid(Tab, State), Data), {more, State}; -receive_data(Data, ext_disc_only_copies, Name, Sender, {Name, Tab, Sender} = MnesiaState, State) -> +receive_data(Data, ext_disc_only_copies, Name, Sender, {Name, Tab, Sender} = _MnesiaState, State) -> ?DBG({Data, ext_disc_only_copies, Name, Sender, {Name, tab_to_list(Tab), Sender}}), ok = dets:insert(tab_to_tid(Tab, State), Data), {more, State}; -receive_data(Data, Alias, Tab, Sender, {Name, Sender} = MnesiaState, State) -> +receive_data(Data, Alias, Tab, Sender, {Name, Sender} = _MnesiaState, State) -> ?DBG({Data, Alias, tab_to_list(Tab), State}), receive_data(Data, Alias, Tab, Sender, {Name, Tab, Sender}, State). @@ -162,7 +162,7 @@ handle_call({delete_table, ext_ram_copies, Tab}, _From, #state{tables = Tables} case ?TRY(ets:delete(Tid)) of #exception{} = Res -> {reply, Res, State}; - Res -> + _Res -> NewState = State#state{tables = maps:remove(Tab, Tables)}, {reply, ok, NewState} end; @@ -395,8 +395,8 @@ handle_call({repair_continuation, ext_ram_copies, Cont, Ms}, _From, State) -> Res = ?TRY(ets:repair_continuation(Cont, Ms)), {reply, Res, State}. -terminate(Reason, _State) -> - ?DBG(Reason). +terminate(_Reason, _State) -> + ?DBG(_Reason). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -425,4 +425,4 @@ tab_to_filename(Tab) -> tab_to_tid(Tab, #state{tables = Tables}) -> Table = maps:get(Tab, Tables), - Table#table.tid. \ No newline at end of file + Table#table.tid. diff --git a/lib/mnesia/test/gen_tcp_blocking_dist.erl b/lib/mnesia/test/gen_tcp_blocking_dist.erl new file mode 100644 index 000000000000..f739439072ed --- /dev/null +++ b/lib/mnesia/test/gen_tcp_blocking_dist.erl @@ -0,0 +1,910 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2026. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(gen_tcp_blocking_dist). + +%% +%% This is a modification of gen_tcp_dist that adds possibility to simulate +%% network outage between nodes. +%% This currently behaves in similar way to inet_tcp_proxy: https://github.com/rabbitmq/inet_tcp_proxy +%% When on Node1 we block Node2, we: +%% - stop consuming distribution messages (we allow them to be queued, but not sent, they will be +%% sent in bulk when node is unblocked later) +%% - stop sending ticks +%% - raise exit exception from setup/5 if node we want to connect to is blocked +%% - raise exit exception after handshake with blocked node completes +%% + +%% Public interface for blocking network communication +-export([block_peer/2, unblock_peer/2, + block_pair/2, unblock_pair/2, + has_network_blocker/0, + set_peer_ref/2, get_peer_ref/1, get_peer_ref/2]). + +%% gen_tcp_dist interface +-export([listen/1, accept/1, accept_connection/5, + setup/5, close/1, select/1, is_node_name/1, + address/0]). + +%% Optional +-export([setopts/2, getopts/2]). + +%% internal exports + +-export([dist_cntrlr_setup/1, dist_cntrlr_input_setup/4, + dist_cntrlr_tick_handler/2]). + +-export([accept_loop/2,do_accept/6,do_setup/6]). + +%% Internal exports for rpc calls +-export([block_internal/1, unblock_internal/1]). + +-import(error_logger,[error_msg/2]). + +-include("gen_tcp_blocking_dist.hrl"). + +-include_lib("kernel/include/net_address.hrl"). + +-include_lib("kernel/include/dist.hrl"). +-include_lib("kernel/include/dist_util.hrl"). + +%% ------------------------------------------------------------ +%% Block outgoing communication (and ticks) from node +%% 'From' to node 'To'. This is a one way block. +%% ------------------------------------------------------------ +block_peer(From, To) when From =:= node() -> + ok = block_internal(To); +block_peer(From, To) -> + ok = remote_call(From, ?MODULE, block_internal, [To]). + +%% ------------------------------------------------------------ +%% Unblock outgoing communication (and ticks) from node +%% 'From' to node 'To'. This is a one way unblock. +%% ------------------------------------------------------------ +unblock_peer(From, To) when From =:= node() -> + ok = unblock_internal(To); +unblock_peer(From, To) -> + ok = remote_call(From, ?MODULE, unblock_internal, [To]). + +%% ------------------------------------------------------------ +%% Block outgoing communication (and ticks) between 2 nodes. +%% This is a two way block. +%% ------------------------------------------------------------ +block_pair(Node1, Node2) when Node1 =:= node() -> + ok = block_peer(Node2, Node1), + %% Block local outgoing communication last + ok = block_peer(Node1, Node2); +block_pair(Node1, Node2) -> + ok = block_peer(Node1, Node2), + ok = block_peer(Node2, Node1). + +%% ------------------------------------------------------------ +%% Unblock outgoing communication (and ticks) between 2 nodes. +%% This is a two way unblock. +%% ------------------------------------------------------------ +unblock_pair(Node1, Node2) when Node2 =:= node() -> + %% Unblock local outgoing communication first + ok = unblock_peer(Node2, Node1), + ok = unblock_peer(Node1, Node2); +unblock_pair(Node1, Node2) -> + ok = unblock_peer(Node1, Node2), + ok = unblock_peer(Node2, Node1). + +%% ------------------------------------------------------------ +%% Internal stuff for network blocking +%% ------------------------------------------------------------ +block_internal(Node) -> + persistent_term:put({?MODULE, Node}, true). + +unblock_internal(Node) -> + case persistent_term:erase({?MODULE, Node}) of + true -> + case lists:keysearch(Node, 1, erlang:system_info(dist_ctrl)) of + {value, {Node, Pid}} -> + %% Notify output handler to flush data + Pid ! allow, + ok; + _ -> + ok + end; + false -> + ok + end. + +is_blocked(Node) -> + persistent_term:get({?MODULE, Node}, false). + +remote_call(Node, M, F, A) -> + case get_peer_ref(Node, undefined) of + undefined -> + rpc:call(Node, M, F, A); + Peer -> + peer:call(Peer, M, F, A) + end. + +%% ------------------------------------------------------------ +%% Check if network blocker is enabled on current node +%% ------------------------------------------------------------ +has_network_blocker() -> + Args = init:get_arguments(), + lists:all(fun(Elem) -> lists:member(Elem, Args) end, ?NETWORK_BLOCKER_DIST_OPTS_PARSED). + +set_peer_ref(Node, Ref) -> + persistent_term:put({peer, Node}, Ref). + +get_peer_ref(Node) -> + persistent_term:get({peer, Node}). + +get_peer_ref(Node, Default) -> + persistent_term:get({peer, Node}, Default). + +%% gen_tcp_dist interface starts here: +%% ------------------------------------------------------------ +%% Select this protocol based on node name +%% select(Node) => Bool +%% ------------------------------------------------------------ + +select(Node) -> + case split_node(atom_to_list(Node), $@, []) of + [_, Host] -> + case inet:getaddr(Host, inet) of + {ok,_} -> true; + _ -> false + end; + _ -> false + end. + +%% ------------------------------------------------------------ +%% Get the address family that this distribution uses +%% ------------------------------------------------------------ +address() -> + get_tcp_address(). + +%% ------------------------------------------------------------ +%% Create the listen socket, i.e. the port that this erlang +%% node is accessible through. +%% ------------------------------------------------------------ + +listen(Name) -> + case do_listen([binary, {active, false}, {packet,2}, {reuseaddr, true}]) of + {ok, Socket} -> + TcpAddress = get_tcp_address(Socket), + {_,Port} = TcpAddress#net_address.address, + ErlEpmd = net_kernel:epmd_module(), + case ErlEpmd:register_node(Name, Port) of + {ok, Creation} -> + {ok, {Socket, TcpAddress, Creation}}; + Error -> + Error + end; + Error -> + Error + end. + +do_listen(Options) -> + {First,Last} = case application:get_env(kernel,inet_dist_listen_min) of + {ok,N} when is_integer(N) -> + case application:get_env(kernel, + inet_dist_listen_max) of + {ok,M} when is_integer(M) -> + {N,M}; + _ -> + {N,N} + end; + _ -> + {0,0} + end, + do_listen(First, Last, listen_options([{backlog,128}|Options])). + +do_listen(First,Last,_) when First > Last -> + {error,eaddrinuse}; +do_listen(First,Last,Options) -> + case gen_tcp:listen(First, Options) of + {error, eaddrinuse} -> + do_listen(First+1,Last,Options); + Other -> + Other + end. + +listen_options(Opts0) -> + Opts1 = + case application:get_env(kernel, inet_dist_use_interface) of + {ok, Ip} -> + [{ip, Ip} | Opts0]; + _ -> + Opts0 + end, + case application:get_env(kernel, inet_dist_listen_options) of + {ok,ListenOpts} -> + ListenOpts ++ Opts1; + _ -> + Opts1 + end. + + +%% ------------------------------------------------------------ +%% Accepts new connection attempts from other Erlang nodes. +%% ------------------------------------------------------------ + +accept(Listen) -> + spawn_opt(?MODULE, accept_loop, [self(), Listen], [link, {priority, max}]). + +accept_loop(Kernel, Listen) -> + ?trace("~p~n",[{?MODULE, accept_loop, self()}]), + case gen_tcp:accept(Listen) of + {ok, Socket} -> + DistCtrl = spawn_dist_cntrlr(Socket), + ?trace("~p~n",[{?MODULE, accept_loop, accepted, Socket, DistCtrl, self()}]), + flush_controller(DistCtrl, Socket), + gen_tcp:controlling_process(Socket, DistCtrl), + flush_controller(DistCtrl, Socket), + Kernel ! {accept,self(),DistCtrl,inet,tcp}, + receive + {Kernel, controller, Pid} -> + call_ctrlr(DistCtrl, {supervisor, Pid}), + Pid ! {self(), controller}; + {Kernel, unsupported_protocol} -> + exit(unsupported_protocol) + end, + accept_loop(Kernel, Listen); + Error -> + exit(Error) + end. + +flush_controller(Pid, Socket) -> + receive + {tcp, Socket, Data} -> + Pid ! {tcp, Socket, Data}, + flush_controller(Pid, Socket); + {tcp_closed, Socket} -> + Pid ! {tcp_closed, Socket}, + flush_controller(Pid, Socket) + after 0 -> + ok + end. + +%% ------------------------------------------------------------ +%% Accepts a new connection attempt from another Erlang node. +%% Performs the handshake with the other side. +%% ------------------------------------------------------------ + +accept_connection(AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) -> + spawn_opt(?MODULE, do_accept, + [self(), AcceptPid, DistCtrl, MyNode, Allowed, SetupTime], + dist_util:net_ticker_spawn_options()). + +do_accept(Kernel, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) -> + ?trace("~p~n",[{?MODULE, do_accept, self(), MyNode}]), + receive + {AcceptPid, controller} -> + Timer = dist_util:start_timer(SetupTime), + case check_ip(DistCtrl) of + true -> + HSData0 = hs_data_common(DistCtrl), + HSData = HSData0#hs_data{kernel_pid = Kernel, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + this_flags = 0, + allowed = Allowed}, + dist_util:handshake_other_started(HSData); + {false,IP} -> + error_msg("** Connection attempt from " + "disallowed IP ~w ** ~n", [IP]), + ?shutdown(no_node) + end + end. + +%% we may not always want the nodelay behaviour +%% for performance reasons + +nodelay() -> + case application:get_env(kernel, dist_nodelay) of + undefined -> + {nodelay, true}; + {ok, true} -> + {nodelay, true}; + {ok, false} -> + {nodelay, false}; + _ -> + {nodelay, true} + end. + +%% ------------------------------------------------------------ +%% Setup a new connection to another Erlang node. +%% Performs the handshake with the other side. +%% ------------------------------------------------------------ + +setup(Node, Type, MyNode, LongOrShortNames,SetupTime) -> + spawn_opt(?MODULE, do_setup, + [self(), Node, Type, MyNode, LongOrShortNames, SetupTime], + dist_util:net_ticker_spawn_options()). + +do_setup(Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) -> + ?trace("~p~n",[{?MODULE, do_setup, self(), Node}]), + case is_blocked(Node) of + true -> + ?trace("(~p): Node: ~p blocked, aborting setup/5~n", [node(), Node]), + ?shutdown(Node); + false -> + [Name, Address] = splitnode(Node, LongOrShortNames), + case inet:getaddr(Address, inet) of + {ok, Ip} -> + Timer = dist_util:start_timer(SetupTime), + ErlEpmd = net_kernel:epmd_module(), + case ErlEpmd:port_please(Name, Ip) of + {port, TcpPort, Version} -> + ?trace("port_please(~p) -> version ~p~n", + [Node,Version]), + dist_util:reset_timer(Timer), + case + gen_tcp:connect( + Ip, TcpPort, + connect_options([binary, {active, false}, {packet, 2}])) + of + {ok, Socket} -> + DistCtrl = spawn_dist_cntrlr(Socket), + call_ctrlr(DistCtrl, {supervisor, self()}), + flush_controller(DistCtrl, Socket), + gen_tcp:controlling_process(Socket, DistCtrl), + flush_controller(DistCtrl, Socket), + HSData0 = hs_data_common(DistCtrl), + HSData = HSData0#hs_data{kernel_pid = Kernel, + other_node = Node, + this_node = MyNode, + socket = DistCtrl, + timer = Timer, + this_flags = 0, + other_version = Version, + request_type = Type}, + dist_util:handshake_we_started(HSData); + _ -> + %% Other Node may have closed since + %% port_please ! + ?trace("other node (~p) " + "closed since port_please.~n", + [Node]), + ?shutdown(Node) + end; + _ -> + ?trace("port_please (~p) " + "failed.~n", [Node]), + ?shutdown(Node) + end; + _Other -> + ?trace("inet_getaddr(~p) " + "failed (~p).~n", [Node,_Other]), + ?shutdown(Node) + end + end. + +connect_options(Opts) -> + case application:get_env(kernel, inet_dist_connect_options) of + {ok,ConnectOpts} -> + ConnectOpts ++ Opts; + _ -> + Opts + end. + +%% +%% Close a socket. +%% +close(Listen) -> + gen_tcp:close(Listen). + + +%% If Node is illegal terminate the connection setup!! +splitnode(Node, LongOrShortNames) -> + case split_node(atom_to_list(Node), $@, []) of + [Name|Tail] when Tail =/= [] -> + Host = lists:append(Tail), + case split_node(Host, $., []) of + [_] when LongOrShortNames =:= longnames -> + case inet:parse_address(Host) of + {ok, _} -> + [Name, Host]; + _ -> + error_msg("** System running to use " + "fully qualified " + "hostnames **~n" + "** Hostname ~ts is illegal **~n", + [Host]), + ?shutdown(Node) + end; + L when length(L) > 1, LongOrShortNames =:= shortnames -> + error_msg("** System NOT running to use fully qualified " + "hostnames **~n" + "** Hostname ~ts is illegal **~n", + [Host]), + ?shutdown(Node); + _ -> + [Name, Host] + end; + [_] -> + error_msg("** Nodename ~p illegal, no '@' character **~n", + [Node]), + ?shutdown(Node); + _ -> + error_msg("** Nodename ~p illegal **~n", [Node]), + ?shutdown(Node) + end. + +split_node([Chr|T], Chr, Ack) -> [lists:reverse(Ack)|split_node(T, Chr, [])]; +split_node([H|T], Chr, Ack) -> split_node(T, Chr, [H|Ack]); +split_node([], _, Ack) -> [lists:reverse(Ack)]. + +%% ------------------------------------------------------------ +%% Fetch local information about a Socket. +%% ------------------------------------------------------------ +get_tcp_address(Socket) -> + {ok, Address} = inet:sockname(Socket), + NetAddr = get_tcp_address(), + NetAddr#net_address{address = Address}. + +get_tcp_address() -> + {ok, Host} = inet:gethostname(), + #net_address { + host = Host, + protocol = tcp, + family = inet + }. + +%% ------------------------------------------------------------ +%% Do only accept new connection attempts from nodes at our +%% own LAN, if the check_ip environment parameter is true. +%% ------------------------------------------------------------ +check_ip(DistCtrl) -> + case application:get_env(check_ip) of + {ok, true} -> + case get_ifs(DistCtrl) of + {ok, IFs, IP} -> + check_ip(IFs, IP); + _ -> + ?shutdown(no_node) + end; + _ -> + true + end. + +get_ifs(DistCtrl) -> + Socket = call_ctrlr(DistCtrl, socket), + case inet:peername(Socket) of + {ok, {IP, _}} -> + case inet:getif(Socket) of + {ok, IFs} -> {ok, IFs, IP}; + Error -> Error + end; + Error -> + Error + end. + +check_ip([{OwnIP, _, Netmask}|IFs], PeerIP) -> + case {inet_tcp:mask(Netmask, PeerIP), inet_tcp:mask(Netmask, OwnIP)} of + {M, M} -> true; + _ -> check_ip(IFs, PeerIP) + end; +check_ip([], PeerIP) -> + {false, PeerIP}. + +is_node_name(Node) when is_atom(Node) -> + case split_node(atom_to_list(Node), $@, []) of + [_, _Host] -> true; + _ -> false + end; +is_node_name(_Node) -> + false. + +hs_data_common(DistCtrl) -> + TickHandler = call_ctrlr(DistCtrl, tick_handler), + Socket = call_ctrlr(DistCtrl, socket), + RejectFlags = case init:get_argument(gen_tcp_dist_reject_flags) of + {ok,[[Flags]]} -> list_to_integer(Flags); + _ -> #hs_data{}#hs_data.reject_flags + end, + #hs_data{f_send = send_fun(), + f_recv = recv_fun(), + f_setopts_pre_nodeup = setopts_pre_nodeup_fun(), + f_setopts_post_nodeup = setopts_post_nodeup_fun(), + f_getll = getll_fun(), + f_handshake_complete = handshake_complete_fun(), + f_address = address_fun(), + mf_setopts = setopts_fun(DistCtrl, Socket), + mf_getopts = getopts_fun(DistCtrl, Socket), + mf_getstat = getstat_fun(DistCtrl, Socket), + mf_tick = tick_fun(DistCtrl, TickHandler), + reject_flags = RejectFlags}. + +%%% ------------------------------------------------------------ +%%% Distribution controller processes +%%% ------------------------------------------------------------ + +%% +%% There will be five parties working together when the +%% connection is up: +%% - The gen_tcp socket. Providing a tcp/ip connection +%% to the other node. +%% - The output handler. It will dispatch all outgoing +%% traffic from the VM to the gen_tcp socket. This +%% process is registered as distribution controller +%% for this channel with the VM. +%% - The input handler. It will dispatch all incoming +%% traffic from the gen_tcp socket to the VM. This +%% process is also the socket owner and receives +%% incoming traffic using active-N. +%% - The tick handler. Dispatches asynchronous tick +%% requests to the socket. It executes on max priority +%% since it is important to get ticks through to the +%% other end. +%% - The channel supervisor (provided by dist_util). It +%% monitors traffic. Issue tick requests to the tick +%% handler when no outgoing traffic is seen and bring +%% the connection down if no incoming traffic is seen. +%% This process also executes on max priority. +%% +%% These parties are linked together so should one +%% of them fail, all of them are terminated and the +%% connection is taken down. +%% + +%% In order to avoid issues with lingering signal binaries +%% we enable off-heap message queue data as well as fullsweep +%% after 0. The fullsweeps will be cheap since we have more +%% or less no live data. +-define(DIST_CNTRL_COMMON_SPAWN_OPTS, + [{message_queue_data, off_heap}, + {fullsweep_after, 0}]). + +tick_fun(DistCtrl, TickHandler) -> + fun (Ctrl) when Ctrl == DistCtrl -> + TickHandler ! tick + end. + +getstat_fun(DistCtrl, Socket) -> + fun (Ctrl) when Ctrl == DistCtrl -> + case inet:getstat(Socket, [recv_cnt, send_cnt, send_pend]) of + {ok, Stat} -> + split_stat(Stat,0,0,0); + Error -> + Error + end + end. + +split_stat([{recv_cnt, R}|Stat], _, W, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_cnt, W}|Stat], R, _, P) -> + split_stat(Stat, R, W, P); +split_stat([{send_pend, P}|Stat], R, W, _) -> + split_stat(Stat, R, W, P); +split_stat([], R, W, P) -> + {ok, R, W, P}. + +setopts_fun(DistCtrl, Socket) -> + fun (Ctrl, Opts) when Ctrl == DistCtrl -> + setopts(Socket, Opts) + end. + +getopts_fun(DistCtrl, Socket) -> + fun (Ctrl, Opts) when Ctrl == DistCtrl -> + getopts(Socket, Opts) + end. + +setopts(S, Opts) -> + case [Opt || {K,_}=Opt <- Opts, + K =:= active orelse K =:= deliver orelse K =:= packet] of + [] -> inet:setopts(S,Opts); + Opts1 -> {error, {badopts,Opts1}} + end. + +getopts(S, Opts) -> + inet:getopts(S, Opts). + +send_fun() -> + fun (Ctrlr, Packet) -> + call_ctrlr(Ctrlr, {send, Packet}) + end. + +recv_fun() -> + fun (Ctrlr, Length, Timeout) -> + case call_ctrlr(Ctrlr, {recv, Length, Timeout}) of + {ok, Bin} when is_binary(Bin) -> + {ok, binary_to_list(Bin)}; + Other -> + Other + end + end. + +getll_fun() -> + fun (Ctrlr) -> + call_ctrlr(Ctrlr, getll) + end. + +address_fun() -> + fun (Ctrlr, Node) -> + case call_ctrlr(Ctrlr, {address, Node}) of + {error, no_node} -> %% No '@' or more than one '@' in node name. + ?shutdown(no_node); + Res -> + Res + end + end. + +setopts_pre_nodeup_fun() -> + fun (Ctrlr) -> + call_ctrlr(Ctrlr, pre_nodeup) + end. + +setopts_post_nodeup_fun() -> + fun (Ctrlr) -> + call_ctrlr(Ctrlr, post_nodeup) + end. + +handshake_complete_fun() -> + fun (Ctrlr, Node, DHandle) -> + call_ctrlr(Ctrlr, {handshake_complete, Node, DHandle}) + end. + +call_ctrlr(Ctrlr, Msg) -> + Ref = erlang:monitor(process, Ctrlr), + Ctrlr ! {Ref, self(), Msg}, + receive + {Ref, Res} -> + erlang:demonitor(Ref, [flush]), + Res; + {'DOWN', Ref, process, Ctrlr, Reason} -> + exit({dist_controller_exit, Reason}) + end. + +%% +%% The tick handler process writes a tick to the +%% socket when it receives a 'tick' message from +%% the connection supervisor. +%% +%% We are not allowed to block the connection +%% superviser when writing a tick and we also want +%% the tick to go through even during a heavily +%% loaded system. gen_tcp does not have a +%% non-blocking send operation exposed in its API +%% and we don't want to run the distribution +%% controller under high priority. Therefore this +%% separate process with max prio that dispatches +%% ticks. +%% +dist_cntrlr_tick_handler(Socket, Node0) -> + receive + tick -> + case is_blocked(Node0) of + true -> + %% Don't send tick... + ok; + false -> + %% May block due to busy port... + sock_send(Socket, "") + end; + {set_node, Node} -> + dist_cntrlr_tick_handler(Socket, Node); + _ -> + ok + end, + dist_cntrlr_tick_handler(Socket, Node0). + +spawn_dist_cntrlr(Socket) -> + spawn_opt(?MODULE, dist_cntrlr_setup, [Socket], + [{priority, max}] ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS). + +dist_cntrlr_setup(Socket) -> + TickHandler = spawn_opt(?MODULE, dist_cntrlr_tick_handler, + [Socket, undefined], + [link, {priority, max}] + ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS), + dist_cntrlr_setup_loop(Socket, TickHandler, undefined). + +%% +%% During the handshake phase we loop in dist_cntrlr_setup(). +%% When the connection is up we spawn an input handler and +%% continue as output handler. +%% +dist_cntrlr_setup_loop(Socket, TickHandler, Sup) -> + receive + {tcp_closed, Socket} -> + exit(connection_closed); + + {Ref, From, {supervisor, Pid}} -> + Res = link(Pid), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Pid); + + {Ref, From, tick_handler} -> + From ! {Ref, TickHandler}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, socket} -> + From ! {Ref, Socket}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {send, Packet}} -> + Res = gen_tcp:send(Socket, Packet), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {recv, Length, Timeout}} -> + Res = gen_tcp:recv(Socket, Length, Timeout), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, getll} -> + From ! {Ref, {ok, self()}}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {address, Node}} -> + Res = case inet:peername(Socket) of + {ok, Address} -> + case split_node(atom_to_list(Node), $@, []) of + [_,Host] -> + #net_address{address=Address,host=Host, + protocol=tcp, family=inet}; + _ -> + {error, no_node} + end + end, + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, pre_nodeup} -> + Res = inet:setopts(Socket, + [{active, false}, + {packet, 4}, + nodelay()]), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, post_nodeup} -> + Res = inet:setopts(Socket, + [{active, false}, + {packet, 4}, + nodelay()]), + From ! {Ref, Res}, + dist_cntrlr_setup_loop(Socket, TickHandler, Sup); + + {Ref, From, {handshake_complete, Node, DHandle}} -> + case is_blocked(Node) of + true -> + ?trace("(~p): Node: ~p blocked, aborting after handshake_complete~n", [node(), Node]), + exit({shutdown, blocked}); + false -> + From ! {Ref, ok}, + TickHandler ! {set_node, Node}, + %% Handshake complete! Begin dispatching traffic... + + %% We use separate process for dispatching input. This + %% is not necessary, but it enables parallel execution + %% of independent work loads at the same time as it + %% simplifies the the implementation... + InputHandler = spawn_opt(?MODULE, dist_cntrlr_input_setup, + [DHandle, Socket, Node, Sup], + [link] ++ ?DIST_CNTRL_COMMON_SPAWN_OPTS), + + flush_controller(InputHandler, Socket), + gen_tcp:controlling_process(Socket, InputHandler), + flush_controller(InputHandler, Socket), + + ok = erlang:dist_ctrl_input_handler(DHandle, InputHandler), + + InputHandler ! DHandle, + + %% From now on we execute on normal priority + process_flag(priority, normal), + erlang:dist_ctrl_get_data_notification(DHandle), + dist_cntrlr_output_loop(DHandle, Socket, Node) + end + end. + +%% We use active 10 for good throughput while still +%% maintaining back-pressure if the input controller +%% isn't able to handle all incoming messages... +-define(ACTIVE_INPUT, 10). + +dist_cntrlr_input_setup(DHandle, Socket, Node, Sup) -> + link(Sup), + %% Ensure we don't try to put data before we are registered + %% as input handler... + receive + DHandle -> + dist_cntrlr_input_loop(DHandle, Socket, Node, 0) + end. + +dist_cntrlr_input_loop(DHandle, Socket, Node, N) when N =< ?ACTIVE_INPUT/2 -> + inet:setopts(Socket, [{active, ?ACTIVE_INPUT - N}]), + dist_cntrlr_input_loop(DHandle, Socket, Node, ?ACTIVE_INPUT); +dist_cntrlr_input_loop(DHandle, Socket, Node, N) -> + receive + {tcp_closed, Socket} -> + %% Connection to remote node terminated... + exit(connection_closed); + + {tcp, Socket, Data} -> + %% Incoming data from remote node... + try erlang:dist_ctrl_put_data(DHandle, Data) + catch _ : _ -> death_row() + end, + dist_cntrlr_input_loop(DHandle, Socket, Node, N - 1); + + _ -> + %% Ignore... + dist_cntrlr_input_loop(DHandle, Socket, Node, N) + end. + +dist_cntrlr_send_data(DHandle, Socket) -> + case erlang:dist_ctrl_get_data(DHandle) of + none -> + erlang:dist_ctrl_get_data_notification(DHandle); + Data -> + sock_send(Socket, Data), + dist_cntrlr_send_data(DHandle, Socket) + end. + + +dist_cntrlr_output_loop(DHandle, Socket, Node) -> + receive + Msg when Msg =:= dist_data; Msg =:= allow -> + %% Outgoing data from this node... + case is_blocked(Node) of + true -> + %% Don't flush message from buffer if peer is blocked + dist_cntrlr_output_loop(DHandle, Socket, Node); + false -> + try dist_cntrlr_send_data(DHandle, Socket) + catch _ : _ -> death_row() + end, + dist_cntrlr_output_loop(DHandle, Socket, Node) + end; + + _ -> + %% Drop garbage message... + dist_cntrlr_output_loop(DHandle, Socket, Node) + + end. + +sock_send(Socket, Data) -> + try gen_tcp:send(Socket, Data) of + ok -> ok; + {error, Reason} -> death_row({send_error, Reason}) + catch + Type : Reason -> death_row({send_error, {Type, Reason}}) + end. + +death_row() -> + death_row(connection_closed). + +death_row(normal) -> + %% We do not want to exit with normal + %% exit reason since it won't bring down + %% linked processes... + death_row(); +death_row(Reason) -> + %% When the connection is on its way down operations + %% begin to fail. We catch the failures and call + %% this function waiting for termination. We should + %% be terminated by one of our links to the other + %% involved parties that began bringing the + %% connection down. By waiting for termination we + %% avoid altering the exit reason for the connection + %% teardown. We however limit the wait to 5 seconds + %% and bring down the connection ourselves if not + %% terminated... + receive after 5000 -> exit(Reason) end. diff --git a/lib/mnesia/test/gen_tcp_blocking_dist.hrl b/lib/mnesia/test/gen_tcp_blocking_dist.hrl new file mode 100644 index 000000000000..8275727efcc7 --- /dev/null +++ b/lib/mnesia/test/gen_tcp_blocking_dist.hrl @@ -0,0 +1,24 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2026. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% + +-define(NETWORK_BLOCKER_DIST_OPTS_PARSED, [{proto_dist, ["gen_tcp_blocking"]}]). +-define(NETWORK_BLOCKER_DIST_OPTS, ["-proto_dist gen_tcp_blocking"]). diff --git a/lib/mnesia/test/mnesia_dirty_access_test.erl b/lib/mnesia/test/mnesia_dirty_access_test.erl index 6ba2dd80c946..c7b32881bb93 100644 --- a/lib/mnesia/test/mnesia_dirty_access_test.erl +++ b/lib/mnesia/test/mnesia_dirty_access_test.erl @@ -919,19 +919,19 @@ add_table(CallFrom, AddNode, [Node1, Node2, Node3], Def) -> ?verify_mnesia([Node1, Node2, Node3], []). -tracer({trace_ts, From, send, Msg, To, {_,S,Ms}}) -> - io:format("~p:~p ~p(~p) >>~p ~w ~n",[S,Ms,From,node(From),To,Msg]); -tracer({trace_ts, Pid, 'receive', Msg, {_,S,Ms}}) -> - io:format("~p:~p ~p(~p) << ~w ~n",[S,Ms,Pid,node(Pid),Msg]); - -tracer({trace_ts, Pid, call, MFA, ST, {_,S,Ms}}) -> - io:format("~p:~p ~p(~p) ~w ~w ~n",[S,Ms,Pid,node(Pid),MFA, ST]); -tracer({trace_ts, Pid, return_from, MFA, Ret, {_,S,Ms}}) -> - io:format("~p:~p ~p(~p) ~w => ~w ~n",[S,Ms,Pid,node(Pid),MFA,Ret]); - -tracer(Msg) -> - io:format("UMsg ~p ~n",[Msg]), - ok. +%% tracer({trace_ts, From, send, Msg, To, {_,S,Ms}}) -> +%% io:format("~p:~p ~p(~p) >>~p ~w ~n",[S,Ms,From,node(From),To,Msg]); +%% tracer({trace_ts, Pid, 'receive', Msg, {_,S,Ms}}) -> +%% io:format("~p:~p ~p(~p) << ~w ~n",[S,Ms,Pid,node(Pid),Msg]); + +%% tracer({trace_ts, Pid, call, MFA, ST, {_,S,Ms}}) -> +%% io:format("~p:~p ~p(~p) ~w ~w ~n",[S,Ms,Pid,node(Pid),MFA, ST]); +%% tracer({trace_ts, Pid, return_from, MFA, Ret, {_,S,Ms}}) -> +%% io:format("~p:~p ~p(~p) ~w => ~w ~n",[S,Ms,Pid,node(Pid),MFA,Ret]); + +%% tracer(Msg) -> +%% io:format("UMsg ~p ~n",[Msg]), +%% ok. move_table_copy_1(suite) -> []; diff --git a/lib/mnesia/test/mnesia_durability_test.erl b/lib/mnesia/test/mnesia_durability_test.erl index 986d0efcb542..faad83c43bb5 100644 --- a/lib/mnesia/test/mnesia_durability_test.erl +++ b/lib/mnesia/test/mnesia_durability_test.erl @@ -47,6 +47,8 @@ master_on_non_local_tables/1, remote_force_load_with_local_master_node/1, master_node_with_ram_copy_2/1, master_node_with_ram_copy_3/1, + force_load_disc_copies_when_network_down/1, + force_load_disc_only_copies_when_network_down/1, dump_ram_copies/1, dump_disc_copies/1, dump_disc_only/1]). -include("mnesia_test_lib.hrl"). @@ -69,7 +71,7 @@ all() -> durability_of_disc_copies, durability_of_disc_only_copies]. -groups() -> +groups() -> [{load_tables, [], [load_latest_data, load_local_contents_directly, load_directly_when_all_are_ram_copiesA, @@ -95,8 +97,14 @@ groups() -> remote_force_load_with_local_master_node, master_node_with_ram_copy_2, master_node_with_ram_copy_3]}, {durability_of_dump_tables, [], - [dump_ram_copies, dump_disc_copies, dump_disc_only]}]. - + [dump_ram_copies, dump_disc_copies, dump_disc_only]}, + {load_tables_with_network_down, [], + [force_load_disc_copies_when_network_down, + force_load_disc_only_copies_when_network_down] + }]. + +init_per_group(load_tables_with_network_down, Config) -> + mnesia_test_lib:skip_if_no_network_blocker(Config); init_per_group(_GroupName, Config) -> Config. @@ -1268,6 +1276,60 @@ master_node_with_ram_copy_3(Config) when is_list(Config) -> ?verify_mnesia(Nodes, []). +force_load_disc_copies_when_network_down(Config) -> + [Node1, Node2] = Nodes = ?acquire_nodes(2, Config), + Table = ?FUNCTION_NAME, + Populate = fun(F, N) when N > 0 -> + mnesia:write({Table, N, []}), + F(F, N - 1); + (_F, _N) -> + ok + end, + ?match({atomic, ok}, mnesia:create_table(Table, [{disc_copies, Nodes}])), + ?match({atomic, ok}, mnesia:transaction(fun() -> Populate(Populate, 1_000_000) end)), + ?match(ok, mnesia:set_master_nodes([Node2])), + ?match(stopped, mnesia:stop()), + ?match(ok, mnesia:start()), + ?match(ok, mnesia_test_lib:block_peer(Node1, Node2)), + ?match({timeout, [Table]}, mnesia:wait_for_tables([Table], 0)), + ?match(pang, net_adm:ping(Node2)), + {Pid, Ref} = spawn_monitor(fun() -> yes = mnesia:force_load_table(Table) end), + receive + {'DOWN', Ref, process, Pid, _} -> + ok + after timer:seconds(15) -> + ct:fail("Pid: ~p stuck in:~n~p~n", + [Pid, process_info(Pid, current_stacktrace)]) + end. + +force_load_disc_only_copies_when_network_down(Config) -> + [Node1, Node2] = Nodes = ?acquire_nodes(2, Config), + Table = ?FUNCTION_NAME, + Populate = fun(F, N) when N > 0 -> + mnesia:write({Table, N, []}), + F(F, N - 1); + (_F, _N) -> + ok + end, + ?match({atomic, ok}, mnesia:create_table(Table, [{disc_only_copies, Nodes}])), + ?match({atomic, ok}, mnesia:transaction(fun() -> Populate(Populate, 500_000) end)), + ?match(ok, mnesia:set_master_nodes([Node2])), + ?match(stopped, mnesia:stop()), + ?match(ok, mnesia:start()), + ?match(ok, mnesia_test_lib:block_peer(Node1, Node2)), + ?match({timeout, [Table]}, mnesia:wait_for_tables([Table], 0)), + ?match(pang, net_adm:ping(Node2)), + ?match(ok, timer:sleep(timer:seconds(1))), + {Pid, Ref} = spawn_monitor(fun() -> yes = mnesia:force_load_table(Table) end), + receive + {'DOWN', Ref, process, Pid, _} -> + ok + after timer:seconds(15) -> + ct:fail("Pid: ~p stuck in:~n~p~n", + [Pid, [process_info(dets_server:get_pid(Table), K) || + K <- [monitored_by, current_stacktrace]]]) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/lib/mnesia/test/mnesia_evil_coverage_test.erl b/lib/mnesia/test/mnesia_evil_coverage_test.erl index 763d9054b7a6..56534c8ef46a 100644 --- a/lib/mnesia/test/mnesia_evil_coverage_test.erl +++ b/lib/mnesia/test/mnesia_evil_coverage_test.erl @@ -51,6 +51,8 @@ -export([info_check/8, index_size/1]). +-compile({nowarn_deprecated_function, {mnesia_registry, create_table, 2}}). + -define(cleanup(N, Config), mnesia_test_lib:prepare_test_case([{reload_appls, [mnesia]}], N, Config, ?FILE, ?LINE)). diff --git a/lib/mnesia/test/mnesia_external_backend_test.erl b/lib/mnesia/test/mnesia_external_backend_test.erl index 0e3d3072171d..a1264279ebcd 100644 --- a/lib/mnesia/test/mnesia_external_backend_test.erl +++ b/lib/mnesia/test/mnesia_external_backend_test.erl @@ -45,6 +45,8 @@ all() -> [ groups() -> []. +init_per_testcase(backup_and_restore_should_work_with_external_backend, _Conf) -> + {skip, "Not implemented"}; init_per_testcase(Func, Conf) -> file:delete("bup0.BUP"), file:delete("bup1.BUP"), diff --git a/lib/mnesia/test/mnesia_isolation_test.erl b/lib/mnesia/test/mnesia_isolation_test.erl index 759735a7ea02..47834b2a2e1a 100644 --- a/lib/mnesia/test/mnesia_isolation_test.erl +++ b/lib/mnesia/test/mnesia_isolation_test.erl @@ -629,7 +629,7 @@ sticky_sync(Config) when is_list(Config) -> end, %% Fill 1000 dc records. At the end all dc records should have value 1. - {Time, ok} = timer:tc(fun() -> lists:foreach(TestFun, lists:seq(1,200)) end), + {_Time, ok} = timer:tc(fun() -> lists:foreach(TestFun, lists:seq(1,200)) end), io:format("Written, check content~n",[]), All = fun() -> mnesia:select(dc, [ {{dc, '_', 0}, [] ,['$_']} ]) end, ?match({atomic, []}, rpc:call(N1, mnesia, sync_transaction, [All])), diff --git a/lib/mnesia/test/mnesia_registry_test.erl b/lib/mnesia/test/mnesia_registry_test.erl index c15b8e97af2b..a8b8742d578f 100644 --- a/lib/mnesia/test/mnesia_registry_test.erl +++ b/lib/mnesia/test/mnesia_registry_test.erl @@ -29,6 +29,8 @@ -include("mnesia_test_lib.hrl"). +-compile(nowarn_deprecated_function). + init_per_testcase(Func, Conf) -> mnesia_test_lib:init_per_testcase(Func, Conf). diff --git a/lib/mnesia/test/mnesia_test_lib.erl b/lib/mnesia/test/mnesia_test_lib.erl index d6f844e34a1b..6880fe5e361b 100644 --- a/lib/mnesia/test/mnesia_test_lib.erl +++ b/lib/mnesia/test/mnesia_test_lib.erl @@ -132,10 +132,19 @@ init_per_testcase/2, end_per_testcase/2, kill_tc/2, - get_ext_test_server_name/0 + get_ext_test_server_name/0, + get_peer_ref/1, + get_peer_ref/2, + has_network_blocker/0, + skip_if_no_network_blocker/1, + block_peer/2, + unblock_peer/2, + block_pair/2, + unblock_pair/2 ]). -include("mnesia_test_lib.hrl"). +-include("gen_tcp_blocking_dist.hrl"). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -241,9 +250,15 @@ node_start_link(Host, Name) -> node_start_link(Host, Name, Retries) -> Debug = atom_to_list(mnesia:system_info(debug)), - Args = ["-mnesia", "debug", Debug, - "-pa", filename:dirname(code:which(?MODULE)), - "-pa", filename:dirname(code:which(mnesia))], + Args0 = ["-mnesia", "debug", Debug, + "-pa", filename:dirname(code:which(?MODULE)), + "-pa", filename:dirname(code:which(mnesia))], + Args = case has_network_blocker() of + true -> + Args0 ++ ?NETWORK_BLOCKER_DIST_OPTS; + false -> + Args0 + end, case starter(Host, Name, Args) of {ok, NewNode} -> ?match(pong, net_adm:ping(NewNode)), @@ -265,7 +280,8 @@ node_start_link(Host, Name, Retries) -> end. starter(Host, Name, Args) -> - {ok, _, Node} = peer:start(#{host => Host, name => Name, args => Args}), + {ok, Peer, Node} = peer:start(#{host => Host, name => Name, args => Args, connection => 0}), + ok = persistent_term:put({peer, Node}, Peer), {ok, Node}. node_sup() -> @@ -623,6 +639,18 @@ prepare_test_case(Actions, N, Config, File, Line) -> NodeList3 = append_unique(NodeList1, NodeList2), This = node(), All = [This | lists:delete(This, NodeList3)], + case has_network_blocker() of + true -> + Pairs0 = [{A, B} || A <- All, B <- All, A < B], + {Local, Remote} = lists:partition(fun({A, B}) -> + A =:= node() orelse B =:= node() + end, Pairs0), + Pairs = Local ++ Remote, + %% Unlock local pairs first, so it works if we don't have peer refs + [unblock_pair(Node1, Node2) || {Node1, Node2} <- Pairs]; + false -> + ok + end, Selected = pick_nodes(N, All, File, Line), case diskless(Config) of true -> @@ -1094,3 +1122,33 @@ sort(W) -> get_ext_test_server_name() -> list_to_atom("ext_test_server_" ++ atom_to_list(node())). + +get_peer_ref(Node) -> + gen_tcp_blocking_dist:get_peer_ref(Node). + +get_peer_ref(Node, Default) -> + gen_tcp_blocking_dist:get_peer_ref(Node, Default). + +%% Simulate network outage, use this to block/unblock communication between 2 nodes +has_network_blocker() -> + gen_tcp_blocking_dist:has_network_blocker(). + +skip_if_no_network_blocker(Config) -> + case has_network_blocker() of + true -> + Config; + false -> + {skip, "Network blocker required"} + end. + +block_peer(From, To) -> + gen_tcp_blocking_dist:block_peer(From, To). + +unblock_peer(From, To) -> + gen_tcp_blocking_dist:unblock_peer(From, To). + +block_pair(Node1, Node2) -> + gen_tcp_blocking_dist:block_pair(Node1, Node2). + +unblock_pair(Node1, Node2) -> + gen_tcp_blocking_dist:unblock_pair(Node1, Node2). diff --git a/lib/mnesia/test/mt b/lib/mnesia/test/mt index f8bfdf09e618..c0a62b147798 100755 --- a/lib/mnesia/test/mt +++ b/lib/mnesia/test/mt @@ -1,5 +1,12 @@ #! /bin/sh -f -# ``Licensed under the Apache License, Version 2.0 (the "License"); +# +# %CopyrightBegin% +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright Ericsson AB 1999-2025. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # @@ -10,29 +17,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# The Initial Developer of the Original Code is Ericsson Utvecklings AB. -# Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings -# AB. All Rights Reserved.'' -# -# $Id$ +# +# %CopyrightEnd% # # # Author: Hakan Mattsson # Purpose: Simplified execution of the test suite # -# Usage: mt +# Usage: mt -#top=".." top="$ERL_TOP/lib/mnesia" h=`hostname` p="-pa $top/examples -pa $top/ebin -pa $top/test -mnesia_test_verbose true" log=test_log$$ latest=test_log_latest +if [ "$1" = "true" ]; then + dist_args="-proto_dist gen_tcp_blocking" + shift +elif [ "$1" = "false" ]; then + shift +fi args=${1+"$@"} -erlcmd="erl -sname a $p $args -mnesia_test_timeout" -erlcmd1="erl -sname a1 $p $args" -erlcmd2="erl -sname a2 $p $args" +tmp=$(mktemp -d) +if [ "$?" != "0" ]; then + echo "Failed to create temp directory" >&2 + exit 1 +fi +trap 'rm -rf "$tmp"' EXIT +erlcmd="erl -sname a@localhost $p $args \ + $dist_args \ + -mnesia_test_timeout -s mt start_peers a1 a2 $tmp $args" if test z"$MT_TERM" = z ; then MT_TERM=xterm @@ -74,8 +88,9 @@ echo "" $MT_TERM $geom0 $title a $exec script -c "$erlcmd" -f $log & -$MT_TERM $geom1 $title a1 $exec $erlcmd1 & -$MT_TERM $geom2 $title a2 $exec $erlcmd2 & +sleep 1 +$MT_TERM $geom1 $title a1 $exec to_erl $tmp/a1/ & +$MT_TERM $geom2 $title a2 $exec to_erl $tmp/a2/ & echo "Give the following command in order to see the outcome from node a@$h"":" echo "" diff --git a/lib/mnesia/test/mt.erl b/lib/mnesia/test/mt.erl index 8263671f392f..02cfc94233ef 100644 --- a/lib/mnesia/test/mt.erl +++ b/lib/mnesia/test/mt.erl @@ -36,9 +36,12 @@ doc/0, doc/1, % Generate test case doc struct/0, struct/1, % View test suite struct shutdown/0, ping/0, start_nodes/0, % Node admin + start_peers/1, % Peers for mt script read_config/0, write_config/1 % Config admin ]). +-include("gen_tcp_blocking_dist.hrl"). + -compile({no_auto_import,[alias/1]}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -275,3 +278,47 @@ ok_result([{_T,{TC,List}}|R]) when is_tuple(TC), is_list(List) -> ok_result(List) andalso ok_result(R); ok_result([]) -> true; ok_result(_) -> error. + +%% Peer nodes for mt script +start_peers(Args) -> + [Name2, Name3, TmpDirA | ExtraArgs0] = Args, + CodePaths = lists:append([["-pa", P] || P <- code:get_path(), P /= "."]), + DistOpts = ?NETWORK_BLOCKER_DIST_OPTS, + ExtraArgs = + case mnesia_test_lib:has_network_blocker() of + true -> + CodePaths ++ DistOpts ++ ExtraArgs0; + false -> + CodePaths ++ ExtraArgs0 + end, + TmpDir = atom_to_list(TmpDirA), + {Name2, P2, N2} = start_peer(Name2, TmpDir, ExtraArgs), + {Name3, P3, N3} = start_peer(Name3, TmpDir, ExtraArgs), + ok = persistent_term:put({peer, N2}, P2), + ok = persistent_term:put({peer, N3}, P3). + +start_peer(Name, TmpDir, ExtraArgs) -> + Dir = TmpDir ++ "/" ++ atom_to_list(Name) ++ "/", + ok = filelib:ensure_dir(Dir), + Erl = case init:get_argument(progname) of + {ok, [[Prog]]} -> + case os:find_executable(Prog) of + false -> "erl"; + Found -> Found + end; + _ -> "erl" + end, + RunErl = os:find_executable("run_erl"), + {ok, Peer, Node} = peer:start(#{ + name => Name, + host => "localhost", + connection => 0, + detached => false, + exec => {RunErl, ["-daemon"]}, + post_process_args => fun(Args) -> + Cmd = lists:flatten([Erl, " ", lists:join(" ", Args)]), + [Dir, Dir, "exec " ++ Cmd] + end, + args => ExtraArgs + }), + {Name, Peer, Node}.