@@ -20,6 +20,7 @@ use std::time::Duration;
2020use anyhow:: { Context , anyhow} ;
2121use log:: { debug, error, info, warn} ;
2222use serde_json;
23+ use socket2:: { SockRef , TcpKeepalive } ;
2324use vm_memory:: bitmap:: BitmapSlice ;
2425use vm_memory:: {
2526 Bytes , GuestAddress , GuestAddressSpace , GuestMemoryAtomic , ReadVolatile , VolatileMemoryError ,
@@ -39,33 +40,10 @@ use crate::{GuestMemoryMmap, VmMigrationConfig};
3940/// receiver side.
4041pub ( crate ) const MAX_MIGRATION_CONNECTIONS : u32 = 128 ;
4142
42- /// The time a writer may block on a socket until it throws an error.
43- ///
44- /// Also the interval at which the [`KeepAliveStream`] sends keep alive messages.
45- ///
46- /// # Relation with [`MIGRATION_READ_TIMEOUT_DURATION`]
47- ///
48- /// This timeout has to be smaller than [`MIGRATION_READ_TIMEOUT_DURATION`],
49- /// otherwise spurious timeouts may happen.
50- const MIGRATION_WRITE_TIMEOUT_DURATION : Duration = Duration :: from_secs ( 5 ) ;
51-
52- /// The time a reader may block on a socket until it throws an error.
53- ///
54- /// # Relation with [`MIGRATION_WRITE_TIMEOUT_DURATION`]
55- ///
56- /// This timeout has to be larger than [`MIGRATION_WRITE_TIMEOUT_DURATION`],
57- /// otherwise spurious timeouts may happen.
58- const MIGRATION_READ_TIMEOUT_DURATION : Duration = {
59- let migration_read_timeout_duration = Duration :: from_secs ( 10 ) ;
60-
61- // This timeout has to be larger than [`MIGRATION_WRITE_TIMEOUT_DURATION`],
62- // otherwise spurious timeouts may happen.
63- assert ! (
64- MIGRATION_WRITE_TIMEOUT_DURATION . as_millis( ) < migration_read_timeout_duration. as_millis( ) ,
65- "MIGRATION_WRITE_TIMEOUT_DURATION must be smaller than MIGRATION_READ_TIMEOUT_DURATION" ,
66- ) ;
67- migration_read_timeout_duration
68- } ;
43+ /// Interval at which the [`KeepAliveStream`] sends keep alive messages.
44+ /// This must remain shorter than the old sender's 10-second read timeout.
45+ // TODO(keepalive): remove this interval when gardenlinux-release-26-09-18 compatibility ends.
46+ const MIGRATION_KEEP_ALIVE_INTERVAL : Duration = Duration :: from_secs ( 5 ) ;
6947
7048/// The timeout of the migration-receiver.
7149///
@@ -76,16 +54,6 @@ const MIGRATION_READ_TIMEOUT_DURATION: Duration = {
7654/// if the management software doesn't kill the process correctly.
7755const MIGRATION_ACCEPT_TIMEOUT_DURATION : Duration = Duration :: from_secs ( 60 ) ;
7856
79- fn set_migration_socket_timeouts ( socket : & TcpStream ) -> anyhow:: Result < ( ) > {
80- socket
81- . set_read_timeout ( Some ( MIGRATION_READ_TIMEOUT_DURATION ) )
82- . context ( "Error setting read timeout on TCP socket" ) ?;
83- socket
84- . set_write_timeout ( Some ( MIGRATION_WRITE_TIMEOUT_DURATION ) )
85- . context ( "Error setting write timeout on TCP socket" ) ?;
86- Ok ( ( ) )
87- }
88-
8957/// Transport-agnostic listener used to receive connections.
9058#[ derive( Debug ) ]
9159pub ( crate ) enum ReceiveListener {
@@ -98,6 +66,7 @@ impl ReceiveListener {
9866 /// Block until a connection is accepted.
9967 pub ( crate ) fn accept (
10068 & mut self ,
69+ // TODO(keepalive): remove this parameter when gardenlinux-release-26-09-18 compatibility ends.
10170 main_connection : bool ,
10271 ) -> Result < SocketStream , MigratableError > {
10372 match self {
@@ -109,11 +78,16 @@ impl ReceiveListener {
10978 let ( socket, _) = accept_with_timeout ( listener, MIGRATION_ACCEPT_TIMEOUT_DURATION )
11079 . context ( "Failed to accept TCP migration connection" )
11180 . map_err ( MigratableError :: MigrateReceive ) ?;
112- set_migration_socket_timeouts ( & socket) . map_err ( MigratableError :: MigrateReceive ) ?;
81+
82+ set_tcp_keepalive_and_user_timeout ( & socket)
83+ . context ( "Failed to set socket options" )
84+ . map_err ( MigratableError :: MigrateReceive ) ?;
11385
11486 let socket = SocketStream :: Tcp ( socket) ;
87+
88+ // TODO(keepalive): remove this wrapper when gardenlinux-release-26-09-18 compatibility ends.
11589 if main_connection {
116- KeepAliveStream :: new ( socket, MIGRATION_WRITE_TIMEOUT_DURATION , false )
90+ KeepAliveStream :: new ( socket, MIGRATION_KEEP_ALIVE_INTERVAL )
11791 . map ( SocketStream :: KeepAlive )
11892 . context ( "Error creating keep-alive migration stream" )
11993 . map_err ( MigratableError :: MigrateReceive )
@@ -134,16 +108,20 @@ impl ReceiveListener {
134108 let ( socket, _) = accept_with_timeout ( listener, MIGRATION_ACCEPT_TIMEOUT_DURATION )
135109 . context ( "Failed to accept TCP connection" )
136110 . map_err ( MigratableError :: MigrateReceive ) ?;
137- set_migration_socket_timeouts ( & socket) . map_err ( MigratableError :: MigrateReceive ) ?;
111+
112+ set_tcp_keepalive_and_user_timeout ( & socket)
113+ . context ( "Failed to set socket options" )
114+ . map_err ( MigratableError :: MigrateReceive ) ?;
138115
139116 let socket = TlsStream :: new_server ( socket, config)
140117 . map ( Box :: new)
141118 . map ( SocketStream :: Tls )
142119 . context ( "Failed to accept TLS migration connection" )
143120 . map_err ( MigratableError :: MigrateReceive ) ?;
144121
122+ // TODO(keepalive): remove this wrapper when gardenlinux-release-26-09-18 compatibility ends.
145123 if main_connection {
146- KeepAliveStream :: new ( socket, MIGRATION_WRITE_TIMEOUT_DURATION , false )
124+ KeepAliveStream :: new ( socket, MIGRATION_KEEP_ALIVE_INTERVAL )
147125 . map ( SocketStream :: KeepAlive )
148126 . context ( "Error creating keep-alive migration stream" )
149127 . map_err ( MigratableError :: MigrateReceive )
@@ -228,6 +206,7 @@ pub(crate) enum SocketStream {
228206 Unix ( UnixStream ) ,
229207 Tcp ( TcpStream ) ,
230208 Tls ( Box < TlsStream > ) ,
209+ // TODO(keepalive): remove this variant when gardenlinux-release-26-09-18 compatibility ends.
231210 KeepAlive ( KeepAliveStream ) ,
232211}
233212
@@ -282,6 +261,7 @@ impl ReadVolatile for SocketStream {
282261 SocketStream :: Unix ( s) => s. read_volatile ( buf) ,
283262 SocketStream :: Tcp ( s) => s. read_volatile ( buf) ,
284263 SocketStream :: Tls ( s) => s. read_volatile ( buf) ,
264+
285265 SocketStream :: KeepAlive ( s) => s. read_volatile ( buf) ,
286266 }
287267 }
@@ -940,6 +920,33 @@ fn tcp_address_to_server_name(address: &str) -> Result<&str, anyhow::Error> {
940920 }
941921}
942922
923+ /// Enables `SO_KEEPALIVE` and `TCP_USER_TIMEOUT` for the `tcp_stream`'s socket.
924+ ///
925+ /// The set options target a failure detection time of two to three minutes.
926+ fn set_tcp_keepalive_and_user_timeout ( tcp_stream : & TcpStream ) -> io:: Result < ( ) > {
927+ /// [`TcpKeepalive`] config for migration TCP sockets.
928+ ///
929+ /// After 60 seconds, the kernel starts sending keepalive probes.
930+ /// Every 15 seconds another keepalive probe is sent.
931+ /// If 4 probes are sent without response, the connection is dropped.
932+ const MIGRATION_TCP_KEEPALIVE : TcpKeepalive = TcpKeepalive :: new ( )
933+ . with_retries ( 4 )
934+ . with_time ( Duration :: from_secs ( 60 ) )
935+ . with_interval ( Duration :: from_secs ( 15 ) ) ;
936+
937+ let socket_ref = SockRef :: from ( & tcp_stream) ;
938+
939+ // `TCP_USER_TIMEOUT` overrides `SO_KEEPALIVE` partially as per documentation:
940+ // "[...] TCP_USER_TIMEOUT will override keepalive to determine when
941+ // to close a connection due to keepalive failure."
942+ // https://man7.org/linux/man-pages/man7/tcp.7.html
943+ //TODO: Look into setting the keepalive via `std` once
944+ // https://github.com/rust-lang/rust/issues/155889 is stabilized.
945+ socket_ref. set_tcp_keepalive ( & MIGRATION_TCP_KEEPALIVE ) ?;
946+
947+ socket_ref. set_tcp_user_timeout ( Some ( Duration :: from_secs ( 120 ) ) )
948+ }
949+
943950/// Connect to a migration endpoint and return the established stream.
944951pub ( crate ) fn send_migration_socket (
945952 destination_url : & str ,
@@ -951,7 +958,10 @@ pub(crate) fn send_migration_socket(
951958 let socket = TcpStream :: connect ( address) . map_err ( |e| {
952959 MigratableError :: MigrateSend ( anyhow ! ( "Error connecting to TCP socket: {e}" ) )
953960 } ) ?;
954- set_migration_socket_timeouts ( & socket) . map_err ( MigratableError :: MigrateSend ) ?;
961+
962+ set_tcp_keepalive_and_user_timeout ( & socket)
963+ . context ( "Failed to set socket options" )
964+ . map_err ( MigratableError :: MigrateSend ) ?;
955965
956966 if let Some ( tls_dir) = tls_dir {
957967 let server_name = tcp_address_to_server_name ( address)
@@ -977,23 +987,6 @@ pub(crate) fn send_migration_socket(
977987 }
978988}
979989
980- /// Connect to the main migration endpoint and keep the connection active while
981- /// memory is transferred over additional streams.
982- pub ( crate ) fn send_migration_socket_with_keep_alive (
983- destination_url : & str ,
984- tls_dir : Option < & Path > ,
985- ) -> Result < SocketStream , MigratableError > {
986- match send_migration_socket ( destination_url, tls_dir) ? {
987- socket @ ( SocketStream :: Tcp ( _) | SocketStream :: Tls ( _) ) => {
988- KeepAliveStream :: new ( socket, MIGRATION_WRITE_TIMEOUT_DURATION , true )
989- . map ( SocketStream :: KeepAlive )
990- . context ( "Error creating keep-alive migration stream" )
991- . map_err ( MigratableError :: MigrateSend )
992- }
993- socket => Ok ( socket) ,
994- }
995- }
996-
997990/// Bind a migration listener for the receiver side.
998991pub ( crate ) fn receive_migration_listener (
999992 receiver_url : & str ,
0 commit comments