Merge pull request #63 from neyoneit/dl-source-fallback

Add dl_source2: dual download source with speed test and fallback

Author: neyoneit

code/client/cl_curl.c

@@ -896,6 +896,148 @@ static size_t Com_DL_HeaderCallback( void *ptr, size_t size, size_t nmemb, void
 }
 
 
+/*
+===============================================================
+Com_DL_SpeedTest()
+
+Run a parallel 1-second speed test against two download sources.
+Returns 1 if source1 is faster/equal, 2 if source2 is faster.
+Returns 0 on error (use source1 as default).
+Both URLs should have %m already replaced with the actual filename.
+===============================================================
+*/
+typedef struct {
+	int bytesReceived;
+} speedTestData_t;
+
+static size_t Com_DL_SpeedTestWrite( void *ptr, size_t size, size_t nmemb, void *userdata )
+{
+	speedTestData_t *data = (speedTestData_t *)userdata;
+	data->bytesReceived += (int)( size * nmemb );
+	return size * nmemb;
+}
+
+int Com_DL_SpeedTest( download_t *dl, const char *url1, const char *url2 )
+{
+	CURL *easy1, *easy2;
+	CURLM *multi;
+	speedTestData_t data1, data2;
+	int running, startTime, elapsed;
+	float speed1, speed2;
+
+	if ( !Com_DL_Init( dl ) ) {
+		Com_Printf( S_COLOR_YELLOW "Speed test: failed to init cURL\n" );
+		return 0;
+	}
+
+	easy1 = dl->func.easy_init();
+	easy2 = dl->func.easy_init();
+	multi = dl->func.multi_init();
+
+	if ( !easy1 || !easy2 || !multi ) {
+		Com_Printf( S_COLOR_YELLOW "Speed test: failed to create cURL handles\n" );
+		if ( easy1 ) dl->func.easy_cleanup( easy1 );
+		if ( easy2 ) dl->func.easy_cleanup( easy2 );
+		if ( multi ) dl->func.multi_cleanup( multi );
+		Com_DL_Done( dl );
+		return 0;
+	}
+
+	memset( &data1, 0, sizeof( data1 ) );
+	memset( &data2, 0, sizeof( data2 ) );
+
+	// setup source 1
+	dl->func.easy_setopt( easy1, CURLOPT_URL, url1 );
+	dl->func.easy_setopt( easy1, CURLOPT_WRITEFUNCTION, Com_DL_SpeedTestWrite );
+	dl->func.easy_setopt( easy1, CURLOPT_WRITEDATA, &data1 );
+	dl->func.easy_setopt( easy1, CURLOPT_FOLLOWLOCATION, 1 );
+	dl->func.easy_setopt( easy1, CURLOPT_MAXREDIRS, 5 );
+	dl->func.easy_setopt( easy1, CURLOPT_FAILONERROR, 1 );
+	dl->func.easy_setopt( easy1, CURLOPT_NOPROGRESS, 1 );
+	dl->func.easy_setopt( easy1, CURLOPT_CONNECTTIMEOUT, 3L );
+	dl->func.easy_setopt( easy1, CURLOPT_USERAGENT, Q3_VERSION );
+#if CURL_AT_LEAST_VERSION(7, 85, 0)
+	dl->func.easy_setopt( easy1, CURLOPT_PROTOCOLS_STR, ALLOWED_PROTOCOLS_STR );
+#else
+	dl->func.easy_setopt( easy1, CURLOPT_PROTOCOLS, ALLOWED_PROTOCOLS );
+#endif
+
+	// setup source 2
+	dl->func.easy_setopt( easy2, CURLOPT_URL, url2 );
+	dl->func.easy_setopt( easy2, CURLOPT_WRITEFUNCTION, Com_DL_SpeedTestWrite );
+	dl->func.easy_setopt( easy2, CURLOPT_WRITEDATA, &data2 );
+	dl->func.easy_setopt( easy2, CURLOPT_FOLLOWLOCATION, 1 );
+	dl->func.easy_setopt( easy2, CURLOPT_MAXREDIRS, 5 );
+	dl->func.easy_setopt( easy2, CURLOPT_FAILONERROR, 1 );
+	dl->func.easy_setopt( easy2, CURLOPT_NOPROGRESS, 1 );
+	dl->func.easy_setopt( easy2, CURLOPT_CONNECTTIMEOUT, 3L );
+	dl->func.easy_setopt( easy2, CURLOPT_USERAGENT, Q3_VERSION );
+#if CURL_AT_LEAST_VERSION(7, 85, 0)
+	dl->func.easy_setopt( easy2, CURLOPT_PROTOCOLS_STR, ALLOWED_PROTOCOLS_STR );
+#else
+	dl->func.easy_setopt( easy2, CURLOPT_PROTOCOLS, ALLOWED_PROTOCOLS );
+#endif
+
+	dl->func.multi_add_handle( multi, easy1 );
+	dl->func.multi_add_handle( multi, easy2 );
+
+	Com_Printf( "Speed test: testing download sources...\n" );
+
+	startTime = Sys_Milliseconds();
+
+	// run both downloads for 2 seconds
+	do {
+		dl->func.multi_perform( multi, &running );
+		elapsed = Sys_Milliseconds() - startTime;
+		if ( running == 0 )
+			break;
+	} while ( elapsed < 3000 );
+
+	elapsed = Sys_Milliseconds() - startTime;
+	if ( elapsed < 1 ) elapsed = 1;
+
+	speed1 = (float)data1.bytesReceived / ( (float)elapsed / 1000.0f );
+	speed2 = (float)data2.bytesReceived / ( (float)elapsed / 1000.0f );
+
+	// log results
+	if ( speed1 >= 1024.0f * 1024.0f )
+		Com_Printf( "  dl_source:  %.1f MB/s\n", speed1 / ( 1024.0f * 1024.0f ) );
+	else
+		Com_Printf( "  dl_source:  %.0f KB/s\n", speed1 / 1024.0f );
+
+	if ( speed2 >= 1024.0f * 1024.0f )
+		Com_Printf( "  dl_source2: %.1f MB/s\n", speed2 / ( 1024.0f * 1024.0f ) );
+	else
+		Com_Printf( "  dl_source2: %.0f KB/s\n", speed2 / 1024.0f );
+
+	// cleanup
+	dl->func.multi_remove_handle( multi, easy1 );
+	dl->func.multi_remove_handle( multi, easy2 );
+	dl->func.easy_cleanup( easy1 );
+	dl->func.easy_cleanup( easy2 );
+	dl->func.multi_cleanup( multi );
+	Com_DL_Done( dl );
+
+	if ( speed2 > speed1 && data2.bytesReceived > 0 ) {
+		Com_Printf( S_COLOR_GREEN "Speed test: dl_source2 is faster, using as primary.\n" );
+		return 2;
+	}
+
+	if ( data1.bytesReceived > 0 ) {
+		Com_Printf( S_COLOR_GREEN "Speed test: dl_source is faster, using as primary.\n" );
+		return 1;
+	}
+
+	if ( data2.bytesReceived > 0 ) {
+		Com_Printf( S_COLOR_YELLOW "Speed test: dl_source failed, using dl_source2.\n" );
+		return 2;
+	}
+
+	Com_Printf( S_COLOR_RED "Speed test: both sources failed.\n" );
+	return 0;
+}
+
+
 /*
 ===============================================================
 Com_DL_Begin()
@@ -1124,12 +1266,32 @@ qboolean Com_DL_Perform( download_t *dl )
 	else
 	{
 		qboolean autoDownload = dl->mapAutoDownload;
+		char fallbackURL[MAX_OSPATH];
+		char originalName[MAX_OSPATH];
+
 		dl->func.easy_getinfo( msg->easy_handle, CURLINFO_RESPONSE_CODE, &code );
 		Com_Printf( S_COLOR_RED "Download Error: %s Code: %ld\n",
 			dl->func.easy_strerror( msg->data.result ), code );
+
+		// save fallback info before cleanup
+		Q_strncpyz( fallbackURL, dl->fallbackURL, sizeof( fallbackURL ) );
+		Q_strncpyz( originalName, dl->originalName, sizeof( originalName ) );
+
 		strcpy( name, dl->TempName );
 		Com_DL_Cleanup( dl );
 		FS_Remove( name );
+
+		// try fallback source if available
+		if ( fallbackURL[0] != '\0' && originalName[0] != '\0' )
+		{
+			Com_Printf( S_COLOR_YELLOW "Trying fallback source...\n" );
+			if ( Com_DL_Begin( dl, originalName, fallbackURL, autoDownload ) )
+			{
+				dl->fallbackURL[0] = '\0'; // don't fallback again
+				return qtrue;
+			}
+		}
+
 		if ( autoDownload )
 		{
 			if ( cls.state == CA_CONNECTED )

code/client/cl_curl.h

@@ -113,6 +113,10 @@ typedef struct download_s {
 	qboolean	headerCheck;
 	qboolean	mapAutoDownload;
 
+	// fallback source support
+	char		fallbackURL[MAX_OSPATH];
+	char		originalName[MAX_OSPATH];
+
 	struct func_s {
 		char*		(*version)(void);
 		char *		(*easy_escape)(CURL *curl, const char *string, int length);

code/client/cl_main.c

@@ -68,6 +68,7 @@ cvar_t	*cl_lanForcePackets;
 cvar_t	*cl_guidServerUniq;
 
 cvar_t	*dl_source;
+cvar_t	*dl_source2;
 cvar_t	*dl_usebaseq3;
 
 cvar_t	*cl_reconnectArgs;
@@ -3988,7 +3989,67 @@ void CL_Init( void ) {
 	Cvar_SetDescription( cl_guidServerUniq, "Makes cl_guid unique for each server." );
 
 	dl_source = Cvar_Get( "dl_source", "http://ws.q3df.org/maps/download/%m", CVAR_ARCHIVE );
-	Cvar_SetDescription( dl_source, "Cvar must point to download location." );
+	Cvar_SetDescription( dl_source, "Primary download source URL. Use %m for map/pak name placeholder." );
+
+	dl_source2 = Cvar_Get( "dl_source2", "https://defrag.racing/maps/download/%m", CVAR_ARCHIVE );
+	Cvar_SetDescription( dl_source2, "Secondary download source URL. If set, a 3-second speed test picks the faster source automatically. Use %m for map/pak name placeholder." );
+
+	// auto-fill empty sources: ensure both known mirrors are available
+	{
+		#define DL_DEFRAG_RACING "https://defrag.racing/maps/download/%m"
+		#define DL_WS_Q3DF      "http://ws.q3df.org/maps/download/%m"
+
+		qboolean src1_empty = ( dl_source->string[0] == '\0' );
+		qboolean src2_empty = ( dl_source2->string[0] == '\0' );
+		qboolean src1_is_defrag = ( strstr( dl_source->string, "defrag.racing" ) != NULL );
+		qboolean src1_is_ws     = ( strstr( dl_source->string, "ws.q3df.org" ) != NULL );
+		qboolean src2_is_defrag = ( strstr( dl_source2->string, "defrag.racing" ) != NULL );
+		qboolean src2_is_ws     = ( strstr( dl_source2->string, "ws.q3df.org" ) != NULL );
+
+		if ( src1_empty && src2_empty ) {
+			// both empty: set defaults
+			Cvar_Set( "dl_source", DL_WS_Q3DF );
+			Cvar_Set( "dl_source2", DL_DEFRAG_RACING );
+		} else if ( src1_empty ) {
+			// dl_source empty: fill with whichever mirror dl_source2 is NOT
+			Cvar_Set( "dl_source", src2_is_defrag ? DL_WS_Q3DF : DL_DEFRAG_RACING );
+		} else if ( src2_empty ) {
+			// dl_source2 empty: fill with whichever mirror dl_source is NOT
+			Cvar_Set( "dl_source2", src1_is_defrag ? DL_WS_Q3DF : DL_DEFRAG_RACING );
+		}
+
+		// fix duplicate sources: if both point to the same server, set the other one
+		if ( !src1_empty && !src2_empty ) {
+			// re-read after potential changes above
+			src1_is_defrag = ( strstr( dl_source->string, "defrag.racing" ) != NULL );
+			src1_is_ws     = ( strstr( dl_source->string, "ws.q3df.org" ) != NULL );
+			src2_is_defrag = ( strstr( dl_source2->string, "defrag.racing" ) != NULL );
+			src2_is_ws     = ( strstr( dl_source2->string, "ws.q3df.org" ) != NULL );
+
+			if ( src1_is_defrag && src2_is_defrag ) {
+				Cvar_Set( "dl_source2", DL_WS_Q3DF );
+			} else if ( src1_is_ws && src2_is_ws ) {
+				Cvar_Set( "dl_source2", DL_DEFRAG_RACING );
+			}
+		}
+
+		// fix common misconfiguration: ws.q3df.org only works over HTTP, not HTTPS
+		if ( strstr( dl_source->string, "https://ws.q3df.org" ) != NULL ) {
+			char fixed[MAX_CVAR_VALUE_STRING];
+			Q_strncpyz( fixed, dl_source->string, sizeof( fixed ) );
+			Q_replace( "https://ws.q3df.org", "http://ws.q3df.org", fixed, sizeof( fixed ) );
+			Cvar_Set( "dl_source", fixed );
+		}
+		if ( strstr( dl_source2->string, "https://ws.q3df.org" ) != NULL ) {
+			char fixed[MAX_CVAR_VALUE_STRING];
+			Q_strncpyz( fixed, dl_source2->string, sizeof( fixed ) );
+			Q_replace( "https://ws.q3df.org", "http://ws.q3df.org", fixed, sizeof( fixed ) );
+			Cvar_Set( "dl_source2", fixed );
+		}
+
+		#undef DL_DEFRAG_RACING
+		#undef DL_WS_Q3DF
+	}
 
 	dl_usebaseq3 = Cvar_Get( "dl_usebaseq3", "0", CVAR_ARCHIVE_ND );
 	Cvar_CheckRange( dl_usebaseq3, "0", "1", CV_INTEGER );
@@ -5043,11 +5104,27 @@ static void CL_ShowIP_f( void ) {
 
 #ifdef USE_CURL
 
+static int dl_preferredSource = 0; // 0=not tested, 1=dl_source, 2=dl_source2
+
+static const char *CL_DL_BuildURL( char *buf, int bufSize, const char *sourceURL, const char *pakname )
+{
+	// simple %m replacement without needing full CURL context
+	Q_strncpyz( buf, sourceURL, bufSize );
+	if ( !Q_replace( "%m", pakname, buf, bufSize ) ) {
+		if ( buf[strlen(buf)-1] != '/' )
+			Q_strcat( buf, bufSize, "/" );
+		Q_strcat( buf, bufSize, pakname );
+	}
+	return buf;
+}
+
 qboolean CL_Download( const char *cmd, const char *pakname, qboolean autoDownload )
 {
 	char url[MAX_OSPATH];
 	char name[MAX_CVAR_VALUE_STRING];
 	const char *s;
+	const char *primarySource;
+	const char *fallbackSource;
 
 	if ( dl_source->string[0] == '\0' )
 	{
@@ -5084,7 +5161,45 @@ qboolean CL_Download( const char *cmd, const char *pakname, qboolean autoDownloa
 		}
 	}
 
-	return Com_DL_Begin( &download, pakname, dl_source->string, autoDownload );
+	// speed test on first download if both sources are set
+	if ( dl_preferredSource == 0 && dl_source2->string[0] != '\0' )
+	{
+		char url1[MAX_OSPATH], url2[MAX_OSPATH];
+		CL_DL_BuildURL( url1, sizeof( url1 ), dl_source->string, pakname );
+		CL_DL_BuildURL( url2, sizeof( url2 ), dl_source2->string, pakname );
+		dl_preferredSource = Com_DL_SpeedTest( &download, url1, url2 );
+		if ( dl_preferredSource == 0 )
+			dl_preferredSource = 1; // both failed, default to dl_source
+	}
+
+	// select primary and fallback sources
+	if ( dl_preferredSource == 2 ) {
+		primarySource = dl_source2->string;
+		fallbackSource = dl_source->string;
+	} else {
+		primarySource = dl_source->string;
+		fallbackSource = ( dl_source2->string[0] != '\0' ) ? dl_source2->string : NULL;
+	}
+
+	if ( Com_DL_Begin( &download, pakname, primarySource, autoDownload ) ) {
+		// set fallback info for mid-download failure recovery
+		if ( fallbackSource ) {
+			Q_strncpyz( download.fallbackURL, fallbackSource, sizeof( download.fallbackURL ) );
+			Q_strncpyz( download.originalName, pakname, sizeof( download.originalName ) );
+		} else {
+			download.fallbackURL[0] = '\0';
+			download.originalName[0] = '\0';
+		}
+		return qtrue;
+	}
+
+	// primary failed to start, try fallback
+	if ( fallbackSource ) {
+		Com_Printf( S_COLOR_YELLOW "Primary source failed, trying fallback...\n" );
+		return Com_DL_Begin( &download, pakname, fallbackSource, autoDownload );
+	}
+
+	return qfalse;
 }
 
 

code/client/client.h

@@ -372,6 +372,7 @@ void		Com_DL_Cleanup( download_t *dl );
 qboolean	Com_DL_Begin( download_t *dl, const char *localName, const char *remoteURL, qboolean autoDownload );
 qboolean	Com_DL_InProgress( const download_t *dl );
 qboolean	Com_DL_ValidFileName( const char *fileName );
+int			Com_DL_SpeedTest( download_t *dl, const char *url1, const char *url2 );
 qboolean	CL_Download( const char *cmd, const char *pakname, qboolean autoDownload );
 
 #endif