Options to enforce fallback if config is out of sync (#344)

Context:
1. Forward proxy can be out of sync, keep sending back has no update
value
2. We want to provide a option for them to enforce fallback if sdk is
out of sync for x hours

Default this option is off

Test
see added test

Author: xinlili-statsig

statsig/spec_updater.py

@@ -1,5 +1,6 @@
 import json
 import threading
+import time
 from typing import Optional, Callable, List, Any, Tuple
 
 from . import globals
@@ -44,6 +45,10 @@ def __init__(
         self._background_download_id_lists = None
         self._config_sync_strategies = self._get_sync_dcs_strategies()
         self._dcs_process_lock = threading.Lock()
+        if options.out_of_sync_threshold_in_s is not None:
+            self._enforce_sync_fallback_threshold_in_ms: Optional[float] = options.out_of_sync_threshold_in_s * 1000
+        else:
+            self._enforce_sync_fallback_threshold_in_ms = None
 
         self.initialized = False
         self.last_update_time = 0
@@ -369,8 +374,11 @@ def sync_config_spec():
             for i, strategy in enumerate(self._config_sync_strategies):
                 prev_failure_count = self._sync_failure_count
                 self.get_config_spec(strategy)
-
-                if prev_failure_count == self._sync_failure_count:
+                outof_sync = False
+                time_elapsed = time.time() * 1000 - self.last_update_time
+                if (self._enforce_sync_fallback_threshold_in_ms is not None and time_elapsed > self._enforce_sync_fallback_threshold_in_ms):
+                    outof_sync = True
+                if prev_failure_count == self._sync_failure_count and not outof_sync:
                     globals.logger.log_process("Config Sync", f"Syncing config values with {strategy.value} successful")
                     break
                 if i < len(self._config_sync_strategies) - 1:

statsig/statsig_options.py

@@ -108,6 +108,7 @@ def __init__(
             retry_queue_size: int = DEFAULT_RETRY_QUEUE_SIZE,
             proxy_configs: Optional[Dict[NetworkEndpoint, ProxyConfig]] = None,
             fallback_to_statsig_api: Optional[bool] = False,
+            out_of_sync_threshold_in_s: Optional[float] = None , # If config is out of sync for {threshold} s, we enforce to fallback logic you defined
             initialize_sources: Optional[List[DataSource]] = None,
             config_sync_sources: Optional[List[DataSource]] = None,
             output_logger_level: Optional[LogLevel] = LogLevel.WARNING,
@@ -148,6 +149,7 @@ def __init__(
         self.evaluation_callback = evaluation_callback
         self.retry_queue_size = retry_queue_size
         self.fallback_to_statsig_api = fallback_to_statsig_api
+        self.out_of_sync_threshold_in_s = out_of_sync_threshold_in_s
         if proxy_configs is None:
             self.proxy_configs = DEFAULT_PROXY_CONFIG
         else:

tests/network_stub.py

@@ -2,9 +2,11 @@
 from typing import Callable, Union
 from urllib.parse import urlparse, ParseResult
 
+STATSIG_APIS = ["https://api.statsigcdn.com/", "https://statsigapi.net/"]
 
 class NetworkStub:
     host: str
+    mock_statsig_api: bool
 
     class StubResponse:
         def __init__(self, status, data=None, headers=None):
@@ -20,8 +22,9 @@ def __init__(self, status, data=None, headers=None):
         def json(self):
             return self._json
 
-    def __init__(self, host: str):
+    def __init__(self, host: str, mock_statsig_api = False):
         self.host = host
+        self.mock_statsig_api = mock_statsig_api
         self._stubs = {}
 
     def reset(self):
@@ -52,8 +55,8 @@ def mock(*args, **kwargs):
         instance: NetworkStub = args[0]
         method: str = args[1]
         url: ParseResult = urlparse(args[2])
-
-        if (url.scheme + "://" + url.hostname) != instance.host:
+        request_host = (url.scheme + "://" + url.hostname)
+        if request_host != instance.host and (instance.mock_statsig_api and request_host not in STATSIG_APIS):
             return
 
         paths = list(instance._stubs.keys())

tests/test_sync_config_fallback.py

@@ -0,0 +1,75 @@
+import json
+import os
+import time
+import unittest
+
+from unittest.mock import patch
+from network_stub import NetworkStub
+from statsig import StatsigOptions, statsig, StatsigUser
+from statsig.evaluation_details import EvaluationDetails, EvaluationReason
+from statsig.http_worker import HttpWorker
+
+_network_stub = NetworkStub("http://test-sync-config-fallback", mock_statsig_api=True)
+with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../testdata/download_config_specs.json')) as r:
+    CONFIG_SPECS_RESPONSE = r.read()
+
+class TestSyncConfigFallback(unittest.TestCase):
+  @classmethod
+  @patch('requests.request', side_effect=_network_stub.mock)
+  def setUpClass(cls, mock_proxy):
+    cls.dcs_hit = 0
+    _network_stub.reset()
+    def dcs_proxy_callback(url: str, **kwargs):
+      cls.dcs_hit += 1
+      return json.loads(CONFIG_SPECS_RESPONSE)
+
+    _network_stub.stub_request_with_function(
+        "download_config_specs/.*", 200, dcs_proxy_callback)
+
+    cls.test_user = StatsigUser("123", email="[email protected]")
+  
+  def tearDown(self):
+    self.dcs_hit = 0
+    statsig.shutdown()
+      
+  @patch('requests.request', side_effect=_network_stub.mock)
+  @patch.object(HttpWorker, 'get_dcs_fallback')
+  def test_default_behavior(self, fallback_mock, request_mock):
+    # default behavior is no fallback if is out of sync
+    options = StatsigOptions(api=_network_stub.host, fallback_to_statsig_api=True, rulesets_sync_interval=1)
+    statsig.initialize("secret-key", options)
+    gate = statsig.get_feature_gate(self.test_user, "always_on_gate")
+    eval_detail: EvaluationDetails = gate.get_evaluation_details()
+    self.assertEqual(eval_detail.reason, EvaluationReason.network)
+    self.assertEqual(eval_detail.config_sync_time, 1631638014811)
+    time.sleep(1.1)
+    
+    fallback_mock.assert_not_called()
+    
+  @patch('requests.request', side_effect=_network_stub.mock)
+  @patch.object(HttpWorker, 'get_dcs_fallback')
+  def test_fallback_when_out_of_sync(self, fallback_mock, request_mock):
+    # default behavior is no fallback if is out of sync
+    options = StatsigOptions(api_for_download_config_specs=_network_stub.host, fallback_to_statsig_api=True, rulesets_sync_interval=1, out_of_sync_threshold_in_s=0.5)
+    statsig.initialize("secret-key", options)
+    gate = statsig.get_feature_gate(self.test_user, "always_on_gate")
+    eval_detail: EvaluationDetails = gate.get_evaluation_details()
+    self.assertEqual(eval_detail.reason, EvaluationReason.network)
+    self.assertEqual(eval_detail.config_sync_time, 1631638014811)
+    time.sleep(1.1)
+    #ensure it falls back
+    fallback_mock.assert_called_once()
+    
+  @patch('requests.request', side_effect=_network_stub.mock)
+  @patch.object(HttpWorker, 'get_dcs_fallback')
+  def test_behavior_when_not_out_of_sync(self, fallback_mock, request_mock):
+    # default behavior is no fallback if is out of sync
+    options = StatsigOptions(api_for_download_config_specs=_network_stub.host, fallback_to_statsig_api=True, rulesets_sync_interval=1, out_of_sync_threshold_in_s=4e10)
+    statsig.initialize("secret-key", options)
+    gate = statsig.get_feature_gate(self.test_user, "always_on_gate")
+    eval_detail: EvaluationDetails = gate.get_evaluation_details()
+    self.assertEqual(eval_detail.reason, EvaluationReason.network)
+    self.assertEqual(eval_detail.config_sync_time, 1631638014811)
+    time.sleep(1.1)
+    #ensure no fallback
+    fallback_mock.assert_not_called()