217 api validation b

.gitignore

@@ -5,3 +5,4 @@ vendor/
 docs/_build/
 docs/html/
 docs/doxyxml
+.vscode

includes/api_rate_limit.inc

@@ -0,0 +1,74 @@
+<?php
+
+/**
+ * Utility/timer functions for API Rate Limiting 
+ */
+
+/**
+ * Utility function to get the SSE and microseconds
+ */
+function get_mtime() {
+    $right_now = DateTime::createFromFormat('U.u', microtime(true));
+    // Occasionally calling DateTime will fail, keep trying until it doesn't.
+    while(!$right_now)
+    {
+        usleep(50000);
+        $right_now = DateTime::createFromFormat('U.u', microtime(true));
+    }
+    return floatval($right_now->format("U.u")); 
+}
+
+/**
+ * Adds an entry to the global variable that an NCBI request has been made
+ */
+function tripal_eutils_register_access() {
+    $last_access = variable_get('tripal_eutils_api_timer',NULL);
+
+    $last_access = get_mtime();
+
+    variable_set('tripal_eutils_api_timer',$last_access);
+}
+
+/**
+ * Checks that EUtils can continue making requests to NCBI without
+ * exceeding the API rate limits
+ * 
+ * @return boolean
+ *  TRUE if EUtils has to wait, false otherwise
+ *    Enough time has passed or no recent requests made
+ */
+function api_timer_wait() {
+    $last_access = variable_get('tripal_eutils_api_timer',NULL);
+    // Determine the rate limit
+    if (variable_get('tripal_eutils_ncbi_api_key'))
+    {
+        // API Key exists, was checked at previous check. Full speed ahead
+        $limit = 10;
+    }
+    else {
+        // No API key exists. Rate limit to 3 per second 
+        $limit = 3;
+    }
+
+    // No known previous access, go for it
+    if (!$last_access) {
+        return false;
+    }
+    else {
+        // Determine appropriate time to wait (+ a little padding, NCBI is sensitive)
+        $wait_time = 1.0/$limit + 0.2;
+
+        // Current time (format: seconds since epoch.microseconds)
+        $now = get_mtime();
+
+        $variance = number_format($now - $last_access, 5);
+        if ($variance > $wait_time) {
+            // Don't wait
+            return false;
+        }
+        else {
+            // Gotta wait
+            return true;
+        }
+    }
+}

includes/resources/EFetch.inc

@@ -20,7 +20,7 @@ class EFetch extends EUtilsRequest {
     $this->addParam('db', $db);
 
     $api_key = variable_get('tripal_eutils_ncbi_api_key');
-    if ($api_key) {
+    if ($api_key && $this->api_key_validate($api_key)) {
       $this->addParam('api_key', $api_key);
     }
   }

includes/resources/ESearch.inc

@@ -18,7 +18,7 @@ class ESearch extends EUtilsRequest {
     $this->setBaseURL('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi');
     $this->addParam('db', $db);
     $api_key = variable_get('tripal_eutils_ncbi_api_key');
-    if ($api_key) {
+    if ($api_key && $this->api_key_validate($api_key)) {
       $this->addParam('api_key', $api_key);
     }
   }

includes/resources/ESummary.inc

@@ -18,7 +18,7 @@
     $this->setBaseURL('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi');
     $this->addParam('db', $db);
     $api_key = variable_get('tripal_eutils_ncbi_api_key');
-    if ($api_key) {
+    if ($api_key && $this->api_key_validate($api_key)) {
       $this->addParam('api_key', $api_key);
     }
   }

includes/resources/EUtils.inc

@@ -96,9 +96,13 @@ class EUtils {
       $provider->addParam('retmode', 'xml');
     }
 
-    // TODO:  global static timer.
-    // That pauses if we're passing our alloted queries/second.
+    // Wait until timer has accumulated enough time to continue
+    while (api_timer_wait()){}
+
     $response = $provider->get();
+    // Request was made, register this timestamp for API rate limit
+    tripal_eutils_register_access();
+
     $this->checkResponseSuccess($response, $db, $accession);
     $xml = $response->xml();
 
@@ -259,4 +263,4 @@ class EUtils {
     }
   }
 
-}
+}

includes/resources/EUtilsRequest.inc

@@ -170,4 +170,61 @@ class EUtilsRequest {
     return substr($str, 0, strlen($with)) === $with;
   }
 
+  /**
+   * Check whether the API key is still valid
+   * 
+   * @param string $api_key
+   * @param string $db_url
+   * 
+   * @return bool
+   */
+  protected function api_key_validate($api_key) {
+    // Get the value of tripal_eutils_recent_validation [last_validation, value]
+    $recent_validation = variable_get('tripal_eutils_recent_validation',NULL);
+    if ($recent_validation)
+    {
+      $now = intval(date("u"));
+      // At some point, the API key was positively validated
+      if ($recent_validation[1])
+      {
+        // Test if expired
+        if ($now - $recent_validation[1] > 1800)
+        {
+          return false;
+        }
+        else
+        {
+          return true;
+        }
+      }
+      // The API key was most recently found to be invalid
+      else {
+        return true;
+      } 
+    }
+    else  // No known API validations or API validation has expired
+    {
+      $now = intval(date("u"));
+      $test_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi";
+      $test_url = $test_url . '?api_key=' . $api_key;
+
+      // Make a request to NCBI, collect and parse the response
+      $response = drupal_http_request($test_url);
+
+      if (array_key_exists('error',$response)) {
+        // Some error exists with the API key. Print this
+        // Set the recent
+        variable_set('tripal_eutils_recent_validation',[$now,FALSE]);
+        // and return false
+        return false;
+      }
+      else {
+        // Set the recent
+        variable_set('tripal_eutils_recent_validation',[$now,TRUE]);
+        return true;
+      }
+
+    }
+  }
+
 }

includes/tripal_euitils_admin_settings.form.inc

@@ -31,51 +31,56 @@ function tripal_eutils_admin_settings_form($form, &$form_state) {
  * Implements hook_validate().
  */
 function tripal_eutils_admin_settings_form_validate($form, &$form_state) {
+  // Check to see if there exists an API key
+  if ($form_state['values']['api_key']) {
+    // Prepare the URL and parameters (with API key)
+    $url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi';
+    $url .= '?api_key='.$form_state['values']['api_key'];
 
-  // TODO We would validate the key here.  Perhaps make a test query?
+    // Make the request
+    $response = drupal_http_request($url);
 
-  // Prepare the URL and parameters (with API key)
-  $url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi';
-  $url .= '?api_key='.$form_state['values']['api_key'];
-
-  // Make the request
-  $response = drupal_http_request($url);
-
-  // Error handling
-  if (array_key_exists('error',$response))
-  {
-    if(preg_match('/\b(API key invalid)\b/',$response->data))
-    {
-      //drupal_set_message("Invalid API Key",'error');
-      form_set_error('api_key',t("Invalid API Key"));
-    }
-    else if(preg_match('/\b(API key not wellformed)\b/',$response->data))
+    // Error handling
+    if (array_key_exists('error',$response))
     {
-      //drupal_set_message("Unknown or malformed API Key",'error');
-      form_set_error('api_key',t("Unknown or malformed API Key"));
-    }
-    else
-    {
-      drupal_set_message("Other unknown error:".$response->data,'error');
-      form_set_error('api_key',t("Unknown issue with API key"));
+      if(preg_match('/\b(API key invalid)\b/',$response->data))
+      {
+        //drupal_set_message("Invalid API Key",'error');
+        form_set_error('api_key',t("Invalid API Key"));
+      }
+      else if(preg_match('/\b(API key not wellformed)\b/',$response->data))
+      {
+        //drupal_set_message("Unknown or malformed API Key",'error');
+        form_set_error('api_key',t("Unknown or malformed API Key"));
+      }
+      else
+      {
+        drupal_set_message("Other unknown error:".$response->data,'error');
+        form_set_error('api_key',t("Unknown issue with API key"));
+      }
     }
+    // No apparent errors. Check that NCBI responded with the correct number of
+    //   requests per second that API key users should have (currently 10)
   }
-  // No apparent errors. Check that NCBI responded with the correct number of
-  //   requests per second that API key users should have (currently 10)
+
 }
 
 /**
  * Implements hook_submit().
  */
 function tripal_eutils_admin_settings_form_submit($form, &$form_state) {
-  if (isset($form_state['values']['api_key'])) {
+  // Valid API key
+  if (!empty($form_state['values']['api_key'])) {
     $api_key = $form_state['values']['api_key'];
     variable_set('tripal_eutils_ncbi_api_key', $api_key);
+    $now = $now = intval(date("u"));
+    variable_set('tripal_eutils_recent_validation', [$now, TRUE]);
+    drupal_set_message("API Key validated and saved.");
   }
+  // No API key
   else {
     variable_set('tripal_eutils_ncbi_api_key', NULL);
+    variable_set('tripal_eutils_recent_validation', NULL);
+    drupal_set_message("API Key removed.");
   }
-
-  drupal_set_message("API Key validated and saved.");
-
 }

tripal_eutils.module

@@ -35,6 +35,7 @@ require_once 'includes/repositories/EUtilsPubmedRepository.inc';
 
 // Repository helpers.  Assist with looking up terms.
 require_once 'includes/TagMapper.inc';
+require_once 'includes/api_rate_limit.inc';
 
 // Formatters.
 require_once 'includes/formatters/EUtilsFormatter.inc';