Adds callback to provide headers for the source definition hook

circle.yml

@@ -27,7 +27,8 @@ checkout:
 
 ## Customize dependencies
 dependencies:
-
+  pre:
+    - echo "memory_limit = 256M" > ~/.phpenv/versions/5.5.11/etc/conf.d/memory.ini
   cache_directories:
      #- "test/vendor"
      #- "~/.composer"
@@ -39,6 +40,9 @@ dependencies:
     - 'bash dkan-module-init.sh --deps --build=$DATABASE_URL'
     - cd dkan_harvest && composer install && cd ..
     - ahoy drush -y en dkan_harvest_test dkan_harvest_example
+    # Run very simple server to test header decoration callback.
+    - 'ahoy cmd-proxy php -S 127.0.0.1:8080 -t dkan_harvest/test dkan_harvest/test/data_json_server_with_token.php':
+        background: true
   # we automatically cache and restore many dependencies between
   # builds. If you need to, you can add custom paths to cache:
   #cache_directories:

composer.json

@@ -2,7 +2,9 @@
     "require-dev": {
       "phpunit/phpunit": "4.6.*",
       "myplanetdigital/function_mock": "dev-master",
-      "drupal/drupal-driver": "~1.0"
+      "drupal/drupal-driver": "~1.0",
+      "silex/silex": "2.0.2",
+      "monolog/monolog": "~1.7"
     },
     "config": {
         "bin-dir": "bin"

dkan_harvest.api.inc

@@ -18,6 +18,8 @@ function dkan_harvest_harvest_sources() {
       'defaults' => array('keyword' => array('harvested dataset'),
       // Provide overrides (Optional).
       'overrides' => array('author' => 'Author'),
+      // Provide a callback to attach headers to file_get_content call
+      'attach_headers_callback' => 'dkan_harvest_attach_headers'
     ),
   );
 }

dkan_harvest.module

@@ -63,46 +63,57 @@ function dkan_harvest_cache_data($options = array()) {
  * Callback for dkan-cache-harvested-data.
  */
 function dkan_harvest_cache_data_process($sources, $harvest_last_updated) {
-  $context = stream_context_create(
-    array(
-      'http' => array(
-        'timeout' => 36000,
-      ),
-      'https' => array(
-        'timeout' => 36000,
-      ),
-    )
+  $options = array(
+    'timeout' => 36000,
+    'method' => 'GET',
   );
   $counters = array();
   foreach ($sources as $key => $source) {
-    $remote = file_get_contents($source['remote'], 0, $context);
-    $counters[$key] = 0;
-    if ($remote) {
-      switch ($source['type']) {
-        case 'data.xml':
-          $xml = simplexml_load_string($remote);
-          $json  = json_encode($xml);
-          $array = json_decode($json, TRUE);
+    $context = $base_context;
+    if (isset($source['attach_headers_callback'])) {
+      $callback = $source['attach_headers_callback'];
+      $options['headers'] = $callback();
+    }
+    $request = array();
+    if (filter_var($source['remote'], FILTER_VALIDATE_URL)) {
+      $request = drupal_http_request($source['remote'], $options);
+      $remote = $request->data;
+    }
+    else {
+      $remote = file_get_contents($source['remote']);
+    }
+    if (!isset($request->error)) {
+      $counters[$key] = 0;
+      if ($remote) {
+        switch ($source['type']) {
+          case 'data.xml':
+            $xml = simplexml_load_string($remote);
+            $json  = json_encode($xml);
+            $array = json_decode($json, TRUE);
 
-          if ($array) {
-            dkan_harvest_cache_pod_1_1_json($array, $key, $source, $counters);
-          }
-          break;
+            if ($array) {
+              dkan_harvest_cache_pod_1_1_json($array, $key, $source, $counters);
+            }
+            break;
 
-        case 'data.json':
-          $remote = utf8_encode($remote);
-          $json = drupal_json_decode($remote);
-          if (!$json) {
-            # Anticipate encoding errors.
+          case 'data.json':
             $remote = utf8_encode($remote);
             $json = drupal_json_decode($remote);
-          }
-          if ($json) {
-            dkan_harvest_cache_pod_1_1_json($json, $key, $source, $counters);
-          }
-          break;
+            if (!$json) {
+              # Anticipate encoding errors.
+              $remote = utf8_encode($remote);
+              $json = drupal_json_decode($remote);
+            }
+            if ($json) {
+              dkan_harvest_cache_pod_1_1_json($json, $key, $source, $counters);
+            }
+            break;
+        }
       }
     }
+    else {
+      dkan_harvest_log($request->error, 'error');
+    }
   }
   $n = 0;
   foreach ($counters as $key => $counter) {

dkan_harvest_example/dkan_harvest_example.module

@@ -28,7 +28,15 @@ function dkan_harvest_example_harvest_sources() {
         'catalog_type' => 'State Catalog',
       ),
       'overrides' => array(),
+      'attach_headers_callback' => 'dkan_harvest_example_attach_headers',
     ),
   );
   return $sources;
 }
+
+function dkan_harvest_example_attach_headers() {
+  $headers = array(
+    'Accept' => '*/*',
+  );
+  return $headers;
+}

test/DKANHarvestBaseTest.php

@@ -135,4 +135,17 @@ public function testDKANHarvestMigrate()
       $node = $this->getNodeByTitle($title);
       $this->assertEquals($node->title, $title);
     }
+
+    /**
+     * @covers dkan_harvest_cache_data().
+     */
+    public function testDKANHarvestHeadersCallback()
+    {
+      $options = array(
+        'sources' => array('source.with.auth'),
+      );
+      $expected = "4 datasets harvested in total";
+      $actual = dkan_harvest_cache_data($options);
+      $this->assertEquals($expected, $actual);
+    }
 }

test/data_json_server_with_token.php

@@ -0,0 +1,46 @@
+<?php
+
+require(__DIR__ . '/../vendor/autoload.php');
+
+date_default_timezone_set('America/New_York');
+
+use Symfony\Component\HttpFoundation\Request;
+
+$app = new Silex\Application();
+$app['debug'] = false;
+
+$config = array(
+  'token' => 'simpletoken',
+);
+
+// Register the monolog logging service
+if ($app['debug']) {
+  $app->register(new Silex\Provider\MonologServiceProvider(), array(
+    'monolog.logfile' => 'php://stderr',
+  ));
+}
+
+
+$app->get('/', function(Request $request) use($app, $config) {
+  if ($request->headers->get('token') == $config['token']) {
+    $file = __DIR__ . '/./testData.json';
+    if (!file_exists($file)) {
+        return $app->abort(404, 'The image was not found.');
+    }
+
+    $stream = function () use ($file) {
+        readfile($file);
+    };
+
+    return $app->stream($stream, 200, array('Content-Type' => 'application/json'));
+  }
+  return $app->json(
+    array(
+      'reason' => 'wrong token',
+      'text' => 'Please provide a proper token',
+    ),
+    403
+  );
+});
+
+$app->run();

test/dkan_harvest_test.module

@@ -22,5 +22,20 @@ function dkan_harvest_test_harvest_sources() {
         'identifier' => array('c2150dce-db96-4007-ba3f-fb4f3774902d'),
       ),
     ),
+    'source.with.auth' => array(
+      'remote' => 'http://127.0.0.1:8080',
+      'type' => 'data.json',
+      'overrides' => array(),
+      'filters' => array(),
+      'excludes' => array(),
+      'attach_headers_callback' => 'dkan_harvest_test_attach_token',
+    ),
+  );
+}
+
+function dkan_harvest_test_attach_token() {
+  $headers = array(
+    'token' => 'simpletoken',
   );
+  return $headers;
 }