Merge pull request #175 from hcodes/sitemap2

v7.2.0

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v7.2.0
+- Update deps in package.json
+- Fixed sitemap bug #169
+
 ## v7.1.0
 Added junit report #171 @Lootjs.
 

lib/cli/index.js

@@ -44,4 +44,4 @@ async.series(
 ).then(() => {
     reports.onComplete(mergedOptions.configRelativePath);
     process.exit();
-});
+}).catch(() => {});

lib/yaspeller.js

@@ -244,7 +244,7 @@ function checkUrl(url, callback, settings) {
             if (response.ok) {
                 return response.text();
             } else {
-                throw Error(url + ': returns status code is ' + response.statusCode);
+                throw Error(url + ': returns status code is ' + response.status);
             }
         })
         .then(text => {
@@ -282,7 +282,7 @@ function checkSitemap(url, commonCallback, settings, callback) {
             if (res.ok) {
                 return res.text();
             } else {
-                throw Error(url + ': returns status code is ' + res.statusCode);
+                throw Error(url + ': returns status code is ' + res.status);
             }
         })
         .then(xml => {
@@ -297,17 +297,34 @@ function checkSitemap(url, commonCallback, settings, callback) {
                 }
 
                 const tasks = [];
-                if (result && result.urlset && Array.isArray(result.urlset.url)) {
-                    result.urlset.url.forEach(function(el) {
-                        el.loc && el.loc.forEach(function(url) {
-                            tasks.push(function(cb) {
-                                checkUrl(url, function(err, data, originalText) {
-                                    callback && callback(err, data, originalText);
-                                    cb(null, [err, data]);
-                                }, settings);
+                if (result) {
+                    const urlset = result.urlset;
+                    if (urlset && Array.isArray(urlset.url)) {
+                        urlset.url.forEach(function(el) {
+                            el.loc && el.loc.forEach(function(url) {
+                                tasks.push(function(cb) {
+                                    checkUrl(url, function(err, data, originalText) {
+                                        callback && callback(err, data, originalText);
+                                        cb(null, [err, data]);
+                                    }, settings);
+                                });
                             });
                         });
-                    });
+                    }
+
+                    const sitemapindex = result.sitemapindex;
+                    if (sitemapindex && Array.isArray(sitemapindex.sitemap)) {
+                        sitemapindex.sitemap.forEach(function(el) {
+                            el.loc && el.loc.forEach(function(url) {
+                                tasks.push(function(cb) {
+                                    checkUrl(url, function(err, data, originalText) {
+                                        callback && callback(err, data, originalText);
+                                        cb(null, [err, data]);
+                                    }, settings);
+                                });
+                            });
+                        });
+                    }
                 }
 
                 async.parallelLimit(tasks, getMaxRequest(settings), function(err, data) {
@@ -319,6 +336,7 @@ function checkSitemap(url, commonCallback, settings, callback) {
             const obj = [true, error];
             results.push(obj);
             callback && callback.apply(this, obj);
+
             commonCallback(results);
         });
 }
@@ -393,7 +411,7 @@ function removeDuplicates(data) {
             if (Array.isArray(s) && s.length) {
                 obj[code][word].suggest = s;
             }
-            
+
             if (hasPosition) {
                 obj[code][word].position = el.position;
             }