-
Notifications
You must be signed in to change notification settings - Fork 0
/
scraper.js
59 lines (59 loc) · 1.69 KB
/
scraper.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
//your app id
var app_id = '';
//timestamp for filename
var time = process.hrtime();
var timestamp = Math.round( time[ 0 ] * 1e3 + time[ 1 ] / 1e6 );
//init csv writer
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
const csvWriter = createCsvWriter({
path: 'app_store_reviews_' + timestamp + '.csv',
header: [
{id: 'userName', title: 'User'},
{id: 'userUrl', title: 'User URL'},
{id: 'version', title: 'App Version'},
{id: 'score', title: 'Stars'},
{id: 'title', title: 'Review Title'},
{id: 'text', title: 'Review Text'},
{id: 'url', title: 'Review URL'},
]
});
//init store-scraper
var store = require('app-store-scraper');
//import countries list
var countries = require('./countries.json')
//start scraping page 1
function reviewScraper(page = 1, country_counter = 0){
store.reviews({
id: app_id,
country: countries[country_counter],
page: page
})
.then(function(data){
//if reviews are returned
if(data != ''){
//write reviews to csv
csvWriter
.writeRecords(data)
//wipe data from previous page
delete data;
//scrape next page
page++;
reviewScraper(page, country_counter);
}
//if no more reviews for that country
else if(data == '' && country_counter != (countries.length - 1)){
//wipe data from previous run
delete data;
//scrape next country
country_counter++;
reviewScraper(page = 1, country_counter);
}
//scrape complete!
else{
console.log('scrape successful')
return false;
}
})
.catch(console.log);
}
reviewScraper();