Skip to content

Commit

Permalink
Add last_pushed_within field to project index
Browse files Browse the repository at this point in the history
For multiple types of our users (CfA Staff, Brigade Leader, Project
Leaders), being able to tell which projects are still active is a
crucial aspect of the index.

In civictechindex#26 we discuss using a bucketed approach so as to not create
unnecessary noise by committing the timestamp for every update. This
commit implements a coarse timestamp: for projects updated within the
last week, month, year, or over a year ago.
  • Loading branch information
tdooner committed Sep 15, 2020
1 parent 2086880 commit 8395864
Showing 1 changed file with 21 additions and 1 deletion.
22 changes: 21 additions & 1 deletion crawler/run.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,23 @@ const githubAxios = axios.create({
: null
});

// Calculate the bucket for when the project was most recently pushed
const ONE_WEEK = 60 * 60 * 24 * 7;
const ONE_MONTH = 60 * 60 * 24 * 30;
const ONE_YEAR = 60 * 60 * 24 * 365;
function lastPushedWithin(repoPushedAt) {
const repoPushedDate = new Date(repoPushedAt);
if (repoPushedDate > new Date() - ONE_WEEK) {
return 'week';
} else if (repoPushedDate > new Date() - ONE_MONTH) {
return 'month';
} else if (repoPushedDate > new Date() - ONE_YEAR) {
return 'year';
} else {
return 'over_a_year';
}
}

require('yargs')
.command({
command: '$0',
Expand Down Expand Up @@ -381,7 +398,8 @@ async function loadGithubOrgProjects(repo, username) {
git_url: repo.git_url,
git_branch: repo.default_branch,
link_url: repo.homepage || null,
topics: repo.topics.length ? repo.topics : null
topics: repo.topics.length ? repo.topics : null,
last_pushed_within: lastPushedWithin(repo.pushed_at),
};
const toml = GitSheets.stringifyRecord(projectData);
const blob = await tree.writeChild(`${repo.name}.toml`, toml);
Expand Down Expand Up @@ -632,6 +650,8 @@ async function loadFeedProjects(repo, projectsListUrl) {
if (projectData.topics) {
projectData.topics = projectData.topics.sort();
}

projectData.last_pushed_within = lastPushedWithin(response.data.pushed_at)
} catch (err) {
if (err.response && err.response.status == 404) {
projectData.flags = [ 'github_404' ]
Expand Down

0 comments on commit 8395864

Please sign in to comment.