diff --git a/docroot/sites/all/modules/custom/pp_comparing/pp_comparing.module b/docroot/sites/all/modules/custom/pp_comparing/pp_comparing.module index 856e9dd..dacaddc 100755 --- a/docroot/sites/all/modules/custom/pp_comparing/pp_comparing.module +++ b/docroot/sites/all/modules/custom/pp_comparing/pp_comparing.module @@ -123,7 +123,7 @@ function _pp_comparing_users_data($users, $types, $dates) { $db_data = $query->execute()->fetchAllAssoc('timestamp'); $beginning = (!empty($dates['from'])) ? $dates['from'] : REQUEST_TIME - PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING; - $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_WEEK_GRANULARITY); $till_time = !empty($dates['till']) ? $dates['till'] : REQUEST_TIME; while ($timestamp < $till_time) { @@ -134,7 +134,7 @@ function _pp_comparing_users_data($users, $types, $dates) { $table_data[$timestamp]['timestamp'] = format_date($timestamp, 'custom', 'j M y'); $table_data[$timestamp][$doid] = $commits; - $timestamp += PPGETSTAT_TIME_PERIOD_GRANULARITY; + $timestamp += PPGETSTAT_TIME_WEEK_GRANULARITY; } } diff --git a/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/adduser.inc b/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/adduser.inc index b1e5ede..1d67f84 100755 --- a/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/adduser.inc +++ b/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/adduser.inc @@ -20,107 +20,12 @@ $plugin = array( */ function pp_frontpage_adduser_content_type_render($subtype, $conf, $panel_args, $context) { $block = new stdClass(); + + $block->title = t('Add users'); $block->module = 'pp_frontpage'; - $block->title = t('Add user'); - $form = drupal_get_form('pp_frontpage_adduser'); - $output = '
' . drupal_render($form) . '
'; + $form = menu_execute_active_handler('admin/config/services/sync_user_list', FALSE); + $block->content = '
' . drupal_render($form) . '
'; - $block->content = $output; return $block; } - -/** - * Form constructor. - */ -function pp_frontpage_adduser($form, $form_state) { - $form['nickname'] = array( - '#type' => 'textfield', - '#title' => t('Drupal.org username or user ID'), - '#required' => TRUE, - ); - - $form['submit'] = array( - '#type' => 'submit', - '#value' => t('Add'), - ); - - return $form; -} - -/** - * Validation handler. - */ -function pp_frontpage_adduser_validate($form, &$form_state) { - if (is_numeric($form_state['values']['nickname'])) { - $uid = $form_state['values']['nickname']; - - $exists = db_query('SELECT nid FROM {node} LEFT JOIN {field_data_field_user_id} AS fuid ON node.nid = fuid.entity_id WHERE type = :type AND fuid.field_user_id_value = :uid', - array( - ':type' => PPGETSTAT_USER_NODE_TYPE, - ':uid' => $uid, - ))->fetchField(); - if (!empty($exists)) { - form_set_error('doid', t('User !link already exists', array( - '!link' => l(t('User with Id=@uid already exists', array('@uid' => $uid)), 'node/' . $exists), - ))); - return; - } - - $nickname = ppgetstat_get_nick_by_id($uid); - if (empty($nickname)) { - form_set_error('doid', t('There is no such user ID on drupal.org')); - } - - $form_state['values']['nickname'] = $nickname; - $form_state['values']['doid'] = $uid; - } - elseif (is_string($form_state['values']['nickname'])) { - $nickname = $form_state['values']['nickname']; - - $exists = db_query('SELECT nid FROM {node} WHERE type = :type AND title = :title', - array( - ':type' => PPGETSTAT_USER_NODE_TYPE, - ':title' => $nickname, - ))->fetchField(); - if (!empty($exists)) { - form_set_error('nickname', t('User !link already exists', array( - '!link' => l($nickname, 'node/' . $exists), - ))); - return; - } - - $doid = ppgetstat_get_id_by_nick($nickname); - if (empty($doid)) { - form_set_error('nickname', t('There is no such user on drupal.org')); - } - - $form_state['values']['doid'] = $doid; - } - - return $form; -} - -/** - * Submit handler. - */ -function pp_frontpage_adduser_submit($form, &$form_state) { - $nickname = $form_state['values']['nickname']; - $doid = $form_state['values']['doid']; - - $node = (object) array( - 'title' => $nickname, - 'type' => PPGETSTAT_USER_NODE_TYPE, - 'field_user_id' => array( - LANGUAGE_NONE => array( - array('value' => $doid), - ) - ), - ); - node_object_prepare($node); - node_save($node); - - drupal_set_message(t('User !link has been created', array( - '!link' => l($nickname, 'node/' . $node->nid), - ))); -} diff --git a/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/community_quality.inc b/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/community_quality.inc index e14283e..3829100 100644 --- a/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/community_quality.inc +++ b/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/community_quality.inc @@ -1,9 +1,9 @@ t('Community quality'), -'single' => TRUE, -'category' => t('PP'), + 'title' => t('Community quality'), + 'single' => TRUE, + 'category' => t('PP'), ); /** @@ -13,14 +13,10 @@ function pp_frontpage_community_quality_content_type_render($subtype, $conf, $pa $block = new stdClass(); $block->module = 'pp_frontpage'; $block->title = t('Community statistic'); - - // Getting info for showing count of individual members at this country. - $members_stat = _ppgetstat_get_individual_members(); - //@TODO add user list filtering for future. $all_user_count = db_select('node', 'n') ->fields('n', array('nid')) - ->condition('n.type', 'user') + ->condition('type', PPGETSTAT_USER_NODE_TYPE) ->countQuery() ->execute() ->fetchField(); @@ -40,16 +36,21 @@ function pp_frontpage_community_quality_content_type_render($subtype, $conf, $pa ->countQuery() ->execute() ->fetchField(); - $individual_members_count = $members_stat['members_country_count']['Ukraine']; - $block->content = '
- -
'; + if ($all_user_count > 0) { + // Getting info for showing count of individual members at this country. + $members_stat = _ppgetstat_get_individual_members(); + $individual_members_count = $members_stat['members_country_count']['Ukraine']; + + $block->content = '
+ +
'; + } return $block; } diff --git a/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/members_by_country.inc b/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/members_by_country.inc index a4feb7a..05f6c13 100644 --- a/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/members_by_country.inc +++ b/docroot/sites/all/modules/custom/pp_frontpage/plugins/content_types/members_by_country.inc @@ -1,7 +1,11 @@ t('Association individual memebers by country stat'), + 'title' => t('Association individual members by country'), 'single' => TRUE, 'category' => t('PP'), ); @@ -9,32 +13,25 @@ $plugin = array( /** * Render global individual members statistics. */ -function pp_frontpage_members_by_country_content_type_render($subtype, $conf, $panel_args, $context) { +function pp_frontpage_members_by_country_content_type_render() { $block = new stdClass(); $block->module = 'pp_frontpage'; $block->title = t('Drupal Association individual members by country statistics'); // Getting info for showing count of individual members at this country. $members_stat = _ppgetstat_get_individual_members(); - $i = 1; $rows = array(); + $rows = array(); + $i = 1; + foreach ($members_stat['members_country_count'] as $country => $count) { - $rows[] = array( - $i, - $country, - $count - ); - $i++; + $rows[] = array($i++, $country, $count); } $block->content = array( '#theme' => 'table', - '#header' => array( - t('Position'), - t('Country'), - t('Count of members') - ), - '#rows' => $rows + '#header' => array(t('Position'), t('Country'), t('Count of members')), + '#rows' => $rows, ); return $block; -} \ No newline at end of file +} diff --git a/docroot/sites/all/modules/custom/ppgetstat/includes/ppgetstat.api.inc b/docroot/sites/all/modules/custom/ppgetstat/includes/ppgetstat.api.inc new file mode 100644 index 0000000..938f522 --- /dev/null +++ b/docroot/sites/all/modules/custom/ppgetstat/includes/ppgetstat.api.inc @@ -0,0 +1,115 @@ +url = rtrim($base, '/'); + } + + /** + * Specify resource of the API. + * + * @param string $resource + * API resource. + * + * @return $this + * APICall instance. + */ + public function resource($resource) { + $this->url .= "/$resource"; + + return $this; + } + + /** + * Configure query. + * + * @param array $variables + * Query parameters. + * + * @return $this + * APICall instance. + */ + public function query(array $variables) { + $this->query = array_merge($this->query, $variables); + + return $this; + } + + /** + * Get a page of results. + * + * @param int $page + * Page number. + * + * @return $this + * APICall instance. + */ + public function page($page = 0) { + if ($page > 0) { + $this->query['page'] = $page; + } + + return $this; + } + + /** + * Perform a request to an API. + * + * @param array $options + * Additional options for drupal_http_request(). + * + * @return stdClass|bool + * Query result or FALSE. + */ + public function execute($options = array()) { + if (!empty($this->query)) { + $this->url .= '?' . http_build_query($this->query); + } + + $this->result = drupal_http_request($this->url, array_merge(array('timeout' => 3000), $options)); + + return 200 == $this->result->code ? json_decode($this->result->data) : FALSE; + } + + /** + * Get waw results of a query. + * + * @return stdClass + * Query result. + */ + public function getRawResult() { + if (empty($this->result)) { + throw new \RuntimeException('You must call the "execute" method firstly.'); + } + + return $this->result; + } + +} diff --git a/docroot/sites/all/modules/custom/ppgetstat/includes/ppgetstat.html.inc b/docroot/sites/all/modules/custom/ppgetstat/includes/ppgetstat.html.inc new file mode 100644 index 0000000..bacf7ba --- /dev/null +++ b/docroot/sites/all/modules/custom/ppgetstat/includes/ppgetstat.html.inc @@ -0,0 +1,61 @@ +document = new DOMDocument(); + // Handle errors/warnings and don't mess up output of your script. + // @see http://stackoverflow.com/a/17559716 + $this->libxmlState = libxml_use_internal_errors(true); + $this->document->loadHTML($content); + } + + /** + * Clear XML library errors. + */ + public function __destruct() { + libxml_clear_errors(); + libxml_use_internal_errors($this->libxmlState); + } + + /** + * Make XPath query to the document object. + * + * @return DOMXPath + * XPath object to perform queries. + */ + public function xpath() { + if (NULL === $this->xpath) { + $this->xpath = new DOMXPath($this->document); + } + + return $this->xpath; + } + +} diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.module b/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.module index 47fbec9..96e3cc0 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.module +++ b/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.module @@ -94,7 +94,7 @@ function _ppcc_parse_core_commits($page_content, $data) { continue; } - $period_timestamp = $commit_timestamp - ($commit_timestamp % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $period_timestamp = $commit_timestamp - ($commit_timestamp % PPGETSTAT_TIME_WEEK_GRANULARITY); $last_commits = variable_get('ppgetstat_ppcc_last_commit_timestamp', array($data['doid'] => array($period_timestamp => 0))); // Save latest commit's date per week into variable. diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.pages.inc b/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.pages.inc index c680cca..693773e 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.pages.inc +++ b/docroot/sites/all/modules/custom/ppgetstat/ppcc/ppcc.pages.inc @@ -66,7 +66,7 @@ function _ppcc_commits_data($node) { )->fetchAllKeyed(); $beginning = REQUEST_TIME - PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING; - $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_WEEK_GRANULARITY); $table_data = array(); while ($timestamp < REQUEST_TIME) { @@ -79,7 +79,7 @@ function _ppcc_commits_data($node) { $commits, ); - $timestamp += PPGETSTAT_TIME_PERIOD_GRANULARITY; + $timestamp += PPGETSTAT_TIME_WEEK_GRANULARITY; } return $table_data; diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.module b/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.module index bfbbcd2..bf756e0 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.module +++ b/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.module @@ -45,23 +45,24 @@ function ppcmnt_ppgetstat_stats_job($user_node) { * Build urls for scanning core commits. */ function _ppcmnt_get_comments_tracking_urls($data) { - $last_scan = $data['last_scan']; $cmt_zero_page = _ppgetstat_fetch_page('https://www.drupal.org/api-d7/comment.json?author=' . urlencode($data['doid'])); $user_data = json_decode($cmt_zero_page); $count = parse_url($user_data->last); + $urls = array(); + parse_str($count['query'], $query); + if (empty($user_data->list)) { - return NULL; + return $urls; } if ($query['page'] == 0) { return array($user_data->self); } // In worst scenario we expect one page of posts per four hours. // See webchick as example https://www.drupal.org/user/24967/track - $days_number = round((REQUEST_TIME - $last_scan) / (4 * 60 * 60)); - $urls = array(); - for ($i = 0; $i <= (int)$query['page']; $i++) { - $urls[] = "https://www.drupal.org/api-d7/comment.json?name=" . urlencode($data['doid']) . "&page=" . $i; + // $days_number = round((REQUEST_TIME - $last_scan) / (4 * 60 * 60)); + for ($i = 0; $i <= (int) $query['page']; $i++) { + $urls[] = 'https://www.drupal.org/api-d7/comment.json?name=' . urlencode($data['doid']) . '&page=' . $i; } return $urls; @@ -71,7 +72,6 @@ function _ppcmnt_get_comments_tracking_urls($data) { * Parse user's posts page and create job items to parse individual post pages. */ function _ppcmnt_parse_posts_list($page_content, $data) { - $nickname = $data['do_nickname']; $last_scan = $data['last_scan']; if (empty($page_content)) { throw new Exception('Empty page content.'); @@ -143,27 +143,24 @@ function _ppcmnt_parse_posts_list($page_content, $data) { * Parse post page for comments. */ function _ppcmnt_parse_post_page($page_content, $data) { - $user_data = json_decode($page_content); - $count = count($user_data->list); + $end = REQUEST_TIME - PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING; - $comments_counter_array = []; - $list = &$user_data->list; + $result = []; - $end = REQUEST_TIME - variable_get('ppgetstat_stats_period'); - for ($i = ($count - 1); $i >= 0; $i--) { - if ($list[$i]->created < $end) { + foreach ($user_data->list as $i => $item) { + if ($item->created < $end) { continue; } - $period_timestamp = $list[$i]->created - ($list[$i]->created % PPGETSTAT_TIME_PERIOD_GRANULARITY); - $comments_counter_array[$period_timestamp][] = $list[$i]->cid; + + $result[$item->created - ($item->created % PPGETSTAT_TIME_WEEK_GRANULARITY)][] = $item->cid; } - if (count($comments_counter_array) != 0) { - $comments_counter_array['#type'] = PPGETSTAT_TYPE_COMMENTS; + if (count($result) > 0) { + $result['#type'] = PPGETSTAT_TYPE_COMMENTS; } - return $comments_counter_array; + return $result; } /** diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.pages.inc b/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.pages.inc index d74a102..83231bc 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.pages.inc +++ b/docroot/sites/all/modules/custom/ppgetstat/ppcmnt/ppcmnt.pages.inc @@ -65,7 +65,7 @@ function _pppcmnt_load_the_data($node) { )->fetchAllKeyed(); $beginning = REQUEST_TIME - PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING; - $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_WEEK_GRANULARITY); $table_data = array(); while ($timestamp < REQUEST_TIME) { @@ -78,7 +78,7 @@ function _pppcmnt_load_the_data($node) { $commits, ); - $timestamp += PPGETSTAT_TIME_PERIOD_GRANULARITY; + $timestamp += PPGETSTAT_TIME_WEEK_GRANULARITY; } return $table_data; diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.info b/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.info index f854bd5..1107ed1 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.info +++ b/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.info @@ -2,3 +2,6 @@ name = ppgetstat description = Get drupal.org statistic core = 7.x version = 1.0 + +files[] = includes/ppgetstat.api.inc +files[] = includes/ppgetstat.html.inc diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.module b/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.module index 88011c7..75c9017 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.module +++ b/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.module @@ -9,11 +9,15 @@ define('PPGETSTAT_USER_NODE_TYPE', 'user'); define('PPGETSTAT_STATSJOBS_PROCESS_NUMBER', 100); // Number of calls to drupal.org per cron run. define('PPGETSTAT_DORGSCRAPPING_PROCESS_NUMBER', 800); +// Hours, minutes, seconds. +define('PPGETSTAT_TIME_DAY_GRANULARITY', 24 * 60 * 60); +// Week timestamp. +define('PPGETSTAT_TIME_WEEK_GRANULARITY', 7 * PPGETSTAT_TIME_DAY_GRANULARITY); +// Month timestamp. +define('PPGETSTAT_TIME_MONTH_GRANULARITY', 30 * PPGETSTAT_TIME_DAY_GRANULARITY); // If no stats available we scan commits for last half a year. -define('PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING', variable_get('ppgetstat_stats_period', 12 * 12 * 30 * 24 * 60 * 60)); -// We group statistics per week. -define('PPGETSTAT_TIME_PERIOD_GRANULARITY', 7 * 24 * 60 * 60); -// Commmits type of data. +define('PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING', ppgetstat_stats_period()); +// Commits type of data. define('PPGETSTAT_TYPE_COMMITS', 1); // Page cache expiry time. define('PPGETSTAT_PAGE_CACHE_EXPIRY', 6 * 60 * 60); @@ -67,51 +71,113 @@ function ppgetstat_menu() { } /** - * Get drupal.org user ID using nickname and http://dgo.to. + * Get/set scanning period. * - * @param string $nick - * Nickname. + * @param int $months + * Number of months. * * @return int - * Drupal.org user id. + * Timestamp. */ -function ppgetstat_get_id_by_nick($nick) { - if ($id_cached = cache_get('ppgetstat_nick_' . $nick)) { - return $id_cached->data; +function ppgetstat_stats_period($months = NULL) { + if (isset($months)) { + $do = 'set'; } - $nick_data = drupal_http_request('http://dgo.to/@' . $nick); - $request = parse_url($nick_data->redirect_url); - $id_data = explode("/", $request['path']); - $id = $id_data[2]; - - if (!is_numeric($id)) { - return FALSE; + else { + // If value doesn't specified then work as getter. + $do = 'get'; + // Default number of months to scan. + // 12 years as default. + $months = 12 * 12; } - cache_set('ppgetstat_nick_' . $nick, $id); + $value = $months * PPGETSTAT_TIME_MONTH_GRANULARITY; + $result = call_user_func("variable_$do", __FUNCTION__, $value); - return $id; + return NULL === $result ? $value : $result; } /** - * Get drupal.org user Nickname using uid and https://www.drupal.org/user/. + * Check that user is already imported from D.org to database. * - * @param string|int $id - * User drupal.org Id. + * @param string|int $argument + * Nickname or an ID of a Drupal.org user. * - * @return bool|string - * User drupal.org nickname or FALSE if not user was found. + * @return int + * Node ID of a user. + */ +function ppgetstat_is_user_stored($argument) { + $query = db_select('node', 'node') + ->fields('node', array('nid')); + + $query->leftJoin('field_data_field_user_id', 'field', 'node.nid = field.entity_id'); + + $or = db_or(); + $or->condition('field.field_user_id_value', $argument); + $or->condition('node.title', $argument); + + // @code + // SELECT nid + // FROM node node + // LEFT JOIN field_data_field_user_id field + // ON node.nid = field.entity_id + // WHERE node.type = 'user' + // AND (field.field_user_id_value = '2802285' OR node.title = 'BR0kEN') + // @endcode + return (int) $query->condition('node.type', PPGETSTAT_USER_NODE_TYPE) + ->condition('node.status', NODE_PUBLISHED) + ->condition($or) + ->execute() + ->fetchField(); +} + +/** + * @param string|int $argument + * Nickname or an ID of a Drupal.org user. + * + * @return stdClass|bool + * Compliance of the user ID to his personal data. */ -function ppgetstat_get_nick_by_id($id) { - $userdata = drupal_http_request('https://www.drupal.org/user/' . $id); +function ppgetstat_get_dorg_user($argument) { + $call = new APICall(); + $result = $call->resource('api-d7/user.json') + ->query(array(is_numeric($argument) ? 'uid' : 'name' => $argument)) + ->execute(); - if ($userdata->code > 400) { - return FALSE; + if (!empty($result->list)) { + // Work with zero element because we're check a single user. + return reset($result->list); } - preg_match('/\

(.+)\<\/h1\>/', $userdata->data, $match); + return FALSE; +} - return (!empty($match[1]) ? $match[1] : FALSE); +/** + * Parse GitHub username from Drupal.org. + * + * @param int $doid + * Drupal.org used ID. + * + * @return string + * GitHub user name. + */ +function ppgetstat_get_github_username_by_doid($doid) { + $page = drupal_http_request("https://www.drupal.org/user/$doid"); + + if (!empty($page->data)) { + $data = new PPGetStatHTML($page->data); + $links = $data + ->xpath() + ->query("//div[contains(@class, 'field-social-links')]//a[contains(@href, 'github')]/@href"); + + if ($links->length > 0) { + $url = parse_url($links[$links->length - 1]->value); + + return ltrim($url['path'], '/'); + } + } + + return ''; } /** @@ -131,12 +197,6 @@ function ppgetstat_cron() { // Remove expired cached pages. _ppgetstat_cron_remove_cached_pages(); - //Start users sync process with d.org - _ppgetstat_cron_create_users_queue(); - - //Process step of sync process. - _ppgetstat_cron_process_users_queue(); - $time_post = microtime(TRUE); $exec_time = round($time_post - $time_pre, 2); watchdog('ppgetstat_cron', $exec_time . 's '); @@ -146,34 +206,47 @@ function ppgetstat_cron() { /** * If period has passed, start rescan of available users list. */ -function _ppgetstat_cron_create_users_queue() { - $last_cron_run = variable_get('ppgetstat_last_users_rescan_timestamp'); +function _ppgetstat_cron_create_users_queue($term_id = 0) { // we are running users lists rescan each 2 days. - if ($last_cron_run > REQUEST_TIME - 6 * 60 * 60) { + if (variable_get('ppgetstat_last_users_rescan_timestamp') > REQUEST_TIME - 6 * 60 * 60) { return; } + + $call = new APICall(); + $call->resource('api-d7/user.json'); + $queue = DrupalQueue::get('ppgetstat_users_scan'); $queue->createQueue(); - $vocab = taxonomy_vocabulary_machine_name_load('users_list'); - $terms = taxonomy_get_tree($vocab->vid); - foreach ($terms as $term) { - $term = taxonomy_term_load($term->tid); + + if ($term_id > 0) { + $terms = array(taxonomy_term_load($term_id)); + } + else { + $terms = taxonomy_get_tree(taxonomy_vocabulary_machine_name_load('users_list')->vid); + } + + foreach (array_filter($terms) as $term) { $field_query = field_get_items('taxonomy_term', $term, 'field_query'); - if (!empty($field_query)) { + + if (FALSE !== $field_query) { $variables = array(); foreach ($field_query as $item) { $variables[$item['first']] = $item['second']; } - $r = _ppgetstat_api_call('https://www.drupal.org/','api-d7/user.json', $variables); - $last_url = parse_url($r->last); - parse_str($last_url['query'], $params); - for ($i = $params['page'] - 1; $i >= 0; $i--) { - $queue->createItem(array( - $variables, $i - )); + + $result = $call->query($variables)->execute(); + + if (!empty($result)) { + $last_url = parse_url($result->last); + parse_str($last_url['query'], $params); + + for ($i = $params['page'] - 1; $i >= 0; $i--) { + $queue->createItem(array($variables, $i)); + } } } } + variable_set('ppgetstat_last_users_rescan_timestamp', REQUEST_TIME); } @@ -181,22 +254,24 @@ function _ppgetstat_cron_create_users_queue() { * Process users queue */ function _ppgetstat_cron_process_users_queue() { - $last_cron_run = variable_get('ppgetstat_last_users_process_timestamp'); - // we are running users lists rescan each 5 minutes. - if ($last_cron_run > REQUEST_TIME - 5 * 60) { + // Цe are running users lists rescan each 5 minutes. + if (variable_get('ppgetstat_last_users_process_timestamp') > REQUEST_TIME - 5 * 60) { return; } $queue = DrupalQueue::get('ppgetstat_users_scan'); + if ($queue->NumberOfItems() > 0) { $item = $queue->claimItem(); + if (!empty($item)) { - $variables = $item->data[0]; - $page = $item->data[1]; + list($variables, $page) = $item->data; _ppgetstat_sync_user_list_step($variables, $page); + $queue->deleteItem($item); } - } else { + } + else { $queue->DeleteQueue(); } @@ -207,9 +282,7 @@ function _ppgetstat_cron_process_users_queue() { * If period has passed, scan each user's commits. */ function _ppgetstat_cron_createItem_stats_jobs() { - $last_cron_run = variable_get('ppgetstat_last_statsjobs_timestamp'); - - if ($last_cron_run > REQUEST_TIME - 24 * 60 * 60) { + if (variable_get('ppgetstat_last_statsjobs_timestamp') > REQUEST_TIME - 24 * 60 * 60) { return; } @@ -222,6 +295,7 @@ function _ppgetstat_cron_createItem_stats_jobs() { // Invoke hook so all modules can use it to provide jobs. foreach (module_implements('ppgetstat_stats_job') as $module) { $job = module_invoke($module, 'ppgetstat_stats_job', $user_node); + if (!empty($job)) { $queue->createItem($job); } @@ -543,7 +617,7 @@ function _ppgetstat_parse_commits($page_content, $data) { break; } - $period_timestamp = $commit_timestamp - ($commit_timestamp % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $period_timestamp = $commit_timestamp - ($commit_timestamp % PPGETSTAT_TIME_WEEK_GRANULARITY); $last_commits = variable_get('ppgetstat_ppgetstat_last_commit_timestamp', array($data['doid'] => array($period_timestamp => 0))); // Save latest commit's date per week into variable. @@ -675,145 +749,92 @@ function ppgetstat_node_insert($node) { _ppgetstat_cron_processItem_stats_jobs(); } -/** - * JSON call - * @param $site - * @param $endpoint - * @param $vars - * @param int $page - * @return mixed - */ -function _ppgetstat_api_call($site, $endpoint, $vars, $page = 0) { - - $url = $site . $endpoint; - if (!empty($vars)) { - $url .= '?'; - $variables = array(); - foreach ($vars as $key => $value) { - if (is_array($value)) { - $value = implode('%2C', $value); - } - $variables[] = $key . '=' . $value; - } - $url .= implode('&', $variables); - } - if (!empty($page)) { - $url .= '&page=' . $page; - } - - $request = drupal_http_request($url, array('timeout' => 3000)); - if ($request->code == 200) { - $JSON = $request->data; - } else { - $JSON = NULL; - } - - return json_decode($JSON); -} - /** * Batch step function for syncing user list. */ function _ppgetstat_sync_user_list_step($variables, $page) { + $call = new APICall(); + $call->resource('api-d7/user.json'); + for ($i = 0; $i < 10; $i++) { - $r = _ppgetstat_api_call( - 'https://www.drupal.org/', - 'api-d7/user.json', - $variables, - $page - ); - if (!empty($r)) { + $result = $call->query($variables)->page($page)->execute(); + + if (!empty($result)) { + // Process list and add/update user nodes. + _ppgetstat_process_dorg_users($result->list); break; } - // wait for next 10 seconds to retry. - sleep(10); - } - - if (!empty($r)) { - $list = $r->list; - $users_data = $uids = array(); - - foreach ($list as $item) { - $uids[] = $item->uid; - $users_data[$item->uid] = $item; + else { + watchdog('ppdorg_debug', t('An error returned during request.')); } - // Process list and add/update user nodes. - _ppgetstat_process_dorg_users($users_data, $uids); - - } else { - watchdog('ppdorg_debug', 'Request returned error'); + // Wait for next 10 seconds to retry. + sleep(10); } - } /** * Helper function that process d.org users list and add/update that object to * user nodes. - * @param $users_data - * @param $doids + * + * @param stdClass[] $users_data + * * @throws \Exception */ -function _ppgetstat_process_dorg_users($users_data, $doids) { +function _ppgetstat_process_dorg_users(array $users_data) { + // Restructure an array with users data. + foreach ($users_data as $i => $data) { + unset($users_data[$i]); - $existed_users_query = db_select('field_data_field_user_id', 'u') - ->fields('u', array('entity_id','field_user_id_value')) - ->condition('u.field_user_id_value', $doids, 'IN') - ->execute() - ->fetchAll(); - $existed_users = array(); - foreach ($existed_users_query as $item) { - $existed_users[$item->field_user_id_value] = $item->entity_id; + if (isset($data->uid)) { + $users_data[$data->uid] = $data; + } } - foreach ($doids as $uid) { + // @code + // array( + // '' => '', + // ); + // @endcode + $existed_users = db_select('field_data_field_user_id', 'u') + ->fields('u', array('field_user_id_value', 'entity_id')) + ->condition('field_user_id_value', array_keys($users_data)) + ->execute() + ->fetchAll(PDO::FETCH_KEY_PAIR|PDO::FETCH_GROUP); + + foreach ($users_data as $uid => $data) { // If user is not tracked yet - creating new empty node. - if (!array_key_exists($uid, $existed_users)) { + if (isset($existed_users[$uid])) { + $node = node_load($existed_users[$uid]); + } + else { $node = new stdClass(); - $node->type = 'user'; + $node->type = PPGETSTAT_USER_NODE_TYPE; + node_object_prepare($node); - } else { - $node = node_load($existed_users[$uid]); } - $node->title = $users_data[$uid]->name; - $node->field_registered_at[LANGUAGE_NONE] = array(array('value' => $users_data[$uid]->created)); - $node->field_user_id['und'] = array(array( - 'value' => $uid, - )); + $node->title = $data->name; - $user_mentors = $users_data[$uid]->field_mentors; - if (!empty($user_mentors)) { - $node->field_mentors[LANGUAGE_NONE] = array(); - foreach ($user_mentors as $mentor) { - $node->field_mentors[LANGUAGE_NONE][] = array( - 'value' => $mentor->id, - ); - } + foreach (array( + 'field_user_id' => $uid, + 'field_github_username' => ppgetstat_get_github_username_by_doid($uid), + ) as $field_name => $value) { + $node->{$field_name}[LANGUAGE_NONE] = array(array('value' => $value)); } - $user_organizations = $users_data[$uid]->field_organizations; - if (!empty($user_organizations)) { - $org_name = array(); - $org_title = array(); - foreach ($user_organizations as $org) { - $collection_request = drupal_http_request($org->uri. '.json'); - if ($collection_request->code == 200) { + if (!empty($data->field_organizations)) { + foreach ($data->field_organizations as $organization) { + $collection_request = drupal_http_request($organization->uri. '.json'); + + if (200 == $collection_request->code) { $collection_data = json_decode($collection_request->data); - $org_name[] = $collection_data->field_organization_name; - $org_title[] = $collection_data->field_job_title; + // Processing user companies. + $node->field_organisations[LANGUAGE_NONE][] = _ppgetstat_vocab_term_tid($collection_data->field_organization_name, 'company'); + // Processing user job titles. + $node->field_job_titles[LANGUAGE_NONE][] = _ppgetstat_vocab_term_tid($collection_data->field_job_title, 'job_title'); } } - // Processing user companies. - $node->field_organisations[LANGUAGE_NONE] = array(); - foreach ($org_name as $on) { - $node->field_organisations[LANGUAGE_NONE][] = _ppgetstat_vocab_term_tid($on, 'company'); - } - // Processing user job titles. - $node->field_job_titles[LANGUAGE_NONE] = array(); - foreach ($org_title as $ot) { - $node->field_job_titles[LANGUAGE_NONE][] = _ppgetstat_vocab_term_tid($ot, 'job_title'); - } } node_save($node); @@ -825,50 +846,69 @@ function _ppgetstat_process_dorg_users($users_data, $doids) { */ function _ppgetstat_vocab_term_tid($name, $vocab_name) { $vocab = taxonomy_vocabulary_machine_name_load($vocab_name); + $name = trim(strtolower($name)); $tid = db_select('taxonomy_term_data', 't') ->fields('t', array('tid')) - ->condition('t.vid', $vocab->vid) - ->condition('t.name', trim(strtolower($name))) + ->condition('vid', $vocab->vid) + ->condition('name', $name) ->execute() ->fetchField(); - if (!empty($tid)) { - return array( - 'tid' => $tid, - ); - } else { - - return array( + if (empty($tid)) { + $term = (object) array( 'vid' => $vocab->vid, - 'tid' => 'autocreate', - 'name' => trim(strtolower($name)), + 'name' => $name, + 'vocabulary_machine_name' => $vocab_name, ); + + // If term will not be saved directly then we'll get duplicates. + taxonomy_term_save($term); + + return array('tid' => $term->tid); } + + return array('tid' => $tid); } /** * Helper function that get info about individual membership. */ function _ppgetstat_get_individual_members() { - $assoc_result = cache_get('pp_frontpage_assoc_members'); - if (empty($assoc_result)) { - $r = _ppgetstat_api_call('https://assoc.drupal.org/', 'api/association_members.json', array()); - $members_stat = array(); - foreach ($r->users as $temp) { - $user = $temp->user; - $user->Country = trim($user->Country); - if (empty($user->Country) || ($user->Country == 1)) { - $user->Country = 'COUNTRY_NOT_DEFINED'; + $cache = cache_get('pp_frontpage_assoc_members'); + + if (empty($cache)) { + $call = new APICall('https://assoc.drupal.org'); + $result = $call->resource('api/association_members.json')->execute(); + + if (!empty($result)) { + $stats = array( + 'members_country' => array(), + 'members_country_count' => array(), + ); + + foreach ($result->users as $item) { + $country = trim($item->user->Country); + + if (empty($country) || 1 == $country) { + $country = 'COUNTRY_NOT_DEFINED'; + } + + if (empty($stats['members_country_count'][$country])) { + $stats['members_country_count'][$country] = 0; + } + + $stats['members_country_count'][$country] += 1; + $stats['members_country'][$country][] = $item->user->user_name; } - $members_stat['members_country_count'][$user->Country] += 1; - $members_stat['members_country'][$user->Country][] = $user->user_name; + + array_multisort($stats['members_country_count'], SORT_DESC, SORT_NUMERIC); + cache_set('pp_frontpage_assoc_members', $stats); + + return $stats; } - array_multisort($members_stat['members_country_count'], SORT_DESC, SORT_NUMERIC); - cache_set('pp_frontpage_assoc_members', $members_stat); - } else { - $members_stat = $assoc_result->data; } - return $members_stat; + + return $cache->data; } /** diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.pages.inc b/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.pages.inc index 1b648fc..5c5b27f 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.pages.inc +++ b/docroot/sites/all/modules/custom/ppgetstat/ppgetstat.pages.inc @@ -65,7 +65,7 @@ function _ppgetstat_get_commits_data($node) { )->fetchAllKeyed(); $beginning = REQUEST_TIME - PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING; - $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $timestamp = $beginning - ($beginning % PPGETSTAT_TIME_WEEK_GRANULARITY); $table_data = array(); while ($timestamp < REQUEST_TIME) { @@ -78,7 +78,7 @@ function _ppgetstat_get_commits_data($node) { $commits, ); - $timestamp += PPGETSTAT_TIME_PERIOD_GRANULARITY; + $timestamp += PPGETSTAT_TIME_WEEK_GRANULARITY; } return $table_data; @@ -87,7 +87,12 @@ function _ppgetstat_get_commits_data($node) { /** * Form builder for reset stats form. */ -function ppgetstat_reset_form($form, $form_state) { +function ppgetstat_reset_form(array $form, array &$form_state) { + $titles = array(); + + foreach (node_load_multiple(array(), array('type' => PPGETSTAT_USER_NODE_TYPE)) as $user_node) { + $titles[] = $user_node->title; + } $form['markup'] = array( '#type' => 'item', @@ -96,17 +101,12 @@ function ppgetstat_reset_form($form, $form_state) { $form['stats_period'] = array( '#type' => 'textfield', - '#title' => t('Scanning period'), + '#title' => t('Scanning period (number of months)'), '#element_validate' => array('element_validate_integer_positive'), - '#default_value' => variable_get('ppgetstat_stats_period', 6 * 30 * 24 * 60 * 60), - '#description' => t('For debugging purposes we can shorten period of grabbing stats. By default it is half a year 15552000, but we can set it to one month 2592000.') + '#default_value' => PPGETSTAT_DEFAULT_TIME_PERIOD_FOR_SCANNING / PPGETSTAT_TIME_MONTH_GRANULARITY, + '#description' => t('For debugging purposes we can shorten period of grabbing stats.'), ); - $titles = array(); - foreach (node_load_multiple(array(), array('type' => PPGETSTAT_USER_NODE_TYPE)) as $user_node) { - $titles[] = $user_node->title; - } - $form['users'] = array( '#type' => 'select', '#multiple' => TRUE, @@ -127,9 +127,9 @@ function ppgetstat_reset_form($form, $form_state) { /** * Submit handler for reset form. */ -function ppgetstat_reset_form_submit($form, $form_state) { +function ppgetstat_reset_form_submit(array $form, array &$form_state) { if (!empty($form_state['values']['stats_period'])) { - variable_set('ppgetstat_stats_period', $form_state['values']['stats_period']); + ppgetstat_stats_period($form_state['values']['stats_period']); variable_del('ppgetstat_last_statsjobs_timestamp'); db_query('TRUNCATE TABLE {ppgetstat}'); @@ -179,33 +179,44 @@ function ppgetstat_reset_form_submit($form, $form_state) { /** * Form builder for reset stats form. */ -function ppgetstat_sync_user_list($form, $form_state) { - // Getting list of all available users list for analyzing. - $user_lists_vocab = taxonomy_vocabulary_machine_name_load('users_list'); - $terms = taxonomy_get_tree($user_lists_vocab->vid); - $options = array( - 0 => t('- None -'), - ); - foreach ($terms as $term) { +function ppgetstat_sync_user_list(array $form, array &$form_state) { + $options = array(); + + // Getting list of all available user lists for analyzing. + foreach (taxonomy_get_tree(taxonomy_vocabulary_machine_name_load('users_list')->vid) as $term) { $options[$term->tid] = $term->name; } + $form['users_list'] = array( '#type' => 'select', - '#title' => t('Select users group scan to'), + '#title' => t('Select users group to scan'), '#options' => $options, + '#empty_value' => 0, + '#disabled' => array('direct_user', array('!value' => '')), ); $form['direct_user'] = array( '#type' => 'textfield', - '#title' => t('Direct user rescan'), - '#description' => t('If this field is not empty, only this user would be updated'), + '#title' => t('User name or ID'), + '#description' => t('If this field is not empty, only this user would be updated.'), + '#disabled' => array('users_list', array('!value' => 0)), ); $form['submit'] = array( '#type' => 'submit', - '#value' => t('test request') + '#value' => t('Submit request') ); + foreach ($form as $item => $definition) { + // Transform custom #disabled property into #states. + if (isset($definition['#disabled']) && is_array($definition['#disabled'])) { + list($field, $conditions) = $definition['#disabled']; + unset($form[$item]['#disabled']); + + $form[$item]['#states']['disabled'][":input[name*=$field]"] = $conditions; + } + } + return $form; } @@ -213,10 +224,9 @@ function ppgetstat_sync_user_list($form, $form_state) { * Submit callback for sync user list form */ function ppgetstat_sync_user_list_submit(array $form, array &$form_state) { - _ppgetstat_cron_create_users_queue(); - if (!empty($form_state['values']['direct_user'])) { $user_data = ppgetstat_get_dorg_user($form_state['values']['direct_user']); + if (FALSE === $user_data) { drupal_set_message(t('User cannot be found by "@argument" as an ID or name.', array( '@argument' => $form_state['values']['direct_user'], @@ -230,36 +240,7 @@ function ppgetstat_sync_user_list_submit(array $form, array &$form_state) { } } elseif (!empty($form_state['values']['users_list'])) { - $term = taxonomy_term_load($form_state['values']['users_list']); - $field_query = field_get_items('taxonomy_term', $term, 'field_query'); - $variables = array('limit' => 50); - if (FALSE !== $field_query) { - foreach ($field_query as $item) { - $variables[$item['first']] = $item['second']; - } - } - $r = _ppgetstat_api_call('https://www.drupal.org/', 'api-d7/user.json', $variables); - if (!empty($r->last)) { - $last_url = parse_url($r->last); - parse_str($last_url['query'], $params); - for ($i = $params['page'] - 1; $i >= 0; $i--) { - $operations[] = array('_ppgetstat_sync_user_list_step', array($variables, $i)); - } - if (isset($operations)) { - /*batch_set(array( - 'title' => t('Processing import...'), - 'finished' => '_ppgetstat_batch_finished', - 'operations' => $operations, - 'init_message' => t('Batch is starting.'), - ));*/ - } - else { - drupal_set_message(t('Nothing to process.'), 'warning'); - } - } - else { - drupal_set_message(t('Error with getting initial data.'), 'error'); - } + _ppgetstat_cron_create_users_queue($form_state['values']['users_list']); } else { drupal_set_message(t('You are not selected any import criteria.'), 'warning'); diff --git a/docroot/sites/all/modules/custom/ppgetstat/ppgh/ppgh.module b/docroot/sites/all/modules/custom/ppgetstat/ppgh/ppgh.module index fac9065..cf66172 100755 --- a/docroot/sites/all/modules/custom/ppgetstat/ppgh/ppgh.module +++ b/docroot/sites/all/modules/custom/ppgetstat/ppgh/ppgh.module @@ -49,8 +49,6 @@ function _ppgh_get_github_activity_tracking_url($data) { * Parse github activity data page. */ function _ppgh_parse_github_activity($page_content, $data) { - $last_scan = $data['last_scan']; - if (empty($page_content)) { throw new Exception('Empty page content.'); } @@ -63,7 +61,7 @@ function _ppgh_parse_github_activity($page_content, $data) { foreach ($json_decoded as $day) { $commit_timestamp = strtotime($day[0]); $day_commits_count = (int) $day[1]; - $period_timestamp = $commit_timestamp - ($commit_timestamp % PPGETSTAT_TIME_PERIOD_GRANULARITY); + $period_timestamp = $commit_timestamp - ($commit_timestamp % PPGETSTAT_TIME_WEEK_GRANULARITY); if (!isset($commits_counter_array[$period_timestamp])) { $commits_counter_array[$period_timestamp] = 0;