Skip to content

Commit 0f2d52b

Browse files
author
vg_igsr_adm
committed
changes in ne error
1 parent 95815fd commit 0f2d52b

9 files changed

+286
-34
lines changed

Diff for: ‎config/website_updater.conf

100644100755
+6-31
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
}
2727
},
2828

29-
new_igsr => {
29+
igsr => {
3030
master => {
3131
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
3232
rsync_dests => [
@@ -38,7 +38,7 @@
3838
# pubsubhubbub => 'announcements/rss.xml',
3939

4040
es_sitemap_index => {
41-
hosts => ['wp-np2-1b:9200', 'wp-p1m-a2:9200', 'wp-p2m-3b:9200'],
41+
hosts => [ 'wp-np2-1b:9200', 'wp-p1m-a2:9200', 'wp-p2m-3b:9200'],
4242
search_index_file => 'search_index.json'
4343
},
4444
email_to => '[email protected]',
@@ -54,41 +54,16 @@
5454
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
5555
rsync_dests => [
5656
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
57+
'wp-np2-1c:/nfs/public/rw/igsr/igsr_website_staging/www_static/',
5758
],
5859
email_to => '[email protected]',
5960
},
60-
},
61-
62-
igsr => {
63-
master => {
64-
git_directory => '/nfs/public/rw/reseq-info/igsr_website_staging/gca_1000genomes_website/',
65-
rsync_dests => [
66-
'/nfs/public/rw/reseq-info/igsr_website/www_static/',
67-
'ves-pg-e6:/nfs/public/rw/reseq-info/igsr_website/www_static/',
68-
],
69-
70-
# Disabled pubsubhubbub, to disable automatic tweets
71-
# pubsubhubbub => 'announcements/rss.xml',
72-
73-
es_sitemap_index => {
74-
hosts => ['ves-hx-e4:9200', 'ves-pg-e4:9200', 'ves-oy-e4:9200'],
75-
search_index_file => 'search_index.json'
76-
},
77-
email_to => '[email protected]',
78-
},
79-
'test.internationalgenome.org' => {
80-
git_directory => '/nfs/public/rw/reseq-info/igsr_website_staging/gca_1000genomes_website/',
81-
rsync_dests => [
82-
'/nfs/public/rw/reseq-info/igsr_website_staging/www_static/',
83-
],
84-
email_to => '[email protected]',
85-
},
86-
},
61+
},
8762
},
8863

8964
email_from => '[email protected]',
9065

91-
hypnotoad_log_file => '/nfs/public/rw/igsr/igsr_website_staging/static_website_updater/log/website_updater.hypnotoad.log',
66+
hypnotoad_log_file => '/nfs/public/rw/igsr/static_website_updater/log/website_updater.hypnotoad.log',
9267
hypnotoad => {
9368
# accepts => 1000,
9469
# backlog => 128,
@@ -101,7 +76,7 @@
10176
# keep_alive_requests => 25,
10277
listen => ['http://*:8001'],
10378
# multi_accept => 50,
104-
pid_file => '/nfs/public/rw/igsr/igsr_website_staging/static_website_updater/var/run/website_updater.hypnotoad.pid',
79+
pid_file => '/nfs/public/rw/igsr/static_website_updater/var/run/website_updater.hypnotoad.pid',
10580
# proxy => 0,
10681
# upgrade_timeout => 60,
10782
# user => 'reseq_adm',

Diff for: ‎config/website_updater.conf.forked

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{
2+
projects => {
3+
hipsci => {
4+
master => {
5+
git_directory => '/nfs/public/rw/reseq-info/hipsci_portal_staging/hipsci.github.io/',
6+
rsync_dests => [
7+
'ves-pg-e5:/nfs/public/rw/reseq-info/hipsci_portal/www_static/',
8+
'/nfs/public/rw/reseq-info/hipsci_portal/www_static/',
9+
],
10+
11+
# Disabled pubsubhubbub, to disable automatic tweets
12+
# pubsubhubbub => 'announcements/rss.xml',
13+
14+
es_sitemap_index => {
15+
hosts => ['ves-hx-e4:9200', 'ves-pg-e4:9200', 'ves-oy-e4:9200'],
16+
search_index_file => 'search_index.json'
17+
},
18+
email_to => '[email protected]',
19+
},
20+
'test.hipsci.org' => {
21+
git_directory => '/nfs/public/rw/reseq-info/hipsci_portal_staging/hipsci.github.io/',
22+
rsync_dests => [
23+
'/nfs/public/rw/reseq-info/hipsci_portal_staging/www_static/',
24+
],
25+
email_to => '[email protected]',
26+
}
27+
},
28+
29+
igsr => {
30+
master => {
31+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
32+
rsync_dests => [
33+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
34+
'wp-np2-1c:/nfs/public/rw/igsr/igsr_website_staging/www_static/',
35+
],
36+
email_to => '[email protected]',
37+
es_sitemap_index => {
38+
hosts => [ 'wp-np2-1b:9200'],
39+
search_index_file => 'search_index.json'
40+
},
41+
},
42+
'test.internationalgenome.org' => {
43+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
44+
rsync_dests => [
45+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
46+
],
47+
email_to => '[email protected]',
48+
},
49+
},
50+
51+
igsr_test => {
52+
'test_newvms_elowy' => {
53+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
54+
rsync_dests => [
55+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
56+
],
57+
email_to => '[email protected]',
58+
},
59+
},
60+
61+
},
62+
63+
email_from => '[email protected]',
64+
65+
hypnotoad_log_file => '/nfs/public/rw/igsr/static_website_updater/log/website_updater.hypnotoad.log',
66+
hypnotoad => {
67+
# accepts => 1000,
68+
# backlog => 128,
69+
# clients => 1000,
70+
# graceful_timeout => 20,
71+
# group => 'vg-pub',
72+
# heartbeat_interval => 5,
73+
# heartbeat_timeout => 20,
74+
# inactivity_timeout => 15,
75+
# keep_alive_requests => 25,
76+
listen => ['http://*:8001'],
77+
# multi_accept => 50,
78+
pid_file => '/nfs/public/rw/igsr/static_website_updater/var/run/website_updater.hypnotoad.pid',
79+
# proxy => 0,
80+
# upgrade_timeout => 60,
81+
# user => 'reseq_adm',
82+
83+
# This MUST be set to 1 because we do not handle race conditions between workers
84+
workers => 1,
85+
},
86+
}

Diff for: ‎config/website_updater.conf.old

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
projects => {
3+
hipsci => {
4+
master => {
5+
git_directory => '/nfs/public/rw/reseq-info/hipsci_portal_staging/hipsci.github.io/',
6+
rsync_dests => [
7+
'ves-pg-e5:/nfs/public/rw/reseq-info/hipsci_portal/www_static/',
8+
'/nfs/public/rw/reseq-info/hipsci_portal/www_static/',
9+
],
10+
11+
# Disabled pubsubhubbub, to disable automatic tweets
12+
# pubsubhubbub => 'announcements/rss.xml',
13+
14+
es_sitemap_index => {
15+
hosts => ['ves-hx-e4:9200', 'ves-pg-e4:9200', 'ves-oy-e4:9200'],
16+
search_index_file => 'search_index.json'
17+
},
18+
email_to => '[email protected]',
19+
},
20+
'test.hipsci.org' => {
21+
git_directory => '/nfs/public/rw/reseq-info/hipsci_portal_staging/hipsci.github.io/',
22+
rsync_dests => [
23+
'/nfs/public/rw/reseq-info/hipsci_portal_staging/www_static/',
24+
],
25+
email_to => '[email protected]',
26+
}
27+
},
28+
29+
igsr => {
30+
master => {
31+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
32+
rsync_dests => [
33+
'/nfs/public/rw/igsr/igsr_website/www_static/',
34+
'wp-p1m-a3:/nfs/public/rw/igsr/igsr_website/www_static/',
35+
],
36+
37+
# Disabled pubsubhubbub, to disable automatic tweets
38+
# pubsubhubbub => 'announcements/rss.xml',
39+
40+
es_sitemap_index => {
41+
hosts => [ 'wp-np2-1b:9200', 'wp-p1m-a2:9200', 'wp-p2m-3b:9200'],
42+
search_index_file => 'search_index.json'
43+
},
44+
email_to => '[email protected]',
45+
},
46+
'test.internationalgenome.org' => {
47+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
48+
rsync_dests => [
49+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
50+
],
51+
email_to => '[email protected]',
52+
},
53+
'test_newvms_elowy' => {
54+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
55+
rsync_dests => [
56+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
57+
],
58+
email_to => '[email protected]',
59+
},
60+
},
61+
62+
igsr_test => {
63+
'test_newvms_elowy' => {
64+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
65+
rsync_dests => [
66+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
67+
],
68+
email_to => '[email protected]',
69+
},
70+
},
71+
72+
},
73+
74+
email_from => '[email protected]',
75+
76+
hypnotoad_log_file => '/nfs/public/rw/igsr/static_website_updater/log/website_updater.hypnotoad.log',
77+
hypnotoad => {
78+
# accepts => 1000,
79+
# backlog => 128,
80+
# clients => 1000,
81+
# graceful_timeout => 20,
82+
# group => 'vg-pub',
83+
# heartbeat_interval => 5,
84+
# heartbeat_timeout => 20,
85+
# inactivity_timeout => 15,
86+
# keep_alive_requests => 25,
87+
listen => ['http://*:8001'],
88+
# multi_accept => 50,
89+
pid_file => '/nfs/public/rw/igsr/static_website_updater/var/run/website_updater.hypnotoad.pid',
90+
# proxy => 0,
91+
# upgrade_timeout => 60,
92+
# user => 'reseq_adm',
93+
94+
# This MUST be set to 1 because we do not handle race conditions between workers
95+
workers => 1,
96+
},
97+
}

Diff for: ‎config/website_updater.conf.old1

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
{
2+
projects => {
3+
hipsci => {
4+
master => {
5+
git_directory => '/nfs/public/rw/reseq-info/hipsci_portal_staging/hipsci.github.io/',
6+
rsync_dests => [
7+
'ves-pg-e5:/nfs/public/rw/reseq-info/hipsci_portal/www_static/',
8+
'/nfs/public/rw/reseq-info/hipsci_portal/www_static/',
9+
],
10+
11+
# Disabled pubsubhubbub, to disable automatic tweets
12+
# pubsubhubbub => 'announcements/rss.xml',
13+
14+
es_sitemap_index => {
15+
hosts => ['ves-hx-e4:9200', 'ves-pg-e4:9200', 'ves-oy-e4:9200'],
16+
search_index_file => 'search_index.json'
17+
},
18+
email_to => '[email protected]',
19+
},
20+
'test.hipsci.org' => {
21+
git_directory => '/nfs/public/rw/reseq-info/hipsci_portal_staging/hipsci.github.io/',
22+
rsync_dests => [
23+
'/nfs/public/rw/reseq-info/hipsci_portal_staging/www_static/',
24+
],
25+
email_to => '[email protected]',
26+
}
27+
},
28+
29+
igsr => {
30+
master => {
31+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
32+
rsync_dests => [
33+
'/nfs/public/rw/igsr/igsr_website/www_static/',
34+
'wp-p1m-a3:/nfs/public/rw/igsr/igsr_website/www_static/',
35+
],
36+
37+
# Disabled pubsubhubbub, to disable automatic tweets
38+
# pubsubhubbub => 'announcements/rss.xml',
39+
40+
es_sitemap_index => {
41+
hosts => [ 'wp-np2-1b:9200', 'wp-p1m-a2:9200', 'wp-p2m-3b:9200'],
42+
search_index_file => 'search_index.json'
43+
},
44+
email_to => '[email protected]',
45+
},
46+
'test.internationalgenome.org' => {
47+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
48+
rsync_dests => [
49+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
50+
],
51+
email_to => '[email protected]',
52+
},
53+
},
54+
55+
igsr_test => {
56+
'test_newvms_elowy' => {
57+
git_directory => '/nfs/public/rw/igsr/igsr_website_staging/gca_1000genomes_website/',
58+
rsync_dests => [
59+
'/nfs/public/rw/igsr/igsr_website_staging/www_static/',
60+
],
61+
email_to => '[email protected]',
62+
},
63+
},
64+
65+
},
66+
67+
email_from => '[email protected]',
68+
69+
hypnotoad_log_file => '/nfs/public/rw/igsr/static_website_updater/log/website_updater.hypnotoad.log',
70+
hypnotoad => {
71+
# accepts => 1000,
72+
# backlog => 128,
73+
# clients => 1000,
74+
# graceful_timeout => 20,
75+
# group => 'vg-pub',
76+
# heartbeat_interval => 5,
77+
# heartbeat_timeout => 20,
78+
# inactivity_timeout => 15,
79+
# keep_alive_requests => 25,
80+
listen => ['http://*:8001'],
81+
# multi_accept => 50,
82+
pid_file => '/nfs/public/rw/igsr/static_website_updater/var/run/website_updater.hypnotoad.pid',
83+
# proxy => 0,
84+
# upgrade_timeout => 60,
85+
# user => 'reseq_adm',
86+
87+
# This MUST be set to 1 because we do not handle race conditions between workers
88+
workers => 1,
89+
},
90+
}

Diff for: ‎lib/ReseqTrack/WebsiteUpdater.pm

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package ReseqTrack::WebsiteUpdater;
22
use Mojo::Base 'Mojolicious';
3+
use Data::Dumper;
34

45
sub startup {
56
my ($self) = @_;

Diff for: ‎lib/ReseqTrack/WebsiteUpdater/Controller/WebsiteUpdater.pm

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use ReseqTrack::WebsiteUpdater::Model::GitUpdater;
77
use ReseqTrack::WebsiteUpdater::Model::Rsyncer;
88
use ReseqTrack::WebsiteUpdater::Model::NPMBuilder;
99
use Mojo::IOLoop::ForkCall;
10+
use Data::Dumper;
1011

1112
=pod
1213
@@ -31,7 +32,7 @@ sub update_project {
3132
my ($self) = @_;
3233
eval {
3334
my $payload = $self->req->json;
34-
if (ref($payload) != 'HASH' || !defined $payload->{ref} || ref($payload->{ref})) {
35+
if (ref($payload) ne 'HASH' || !defined $payload->{ref} || ref($payload->{ref})) {
3536
return $self->render(text => 'no "ref" in the payload', status => 400);
3637
}
3738
my ($git_branch) = $payload->{ref} =~ m{^refs/heads/(.+)};
@@ -85,7 +86,6 @@ sub _run_update_process {
8586
ReseqTrack::WebsiteUpdater::Model::NPMBuilder->new(
8687
directory => $project_config->{git_directory},
8788
)->run;
88-
8989
ReseqTrack::WebsiteUpdater::Model::Rsyncer->new(
9090
local_dir => $project_config->{git_directory} .'/_site/',
9191
remote_dests => $project_config->{rsync_dests},

Diff for: ‎lib/ReseqTrack/WebsiteUpdater/Model/ElasticSitemapIndexer.pm

+2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use namespace::autoclean;
33
use Moose;
44
use Mojo::Util qw(slurp);
55
use HTTP::Tiny;
6+
use Data::Dumper;
67
use Mojo::JSON qw(encode_json decode_json);
78
use HTML::Entities qw();
89

@@ -35,6 +36,7 @@ sub load_index {
3536
my $response = $ua->post("$index_type_base_url/_search?search_type=scan&scroll=1m", {
3637
content => encode_json({query => {match_all => {}}}),
3738
});
39+
3840
my $scroll_id = decode_json($response->{content})->{_scroll_id};
3941
die "did not get scroll id" if !$scroll_id;
4042
SCROLL:

0 commit comments

Comments
 (0)