Skip to content

Commit 949d4db

Browse files
andrii-suseandrii-suse
committed
Add Exec service to schedule and execute bash cmds
1 parent 6b96f71 commit 949d4db

17 files changed

Lines changed: 257 additions & 33 deletions

‎dist/systemd/mirrorcache-backstage-exec.service‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
[Unit]
2+
Description=MirrorCache daemon for executing scheduled tasks
3+
4+
[Service]
5+
User=mirrorcache-exec
6+
Group=mirrorcache-exec
7+
ExecStart=/usr/share/mirrorcache/script/mirrorcache-backstage-exec
8+
Nice=19
9+
Restart=on-failure
10+
RestartSec=10
11+
EnvironmentFile=/etc/mirrorcache/conf.env
12+
WorkingDirectory=/var/lib/mirrorcache
13+
Environment="MOJO_TMPDIR=/var/lib/mirrorcache/tmp"
14+
Environment="MOJO_LOG_LEVEL=error"
15+
MemoryHigh=2G
16+
MemoryMax=3G
17+
18+
[Install]
19+
WantedBy=multi-user.target

‎lib/MirrorCache/Task/Exec.pm‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# Copyright (C) 2025 SUSE LLC
2+
#
3+
# This program is free software; you can redistribute it and/or modify
4+
# it under the terms of the GNU General Public License as published by
5+
# the Free Software Foundation; either version 2 of the License, or
6+
# (at your option) any later version.
7+
#
8+
# This program is distributed in the hope that it will be useful,
9+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
# GNU General Public License for more details.
12+
#
13+
# You should have received a copy of the GNU General Public License along
14+
# with this program; if not, see <http://www.gnu.org/licenses/>.
15+
16+
package MirrorCache::Task::Exec;
17+
use diagnostics;
18+
use IPC::Open3;
19+
use IO::Select;
20+
use Symbol 'gensym';
21+
use POSIX ":sys_wait_h";
22+
23+
use Mojo::Base 'Mojolicious::Plugin';
24+
use MirrorCache::Utils 'datetime_now';
25+
26+
# Exec command will execute bash commands provided by argv
27+
#
28+
# Examples to print string "1 2"
29+
# perl:
30+
# my $res = $minion->enqueue(exec => ("echo 1 2"));
31+
# my $res = $minion->enqueue(exec => ("echo", 1, 2));
32+
# my %args = (CMD => 'echo 1', DESC => 'Command that prints "1 2"', TIMEOUT => 1200, LOCK => 'mylock', LOCK_TIMEOUT => 60);
33+
# my $res = $minion->enqueue(exec => (\%args, 1, 2));
34+
#
35+
# shell:
36+
# /usr/share/mirrorcache/script/mirrorcache minion job -e exec -q myqueue -a '["echo 1 2"]'
37+
# /usr/share/mirrorcache/script/mirrorcache minion job -e exec -q myqueue -a '["echo", 1, 2]'
38+
# /usr/share/mirrorcache/script/mirrorcache minion job -e exec -q myqueue -a '[{"CMD":"echo","DESC":"Command that prints","LOCK":"mylock"}, 1, 2]'
39+
40+
sub register {
41+
my ($self, $app) = @_;
42+
$app->minion->add_task(exec => sub { _run($app, @_) });
43+
}
44+
45+
sub _run {
46+
my ($app, $job, $arg0, @argv) = @_;
47+
my $minion = $app->minion;
48+
49+
return $job->finish('No command provided') unless $arg0;
50+
51+
my ($cmd, $cmdline, $desc, $timeout, $lockname, $locktimeout, @args);
52+
53+
if (ref $arg0 eq "HASH") {
54+
$cmdline = $arg0->{CMD};
55+
$desc = $arg0->{DESC};
56+
$timeout = $arg0->{TIMEOUT};
57+
$lockname = $arg0->{LOCK};
58+
$locktimeout = $arg0->{LOCK_TIMEOUT};
59+
} else {
60+
$cmdline = $arg0;
61+
}
62+
63+
my @cmdline = split(/\s/, $cmdline, 2);
64+
if (scalar(@cmdline) > 1) {
65+
$cmd = $cmdline[0];
66+
} else {
67+
$cmd = $cmdline;
68+
}
69+
$desc = $desc // "Command $cmd";
70+
$timeout = $timeout // 1200;
71+
$lockname = $lockname // "EXEC_LOCK_$cmd";
72+
$locktimeout = $locktimeout // $timeout;
73+
74+
return $job->finish("Cannot lock $lockname")
75+
unless my $guard = $minion->guard($lockname, $locktimeout);
76+
77+
my $pid;
78+
my ($infh,$outfh,$errfh);
79+
$errfh = gensym();
80+
81+
my $success = 0;
82+
my $error;
83+
eval {
84+
$pid = open3($infh, $outfh, $errfh, $cmdline);
85+
$success = 1;
86+
};
87+
return $job->fail("open3: $@") unless $success;
88+
close($infh);
89+
90+
my $sel = new IO::Select;
91+
$sel->add($outfh, $errfh);
92+
93+
my $start_time = time;
94+
$job->note(pid => $pid, start_time => $start_time, cmdline => $cmdline);
95+
96+
while(1) {
97+
$! = 0;
98+
my @ready = $sel->can_read(5);
99+
my $last = 0;
100+
if ($!) { # error
101+
$job->note(error_code => $!);
102+
print STDERR "ERRR: $!\n";
103+
waitpid(-1, WNOHANG);
104+
$last = 1;
105+
}
106+
my $curr_time = time;
107+
my (@lines, @elines);
108+
foreach my $fh (@ready) {
109+
my $line = <$fh>;
110+
if(not defined $line){
111+
$sel->remove($fh);
112+
next;
113+
}
114+
chomp($line);
115+
if($fh == $outfh) {
116+
push @lines, $line;
117+
} elsif($fh == $errfh) {# do the same for errfh
118+
push @elines, $line;
119+
}
120+
}
121+
$job->note("$curr_time O" => @lines) if @lines;
122+
$job->note("$curr_time E" => @elines) if @elines;
123+
124+
my $x = waitpid($pid, WNOHANG);
125+
if ($x < 0) {
126+
$job->note(finished => $pid);
127+
$last = 1;
128+
}
129+
130+
last if $last;
131+
if (($curr_time - $start_time) >= $timeout) {
132+
waitpid(-1, WNOHANG);
133+
return $job->fail("timeout expired!");
134+
}
135+
}
136+
137+
waitpid(-1, WNOHANG);
138+
return $job->finish('finish');
139+
}
140+
141+
1;

‎lib/MirrorCache/WebAPI.pm‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -31,23 +31,6 @@ sub new {
3131
# setting pid_file in startup will not help, need to set it earlier
3232
$self->config->{hypnotoad}{pid_file} = $ENV{MIRRORCACHE_HYPNOTOAD_PID} // '/run/mirrorcache/hypnotoad.pid';
3333

34-
# I wasn't able to find reliable way in Mojolicious to detect when WebUI is started
35-
# (e.g. in daemon or hypnotoad in contrast to other commands like backend start / shoot)
36-
# so the code below tries to detect if _setup_ui is needed to be called
37-
38-
my $started = 0;
39-
for (my $i = 0; my @r = caller($i); $i++) {
40-
next unless $r[3] =~ m/Hypnotoad/;
41-
$self->_setup_webui;
42-
$started = 1;
43-
last;
44-
}
45-
46-
$self->hook(before_command => sub {
47-
my ($command, $arg) = @_;
48-
$self->_setup_webui if ref($command) =~ m/daemon|prefork/;
49-
}) unless $started;
50-
5134
$self;
5235
}
5336

@@ -104,15 +87,8 @@ sub startup {
10487
$self->defaults(branding => $ENV{MIRRORCACHE_BRANDING});
10588
$self->defaults(custom_footer_message => $ENV{MIRRORCACHE_CUSTOM_FOOTER_MESSAGE});
10689

107-
$self->plugin('RenderFile');
108-
10990
push @{$self->plugins->namespaces}, 'MirrorCache::WebAPI::Plugin';
110-
11191
$self->plugin('Backstage');
112-
$self->plugin('AuditLog');
113-
$self->plugin('RenderFileFromMirror');
114-
$self->plugin('ReportMirror');
115-
$self->plugin('HashedParams');
11692

11793
if ($geodb_file && $geodb_file =~ /\.mmdb$/i) {
11894
require MaxMind::DB::Reader;
@@ -153,6 +129,21 @@ sub startup {
153129
warn("Could not load plugin from {$plug}: $@\n")
154130
}
155131
}
132+
133+
$self->_setup_webui_plugins if ($ENV{MIRRORCACHE_INTERNAL_SETUP_WEBAPI});
134+
}
135+
136+
sub _setup_webui_plugins {
137+
my ($self) = shift;
138+
my $root = $self->mcconfig->root;
139+
$self->log->info("initializing WebUI");
140+
$self->plugin('RenderFile');
141+
$self->plugin('AuditLog');
142+
$self->plugin('RenderFileFromMirror');
143+
$self->plugin('ReportMirror');
144+
$self->plugin('HashedParams');
145+
146+
$self->_setup_webui;
156147
}
157148

158149
sub _setup_webui {

‎lib/MirrorCache/WebAPI/Plugin/Backstage.pm‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ sub register_tasks {
4747
push @permanent_jobs, 'mirror_provider_sync';
4848
}
4949

50-
$app->plugin($_)
51-
for (
50+
unless ($ENV{MIRRORCACHE_INTERNAL_BACKSTAGE_EXEC}) {
51+
$app->plugin($_) for (
5252
qw(MirrorCache::Task::MirrorCheckFromStat),
5353
qw(MirrorCache::Task::MirrorFileCheck),
5454
qw(MirrorCache::Task::MirrorScanScheduleFromMisses),
@@ -73,6 +73,9 @@ sub register_tasks {
7373
qw(MirrorCache::Task::StatAggSchedule),
7474
qw(MirrorCache::Task::StatAggPkg),
7575
);
76+
} else {
77+
$app->plugin("MirrorCache::Task::Exec");
78+
}
7679
}
7780

7881
sub register {

‎script/mirrorcache-backstage-exec‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh -e
2+
MIRRORCACHE_INTERNAL_BACKSTAGE_EXEC=1 exec "$(dirname "$0")"/mirrorcache backstage run -j ${MIRRORCACHE_BACKSTAGE_WORKERS:-2} -q ${MIRRORCACHE_BACKSTAGE_EXEC_QUEUE:-exec}

‎script/mirrorcache-daemon‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/sh -e
22
echo $MOJO_LISTEN
3-
exec "$(dirname "$0")"/mirrorcache prefork -m production --proxy -w ${MIRRORCACHE_WORKERS:-8} "$@"
3+
MIRRORCACHE_INTERNAL_SETUP_WEBAPI=1 exec "$(dirname "$0")"/mirrorcache prefork -m production --proxy -w ${MIRRORCACHE_WORKERS:-8} "$@"

‎script/mirrorcache-hypnotoad‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/sh -e
22
echo $MOJO_LISTEN
3-
hypnotoad "$(dirname "$0")"/mirrorcache
3+
MIRRORCACHE_INTERNAL_SETUP_WEBAPI=1 hypnotoad "$(dirname "$0")"/mirrorcache
44
[ -f ${MIRRORCACHE_HYPNOTOAD_PID:-/run/mirrorcache/hypnotoad.pid} ] || sleep 0.5
55
[ -f ${MIRRORCACHE_HYPNOTOAD_PID:-/run/mirrorcache/hypnotoad.pid} ] || sleep 0.5
66
[ -f ${MIRRORCACHE_HYPNOTOAD_PID:-/run/mirrorcache/hypnotoad.pid} ] || sleep 0.5

‎t/environ/02-exec.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!lib/test-in-container-environ.sh
2+
set -exo pipefail
3+
4+
mc=$(environ mc $(pwd))
5+
6+
# pass too big value for prev_stat_id and make sure it is automatically adjusted
7+
$mc/backstage/job -e exec -a '["ping google.com -c 3 & (sleep 1; pwd; errreerr; pwd)"]'
8+
$mc/backstage/job -e exec -a '["mkdir tttttt"]'
9+
$mc/backstage-exec/shoot
10+
11+
$mc/backstage/job -e exec -a '["(sleep 1; pwd; errreerr; pwd) & ping -c 20 google.com"]'
12+
$mc/backstage/job -e exec -a '[{"CMD": "(sleep 1; pwd; errreerr; pwd) & ping -c 20 google.com", "TIMEOUT": 6}]'
13+
$mc/backstage-exec/shoot
14+
15+
# mc1/start
16+
ls -lRa $mc | grep tttttt
17+
18+
$mc/sql 'select * from minion_jobs'
19+
20+
$mc/sql_test 3 == "select count(*) from minion_jobs where state = 'finished'"
21+
$mc/sql_test 1 == "select count(*) from minion_jobs where state = 'failed'"
22+
23+
echo success

‎t/lib/Dockerfile.environ.mariadb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-P
1414
perl-Config-IniFiles environ eatmydata
1515

1616
# optional dependencies used in testing
17-
RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools
17+
RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools iputils
1818

1919
# this hack is needed because old nginx versions cannot run as non-root
2020
RUN bbe -e 's,/var/log/nginx/error.log,/tmp/log_nginx_error.log,' /usr/sbin/nginx > /usr/sbin/nginx.hacked

‎t/lib/Dockerfile.environ.mariadb.experimental‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-P
1616
perl-Config-IniFiles environ eatmydata
1717

1818
# optional dependencies used in testing
19-
RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools
19+
RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools iputils
2020

2121
RUN zypper -vvv -n install MariaDB-server-compat MariaDB-client-compat
2222

0 commit comments

Comments
 (0)