Skip to content

Commit 01d6cc0

Browse files
jonasbardinojonasbardino
committed
Skeleton grid_janitor service as outlined in milestone 8. Ported to next
branch for further development.
1 parent 8d6a5d6 commit 01d6cc0

File tree

9 files changed

+255
-13
lines changed

9 files changed

+255
-13
lines changed

‎mig/install/MiGserver-template.conf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,8 @@ enable_notify = __ENABLE_NOTIFY__
648648
enable_imnotify = __ENABLE_IMNOTIFY__
649649
# Enable users to schedule tasks with a cron/at-like interface
650650
enable_crontab = __ENABLE_CRONTAB__
651+
# Enable janitor servide to handel recurring tasks like clean up and cache updates
652+
enable_janitor = __ENABLE_JANITOR__
651653
# Enable 2FA for web access and IO services with any TOTP authenticator client
652654
# IMPORTANT: Do NOT change this option manually here (requires Apache changes)!
653655
# use generateconfs.py --enable_twofactor=True|False

‎mig/install/migrid-init.d-deb-template

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ MIG_MONITOR=${MIG_CODE}/server/grid_monitor.py
5151
MIG_SSHMUX=${MIG_CODE}/server/grid_sshmux.py
5252
MIG_EVENTS=${MIG_CODE}/server/grid_events.py
5353
MIG_CRON=${MIG_CODE}/server/grid_cron.py
54+
MIG_JANITOR=${MIG_CODE}/server/grid_janitor.py
5455
MIG_TRANSFERS=${MIG_CODE}/server/grid_transfers.py
5556
MIG_OPENID=${MIG_CODE}/server/grid_openid.py
5657
MIG_SFTP=${MIG_CODE}/server/grid_sftp.py
@@ -67,7 +68,7 @@ MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py
6768
show_usage() {
6869
echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]"
6970
echo "where daemon is left out for all or given along with DAEMON as one of the following"
70-
echo "(script|monitor|sshmux|events|cron|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)"
71+
echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)"
7172
}
7273

7374
check_enabled() {
@@ -154,6 +155,19 @@ start_cron() {
154155
log_end_msg 1 || true
155156
fi
156157
}
158+
start_janitor() {
159+
check_enabled "janitor" || return 0
160+
DAEMON_PATH=${MIG_JANITOR}
161+
SHORT_NAME=$(basename ${DAEMON_PATH})
162+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
163+
# NOTE: janitor tasks can be quite heavy so we lower sched prio a bit
164+
log_daemon_msg "Starting MiG janitor daemon" ${SHORT_NAME} || true
165+
if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user ${MIG_USER} --chuid ${MIG_USER} --nicelevel 10 --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then
166+
log_end_msg 0 || true
167+
else
168+
log_end_msg 1 || true
169+
fi
170+
}
157171
start_transfers() {
158172
check_enabled "transfers" || return 0
159173
DAEMON_PATH=${MIG_TRANSFERS}
@@ -267,6 +281,7 @@ start_all() {
267281
start_sshmux
268282
start_events
269283
start_cron
284+
start_janitor
270285
start_transfers
271286
start_openid
272287
start_sftp
@@ -374,6 +389,28 @@ stop_cron() {
374389
log_end_msg 1 || true
375390
fi
376391
}
392+
stop_janitor() {
393+
check_enabled "janitor" || return 0
394+
DAEMON_PATH=${MIG_JANITOR}
395+
SHORT_NAME=$(basename ${DAEMON_PATH})
396+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
397+
log_daemon_msg "Stopping MiG janitor" ${SHORT_NAME} || true
398+
# Try graceful shutdown so that state is properly saved
399+
start-stop-daemon --stop --signal INT --quiet --oknodo --pidfile ${PID_FILE}
400+
for delay in 1 2 3; do
401+
status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} || break
402+
sleep $delay
403+
done
404+
# Force kill if still running
405+
status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} && \
406+
start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE}
407+
if ! status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME}; then
408+
rm -f ${PID_FILE}
409+
log_end_msg 0 || true
410+
else
411+
log_end_msg 1 || true
412+
fi
413+
}
377414
stop_transfers() {
378415
check_enabled "transfers" || return 0
379416
DAEMON_PATH=${MIG_TRANSFERS}
@@ -515,6 +552,7 @@ stop_all() {
515552
stop_sshmux
516553
stop_events
517554
stop_cron
555+
stop_janitor
518556
stop_transfers
519557
stop_openid
520558
stop_sftp
@@ -589,6 +627,18 @@ reload_cron() {
589627
log_end_msg 1 || true
590628
fi
591629
}
630+
reload_janitor() {
631+
check_enabled "janitor" || return 0
632+
DAEMON_PATH=${MIG_JANITOR}
633+
SHORT_NAME=$(basename ${DAEMON_PATH})
634+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
635+
log_daemon_msg "Reloading MiG janitor" ${SHORT_NAME} || true
636+
if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then
637+
log_end_msg 0 || true
638+
else
639+
log_end_msg 1 || true
640+
fi
641+
}
592642
reload_transfers() {
593643
check_enabled "transfers" || return 0
594644
DAEMON_PATH=${MIG_TRANSFERS}
@@ -726,6 +776,7 @@ reload_all() {
726776
reload_sshmux
727777
reload_events
728778
reload_cron
779+
reload_janitor
729780
reload_transfers
730781
reload_openid
731782
reload_sftp
@@ -777,6 +828,13 @@ status_cron() {
777828
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
778829
status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME}
779830
}
831+
status_janitor() {
832+
check_enabled "janitor" || return 0
833+
DAEMON_PATH=${MIG_JANITOR}
834+
SHORT_NAME=$(basename ${DAEMON_PATH})
835+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
836+
status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME}
837+
}
780838
status_transfers() {
781839
check_enabled "transfers" || return 0
782840
DAEMON_PATH=${MIG_TRANSFERS}
@@ -860,6 +918,7 @@ status_all() {
860918
status_sshmux
861919
status_events
862920
status_cron
921+
status_janitor
863922
status_transfers
864923
status_openid
865924
status_sftp
@@ -881,7 +940,7 @@ test -f ${MIG_SCRIPT} || exit 0
881940

882941
# Force valid target
883942
case "$2" in
884-
script|monitor|sshmux|events|cron|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)
943+
script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)
885944
TARGET="$2"
886945
;;
887946
'')

‎mig/install/migrid-init.d-rh-template

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
# processname: grid_sshmux.py
2727
# processname: grid_events.py
2828
# processname: grid_cron.py
29+
# processname: grid_janitor.py
2930
# processname: grid_transfers.py
3031
# processname: grid_openid.py
3132
# processname: grid_sftp.py
@@ -81,6 +82,7 @@ MIG_MONITOR=${MIG_CODE}/server/grid_monitor.py
8182
MIG_SSHMUX=${MIG_CODE}/server/grid_sshmux.py
8283
MIG_EVENTS=${MIG_CODE}/server/grid_events.py
8384
MIG_CRON=${MIG_CODE}/server/grid_cron.py
85+
MIG_JANITOR=${MIG_CODE}/server/grid_janitor.py
8486
MIG_TRANSFERS=${MIG_CODE}/server/grid_transfers.py
8587
MIG_OPENID=${MIG_CODE}/server/grid_openid.py
8688
MIG_SFTP=${MIG_CODE}/server/grid_sftp.py
@@ -97,7 +99,7 @@ MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py
9799
show_usage() {
98100
echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]"
99101
echo "where daemon is left out for all or given along with DAEMON as one of the following"
100-
echo "(script|monitor|sshmux|events|cron|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)"
102+
echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)"
101103
}
102104

103105
check_enabled() {
@@ -215,6 +217,22 @@ start_cron() {
215217
[ $RET2 -ne 0 ] && echo "Warning: cron not started."
216218
echo
217219
}
220+
start_janitor() {
221+
check_enabled "janitor" || return
222+
DAEMON_PATH=${MIG_JANITOR}
223+
SHORT_NAME=$(basename ${DAEMON_PATH})
224+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
225+
# NOTE: janitor tasks can be quite heavy so we lower sched prio a bit
226+
echo -n "Starting MiG janitor daemon: $SHORT_NAME"
227+
daemon +10 --user ${MIG_USER} --pidfile ${PID_FILE} \
228+
"${DAEMON_PATH} >> ${MIG_LOG}/janitor.out 2>&1 &"
229+
fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!"
230+
RET2=$?
231+
[ $RET2 ] && success
232+
echo
233+
[ $RET2 ] || echo "Warning: janitor not started."
234+
echo
235+
}
218236
start_transfers() {
219237
check_enabled "transfers" || return
220238
DAEMON_PATH=${MIG_TRANSFERS}
@@ -356,6 +374,7 @@ start_all() {
356374
start_sshmux
357375
start_events
358376
start_cron
377+
start_janitor
359378
start_transfers
360379
start_openid
361380
start_sftp
@@ -432,6 +451,21 @@ stop_cron() {
432451
echo
433452
status ${DAEMON_PATH} && killproc ${DAEMON_PATH}
434453
}
454+
stop_janitor() {
455+
check_enabled "janitor" || return
456+
DAEMON_PATH=${MIG_JANITOR}
457+
SHORT_NAME=$(basename ${DAEMON_PATH})
458+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
459+
echo -n "Shutting down MiG janitor: $SHORT_NAME "
460+
# Try graceful shutdown so that state is properly saved
461+
killproc ${DAEMON_PATH} -INT
462+
for delay in 1 2 3; do
463+
status ${DAEMON_PATH} || break
464+
sleep $delay
465+
done
466+
echo
467+
status ${DAEMON_PATH} && killproc ${DAEMON_PATH}
468+
}
435469
stop_transfers() {
436470
check_enabled "transfers" || return
437471
DAEMON_PATH=${MIG_TRANSFERS}
@@ -542,6 +576,7 @@ stop_all() {
542576
stop_sshmux
543577
stop_events
544578
stop_cron
579+
stop_janitor
545580
stop_transfers
546581
stop_openid
547582
stop_sftp
@@ -600,6 +635,15 @@ reload_cron() {
600635
killproc ${DAEMON_PATH} -HUP
601636
echo
602637
}
638+
reload_janitor() {
639+
check_enabled "janitor" || return
640+
DAEMON_PATH=${MIG_JANITOR}
641+
SHORT_NAME=$(basename ${DAEMON_PATH})
642+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
643+
echo -n "Reloading MiG janitor: $SHORT_NAME "
644+
killproc ${DAEMON_PATH} -HUP
645+
echo
646+
}
603647
reload_transfers() {
604648
check_enabled "transfers" || return
605649
DAEMON_PATH=${MIG_TRANSFERS}
@@ -717,6 +761,7 @@ reload_all() {
717761
reload_sshmux
718762
reload_events
719763
reload_cron
764+
reload_janitor
720765
reload_transfers
721766
reload_openid
722767
reload_sftp
@@ -768,6 +813,13 @@ status_cron() {
768813
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
769814
status ${DAEMON_PATH}
770815
}
816+
status_janitor() {
817+
check_enabled "janitor" || return
818+
DAEMON_PATH=${MIG_JANITOR}
819+
SHORT_NAME=$(basename ${DAEMON_PATH})
820+
PID_FILE="$PID_DIR/${SHORT_NAME}.pid"
821+
status ${DAEMON_PATH}
822+
}
771823
status_transfers() {
772824
check_enabled "transfers" || return
773825
DAEMON_PATH=${MIG_TRANSFERS}
@@ -852,6 +904,7 @@ status_all() {
852904
status_sshmux
853905
status_events
854906
status_cron
907+
status_janitor
855908
status_transfers
856909
status_openid
857910
status_sftp
@@ -873,7 +926,7 @@ test -f ${MIG_SCRIPT} || exit 0
873926

874927
# Force valid target
875928
case "$2" in
876-
script|monitor|sshmux|events|cron|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)
929+
script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)
877930
TARGET="$2"
878931
;;
879932
'')

‎mig/shared/configuration.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@ def fix_missing(config_file, verbose=True):
358358
'user_vmproxy_log': 'vmproxy.log',
359359
'user_events_log': 'events.log',
360360
'user_cron_log': 'cron.log',
361+
'user_janitor_log': 'janitor.log',
361362
'user_transfers_log': 'transfers.log',
362363
'user_notify_log': 'notify.log',
363364
'user_auth_log': 'auth.log',
@@ -626,6 +627,7 @@ def get(self, *args, **kwargs):
626627
'user_vmproxy_log': 'vmproxy.log',
627628
'user_events_log': 'events.log',
628629
'user_cron_log': 'cron.log',
630+
'user_janitor_log': 'janitor.log',
629631
'user_transfers_log': 'transfers.log',
630632
'user_notify_log': 'notify.log',
631633
'user_auth_log': 'auth.log',
@@ -1554,6 +1556,13 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
15541556
self.site_enable_crontab = False
15551557
if config.has_option('GLOBAL', 'user_cron_log'):
15561558
self.user_cron_log = config.get('GLOBAL', 'user_cron_log')
1559+
if config.has_option('SITE', 'enable_janitor'):
1560+
self.site_enable_janitor = config.getboolean(
1561+
'SITE', 'enable_janitor')
1562+
else:
1563+
self.site_enable_janitor = False
1564+
if config.has_option('GLOBAL', 'user_janitor_log'):
1565+
self.user_janitor_log = config.get('GLOBAL', 'user_janitor_log')
15571566
if config.has_option('SITE', 'enable_notify'):
15581567
self.site_enable_notify = config.getboolean(
15591568
'SITE', 'enable_notify')
@@ -2701,10 +2710,10 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
27012710
'user_openid_log', 'user_monitor_log',
27022711
'user_sshmux_log', 'user_vmproxy_log',
27032712
'user_events_log', 'user_cron_log',
2704-
'user_transfers_log', 'user_notify_log',
2705-
'user_imnotify_log', 'user_auth_log',
2706-
'user_chkuserroot_log', 'user_chksidroot_log',
2707-
'user_quota_log'):
2713+
'user_janitor_log', 'user_transfers_log',
2714+
'user_notify_log', 'user_imnotify_log',
2715+
'user_auth_log', 'user_chkuserroot_log',
2716+
'user_chksidroot_log', 'user_quota_log'):
27082717
_log_path = getattr(self, _log_var)
27092718
if not os.path.isabs(_log_path):
27102719
setattr(self, _log_var, os.path.join(self.log_dir, _log_path))

‎mig/shared/install.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,7 @@ def generate_confs(
387387
enable_seafile=False,
388388
enable_duplicati=False,
389389
enable_crontab=True,
390+
enable_janitor=False,
390391
enable_notify=False,
391392
enable_imnotify=False,
392393
enable_dev_accounts=False,
@@ -711,6 +712,7 @@ def _generate_confs_prepare(
711712
enable_seafile,
712713
enable_duplicati,
713714
enable_crontab,
715+
enable_janitor,
714716
enable_notify,
715717
enable_imnotify,
716718
enable_dev_accounts,
@@ -965,6 +967,7 @@ def _generate_confs_prepare(
965967
user_dict['__ENABLE_SEAFILE__'] = "%s" % enable_seafile
966968
user_dict['__ENABLE_DUPLICATI__'] = "%s" % enable_duplicati
967969
user_dict['__ENABLE_CRONTAB__'] = "%s" % enable_crontab
970+
user_dict['__ENABLE_JANITOR__'] = "%s" % enable_janitor
968971
user_dict['__ENABLE_NOTIFY__'] = "%s" % enable_notify
969972
user_dict['__ENABLE_IMNOTIFY__'] = "%s" % enable_imnotify
970973
user_dict['__ENABLE_DEV_ACCOUNTS__'] = "%s" % enable_dev_accounts
@@ -1763,7 +1766,7 @@ def _generate_confs_prepare(
17631766
else:
17641767
user_dict['__APACHE_SUFFIX__'] = ""
17651768

1766-
# Helpers for the migstatecleanup cron job
1769+
# Helpers for the migstatecleanup cron job or janitor service
17671770
user_dict['__CRON_VERBOSE_CLEANUP__'] = '1'
17681771
user_dict['__CRON_EVENT_CLEANUP__'] = '1'
17691772
if 'migoid' in signup_methods or 'migcert' in signup_methods:
@@ -2519,6 +2522,7 @@ def _generate_confs_instructions(options, user_dict):
25192522
chmod 755 %(destination)s/mig{stateclean,errors,sftpmon,importdoi,notifyexpire}
25202523
sudo cp %(destination)s/mig{stateclean,errors,sftpmon,importdoi,notifyexpire} \\
25212524
/etc/cron.daily/
2525+
until the janitor service is ready to take care of those tasks.
25222526
25232527
The migcheckssl, migverifyarchives, migstats, migacctexpire and miglustrequota
25242528
files are cron scripts to automatically check for LetsEncrypt certificate

‎tests/fixture/confs-stdlocal/MiGserver.conf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,8 @@ enable_notify = False
648648
enable_imnotify = False
649649
# Enable users to schedule tasks with a cron/at-like interface
650650
enable_crontab = True
651+
# Enable janitor servide to handel recurring tasks like clean up and cache updates
652+
enable_janitor = False
651653
# Enable 2FA for web access and IO services with any TOTP authenticator client
652654
# IMPORTANT: Do NOT change this option manually here (requires Apache changes)!
653655
# use generateconfs.py --enable_twofactor=True|False

0 commit comments

Comments
 (0)