diff --git a/docs/models/core/datasource.md b/docs/models/core/datasource.md index 0e18a2aae10..527d93939d1 100644 --- a/docs/models/core/datasource.md +++ b/docs/models/core/datasource.md @@ -44,6 +44,12 @@ A set of rules (one per line) identifying filenames to ignore during synchroniza | `*.txt` | Ignore any files with a `.txt` extension | | `data???.json` | Ignore e.g. `data123.json` | +### Sync Interval + +!!! info "This field was introduced in NetBox v4.3." + +The interval at which the data source should automatically synchronize. If not set, the data source must be synchronized manually. + ### Last Synced The date and time at which the source was most recently synchronized successfully. diff --git a/netbox/core/api/serializers_/data.py b/netbox/core/api/serializers_/data.py index 2c155ba6bc5..3f2ddb2a0e6 100644 --- a/netbox/core/api/serializers_/data.py +++ b/netbox/core/api/serializers_/data.py @@ -26,8 +26,8 @@ class Meta: model = DataSource fields = [ 'id', 'url', 'display_url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description', - 'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated', 'last_synced', - 'file_count', + 'sync_interval', 'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated', + 'last_synced', 'file_count', ] brief_fields = ('id', 'url', 'display', 'name', 'description') diff --git a/netbox/core/filtersets.py b/netbox/core/filtersets.py index 21fdaa4abe2..42ec2235058 100644 --- a/netbox/core/filtersets.py +++ b/netbox/core/filtersets.py @@ -29,6 +29,10 @@ class DataSourceFilterSet(NetBoxModelFilterSet): choices=DataSourceStatusChoices, null_value=None ) + sync_interval = django_filters.MultipleChoiceFilter( + choices=JobIntervalChoices, + null_value=None + ) class Meta: model = DataSource diff --git a/netbox/core/forms/bulk_edit.py b/netbox/core/forms/bulk_edit.py index c1f1fca4dc8..73618826d88 100644 --- a/netbox/core/forms/bulk_edit.py +++ b/netbox/core/forms/bulk_edit.py @@ -1,6 +1,7 @@ from django import forms from django.utils.translation import gettext_lazy as _ +from core.choices import JobIntervalChoices from core.models import * from netbox.forms import NetBoxModelBulkEditForm from netbox.utils import get_data_backend_choices @@ -29,6 +30,11 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm): max_length=200, required=False ) + sync_interval = forms.ChoiceField( + choices=JobIntervalChoices, + required=False, + label=_('Sync interval') + ) comments = CommentField() parameters = forms.JSONField( label=_('Parameters'), @@ -42,8 +48,8 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm): model = DataSource fieldsets = ( - FieldSet('type', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules'), + FieldSet('type', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules', 'comments'), ) nullable_fields = ( - 'description', 'description', 'parameters', 'comments', 'parameters', 'ignore_rules', + 'description', 'description', 'sync_interval', 'parameters', 'parameters', 'ignore_rules' 'comments', ) diff --git a/netbox/core/forms/bulk_import.py b/netbox/core/forms/bulk_import.py index 78a859dcb39..a5791c94514 100644 --- a/netbox/core/forms/bulk_import.py +++ b/netbox/core/forms/bulk_import.py @@ -11,5 +11,6 @@ class DataSourceImportForm(NetBoxModelImportForm): class Meta: model = DataSource fields = ( - 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules', + 'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules', + 'comments', ) diff --git a/netbox/core/forms/filtersets.py b/netbox/core/forms/filtersets.py index ab4b869b739..4e72867373d 100644 --- a/netbox/core/forms/filtersets.py +++ b/netbox/core/forms/filtersets.py @@ -27,7 +27,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm): model = DataSource fieldsets = ( FieldSet('q', 'filter_id'), - FieldSet('type', 'status', name=_('Data Source')), + FieldSet('type', 'status', 'enabled', 'sync_interval', name=_('Data Source')), ) type = forms.MultipleChoiceField( label=_('Type'), @@ -46,6 +46,11 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm): choices=BOOLEAN_WITH_BLANK_CHOICES ) ) + sync_interval = forms.ChoiceField( + label=_('Sync interval'), + choices=JobIntervalChoices, + required=False + ) class DataFileFilterForm(NetBoxModelFilterSetForm): diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py index a05377597db..0a683a381e3 100644 --- a/netbox/core/forms/model_forms.py +++ b/netbox/core/forms/model_forms.py @@ -36,7 +36,7 @@ class DataSourceForm(NetBoxModelForm): class Meta: model = DataSource fields = [ - 'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags', + 'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'ignore_rules', 'comments', 'tags', ] widgets = { 'ignore_rules': forms.Textarea( @@ -51,7 +51,10 @@ class Meta: @property def fieldsets(self): fieldsets = [ - FieldSet('name', 'type', 'source_url', 'enabled', 'description', 'tags', 'ignore_rules', name=_('Source')), + FieldSet( + 'name', 'type', 'source_url', 'description', 'tags', 'ignore_rules', name=_('Source') + ), + FieldSet('enabled', 'sync_interval', name=_('Sync')), ] if self.backend_fields: fieldsets.append( diff --git a/netbox/core/migrations/0013_datasource_sync_interval.py b/netbox/core/migrations/0013_datasource_sync_interval.py new file mode 100644 index 00000000000..ec3d2a5d6e6 --- /dev/null +++ b/netbox/core/migrations/0013_datasource_sync_interval.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.6 on 2025-02-26 19:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_job_object_type_optional'), + ] + + operations = [ + migrations.AddField( + model_name='datasource', + name='sync_interval', + field=models.PositiveSmallIntegerField(blank=True, null=True), + ), + ] diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index 39ee8fa575f..3f97fb003ab 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -59,6 +59,12 @@ class DataSource(JobsMixin, PrimaryModel): verbose_name=_('enabled'), default=True ) + sync_interval = models.PositiveSmallIntegerField( + verbose_name=_('sync interval'), + choices=JobIntervalChoices, + blank=True, + null=True + ) ignore_rules = models.TextField( verbose_name=_('ignore rules'), blank=True, diff --git a/netbox/core/signals.py b/netbox/core/signals.py index 06432bf4ce4..bdaa60f9719 100644 --- a/netbox/core/signals.py +++ b/netbox/core/signals.py @@ -8,16 +8,15 @@ from django.utils.translation import gettext_lazy as _ from django_prometheus.models import model_deletes, model_inserts, model_updates -from core.choices import ObjectChangeActionChoices +from core.choices import JobStatusChoices, ObjectChangeActionChoices from core.events import * -from core.models import ObjectChange from extras.events import enqueue_event from extras.utils import run_validators from netbox.config import get_config from netbox.context import current_request, events_queue from netbox.models.features import ChangeLoggingMixin from utilities.exceptions import AbortRequest -from .models import ConfigRevision +from .models import ConfigRevision, DataSource, ObjectChange __all__ = ( 'clear_events', @@ -182,6 +181,25 @@ def clear_events_queue(sender, **kwargs): # DataSource handlers # +@receiver(post_save, sender=DataSource) +def enqueue_sync_job(instance, created, **kwargs): + """ + When a DataSource is saved, check its sync_interval and enqueue a sync job if appropriate. + """ + from .jobs import SyncDataSourceJob + + if instance.enabled and instance.sync_interval: + SyncDataSourceJob.enqueue_once(instance, interval=instance.sync_interval) + elif not created: + # Delete any previously scheduled recurring jobs for this DataSource + for job in SyncDataSourceJob.get_jobs(instance).defer('data').filter( + interval__isnull=False, + status=JobStatusChoices.STATUS_SCHEDULED + ): + # Call delete() per instance to ensure the associated background task is deleted as well + job.delete() + + @receiver(post_sync) def auto_sync(instance, **kwargs): """ diff --git a/netbox/core/tables/data.py b/netbox/core/tables/data.py index 4059ea9bc4d..5d237c689cf 100644 --- a/netbox/core/tables/data.py +++ b/netbox/core/tables/data.py @@ -25,6 +25,9 @@ class DataSourceTable(NetBoxTable): enabled = columns.BooleanColumn( verbose_name=_('Enabled'), ) + sync_interval = columns.ChoiceFieldColumn( + verbose_name=_('Sync interval'), + ) tags = columns.TagColumn( url_name='core:datasource_list' ) @@ -35,10 +38,10 @@ class DataSourceTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = DataSource fields = ( - 'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', - 'created', 'last_updated', 'file_count', + 'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'sync_interval', 'comments', + 'parameters', 'created', 'last_updated', 'file_count', ) - default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count') + default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'sync_interval', 'file_count') class DataFileTable(NetBoxTable): diff --git a/netbox/core/tests/test_filtersets.py b/netbox/core/tests/test_filtersets.py index 310be1d0efc..b7dfd516e24 100644 --- a/netbox/core/tests/test_filtersets.py +++ b/netbox/core/tests/test_filtersets.py @@ -27,7 +27,8 @@ def setUpTestData(cls): source_url='file:///var/tmp/source1/', status=DataSourceStatusChoices.NEW, enabled=True, - description='foobar1' + description='foobar1', + sync_interval=JobIntervalChoices.INTERVAL_HOURLY ), DataSource( name='Data Source 2', @@ -35,14 +36,16 @@ def setUpTestData(cls): source_url='file:///var/tmp/source2/', status=DataSourceStatusChoices.SYNCING, enabled=True, - description='foobar2' + description='foobar2', + sync_interval=JobIntervalChoices.INTERVAL_DAILY ), DataSource( name='Data Source 3', type='git', source_url='https://example.com/git/source3', status=DataSourceStatusChoices.COMPLETED, - enabled=False + enabled=False, + sync_interval=JobIntervalChoices.INTERVAL_WEEKLY ), ) DataSource.objects.bulk_create(data_sources) @@ -73,6 +76,10 @@ def test_status(self): params = {'status': [DataSourceStatusChoices.NEW, DataSourceStatusChoices.SYNCING]} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_sync_interval(self): + params = {'sync_interval': [JobIntervalChoices.INTERVAL_HOURLY, JobIntervalChoices.INTERVAL_DAILY]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests): queryset = DataFile.objects.all() diff --git a/netbox/templates/core/datasource.html b/netbox/templates/core/datasource.html index a5afedec609..0d56c408715 100644 --- a/netbox/templates/core/datasource.html +++ b/netbox/templates/core/datasource.html @@ -46,6 +46,10 @@

{% trans "Data Source" %}

{% trans "Status" %} {% badge object.get_status_display bg_color=object.get_status_color %} + + {% trans "Sync interval" %} + {{ object.get_sync_interval_display|placeholder }} + {% trans "Last synced" %} {{ object.last_synced|placeholder }}