Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check create method + db unit test #6

Merged
merged 39 commits into from
Jul 23, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
06ab246
ResourceProviderAccount
scrungus Jul 4, 2024
38e5725
add consumer_uuid + user_ref
scrungus Jul 4, 2024
4e4ae36
black
scrungus Jul 4, 2024
7bc0b86
create method
scrungus Jul 5, 2024
384202d
add to request
scrungus Jul 5, 2024
7fc3aa5
check_create method
scrungus Jul 5, 2024
242e23b
ConsumerRequest method adapted for blazar requests
scrungus Jul 5, 2024
23625a9
todo select_for_update
scrungus Jul 5, 2024
2449be5
black
scrungus Jul 5, 2024
758eea8
move to ConsumerViewSet
scrungus Jul 9, 2024
fe911f7
handle update case
scrungus Jul 9, 2024
2687b5b
arbitrary length resource request
scrungus Jul 9, 2024
c7eda83
ResourceProviderAccount
scrungus Jul 4, 2024
16b10ad
add consumer_uuid + user_ref
scrungus Jul 4, 2024
b2773b4
black
scrungus Jul 4, 2024
05b7591
Add helm chart with initial functional test (#2)
scrungus Jul 9, 2024
1417ac9
black
scrungus Jul 5, 2024
5dfadcf
move to ConsumerViewSet
scrungus Jul 9, 2024
16ed741
add dry run methods
scrungus Jul 9, 2024
ba767b3
look for 'total' in resource request
scrungus Jul 9, 2024
db201a1
todo
scrungus Jul 9, 2024
4f7ac49
allow empty
scrungus Jul 9, 2024
0bef819
black
scrungus Jul 9, 2024
db4e6fd
unittest-style tests
scrungus Jul 10, 2024
ce4d9b9
test settings
scrungus Jul 10, 2024
1eb6b69
use pytest
scrungus Jul 10, 2024
25fc0ed
consumer route
scrungus Jul 10, 2024
0ae13c7
formatting
scrungus Jul 10, 2024
cd4f5b8
fixes
scrungus Jul 10, 2024
2639563
init test file
scrungus Jul 10, 2024
ebf8d37
add unit test
scrungus Jul 10, 2024
47703e5
black
scrungus Jul 10, 2024
4be7fe2
Merge branch 'add-resourceprovideraccount-table' into check-create-me…
scrungus Jul 10, 2024
fc82077
black
scrungus Jul 10, 2024
56e3e0f
request object classes
scrungus Jul 16, 2024
2eddd7e
exceptions
scrungus Jul 16, 2024
b50b40d
refactor
scrungus Jul 16, 2024
f8944e3
black
scrungus Jul 16, 2024
528fdf3
separate unit tests
scrungus Jul 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Generated by Django 5.0.6 on 2024-07-04 15:54

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("api", "0003_rename_consume_ref_consumer_consumer_ref_and_more"),
]

operations = [
migrations.CreateModel(
name="ResourceProviderAccount",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("project_id", models.UUIDField()),
(
"account",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.creditaccount",
),
),
(
"provider",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="api.resourceprovider",
),
),
],
options={
"unique_together": {
("account", "provider"),
("provider", "project_id"),
},
},
),
migrations.AlterUniqueTogether(
name="consumer",
unique_together=set(),
),
migrations.AddField(
model_name="consumer",
name="resource_provider_account",
field=models.ForeignKey(
default=1,
on_delete=django.db.models.deletion.DO_NOTHING,
to="api.resourceprovideraccount",
),
preserve_default=False,
),
migrations.AlterUniqueTogether(
name="consumer",
unique_together={("consumer_ref", "resource_provider_account")},
),
migrations.RemoveField(
model_name="consumer",
name="account",
),
migrations.RemoveField(
model_name="consumer",
name="resource_provider",
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Generated by Django 5.0.6 on 2024-07-04 16:20

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("api", "0004_resourceprovideraccount_and_more"),
]

operations = [
migrations.AddField(
model_name="consumer",
name="consumer_uuid",
field=models.UUIDField(default=""),
preserve_default=False,
),
migrations.AddField(
model_name="consumer",
name="user_ref",
field=models.UUIDField(default=""),
preserve_default=False,
),
]
35 changes: 32 additions & 3 deletions coral_credits/api/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from auditlog.mixins import LogAccessMixin
from auditlog.registry import auditlog
from django.db import models
from django.db.models import Q

# TODO(tylerchristie): add allocation window in here, to simplify.

Expand Down Expand Up @@ -32,6 +33,24 @@ def __str__(self) -> str:
return f"{self.name}"


class ResourceProviderAccount(models.Model):
account = models.ForeignKey(CreditAccount, on_delete=models.CASCADE)
provider = models.ForeignKey(ResourceProvider, on_delete=models.CASCADE)
project_id = models.UUIDField()

class Meta:
unique_together = (
(
"account",
"provider",
),
("provider", "project_id"),
)

def __str__(self) -> str:
return f"{self.project_id} for {self.account} in {self.provider}"


class CreditAllocation(models.Model):
name = models.CharField(max_length=200)
created = models.DateTimeField(auto_now_add=True)
Expand Down Expand Up @@ -75,16 +94,26 @@ def __str__(self) -> str:

class Consumer(models.Model):
consumer_ref = models.CharField(max_length=200)
resource_provider = models.ForeignKey(ResourceProvider, on_delete=models.DO_NOTHING)
consumer_uuid = models.UUIDField()
resource_provider_account = models.ForeignKey(
ResourceProviderAccount, on_delete=models.DO_NOTHING
)
user_ref = models.UUIDField()
created = models.DateTimeField(auto_now_add=True)
account = models.ForeignKey(CreditAccount, on_delete=models.DO_NOTHING)
start = models.DateTimeField()
end = models.DateTimeField()

class Meta:
# TODO(tylerchristie): allow either/or nullable?
# constraints = [
# models.CheckConstraint(
# check=Q(consumer_ref=False) | Q(consumer_uuid=False),
# name='not_both_null'
# )
# ]
unique_together = (
"consumer_ref",
"resource_provider",
"resource_provider_account",
)

def __str__(self) -> str:
Expand Down
56 changes: 52 additions & 4 deletions coral_credits/api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,56 @@ class Meta:
fields = ["consumer_ref", "resource_provider", "start", "end", "resources"]


class ContextSerializer(serializers.Serializer):
user_id = serializers.UUIDField()
project_id = serializers.UUIDField()
auth_url = serializers.URLField()
region_name = serializers.CharField()

class ResourceRequestSerializer(serializers.Serializer):
# simple case ; we don't distinguish between types of cpu
# TODO(tylerchristie) change this
cpu = serializers.CharField(help_text="Either 'vcpu' or 'pcpu'")
memory = serializers.IntegerField(help_text="Memory in MB")
storage = serializers.IntegerField(help_text="Storage in GB")
storage_type = serializers.CharField()

class AllocationSerializer(serializers.Serializer):
id = serializers.CharField()
hypervisor_hostname = serializers.UUIDField()
extra = serializers.DictField()

class ReservationSerializer(serializers.Serializer):
resource_type = serializers.CharField()
min = serializers.IntegerField()
max = serializers.IntegerField()
hypervisor_properties = serializers.CharField()
resource_properties = serializers.CharField()
allocations = serializers.ListField(child=AllocationSerializer(), required=False, allow_null=True)
resource_requests = ResourceRequestSerializer() # TODO item

class LeaseSerializer(serializers.Serializer):
lease_id = serializers.UUIDField() # TODO item
start_date = serializers.DateTimeField()
end_time = serializers.DateTimeField()
reservations = serializers.ListField(child=ReservationSerializer())

class ConsumerRequest(serializers.Serializer):
consumer_ref = serializers.CharField(max_length=200)
resource_provider_id = serializers.IntegerField()
start = serializers.DateTimeField()
end = serializers.DateTimeField()
def __init__(self, *args, current_lease_required=False, **kwargs):
super().__init__(*args, **kwargs)
# current_lease required on update but not create
self.fields['current_lease'] = LeaseSerializer(
required=current_lease_required,
allow_null=(not current_lease_required)
)

context = ContextSerializer()
lease = LeaseSerializer()

def to_internal_value(self, data):
# Custom validation or processing can be added here if needed
return super().to_internal_value(data)

def to_representation(self, instance):
# Custom representation logic can be added here if needed
return super().to_representation(instance)
152 changes: 151 additions & 1 deletion coral_credits/api/views.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from django.db import transaction
from django.shortcuts import get_object_or_404
from rest_framework import permissions, viewsets
from django.utils import timezone
from datetime import timedelta
from rest_framework import permissions, viewsets, status
from rest_framework.response import Response

from coral_credits.api import models
Expand Down Expand Up @@ -75,6 +77,148 @@ def retrieve(self, request, pk=None):

return Response(account_summary)

@transaction.atomic
def create(self, request, pk=None):
"""
Process a request for a reservation.

Example (blazar) request:
{
"context": {
"user_id": "c631173e-dec0-4bb7-a0c3-f7711153c06c",
"project_id": "a0b86a98-b0d3-43cb-948e-00689182efd4",
"auth_url": "https://api.example.com:5000/v3",
"region_name": "RegionOne"
},
"lease": {
# TODO(assumptionsandg): "lease_id": "e96b5a17-ada0-4034-a5ea-34db024b8e04"
# TODO(assumptionsandg): "lease_name": "e96b5a17-ada0-4034-a5ea-34db024b8e04"
"start_date": "2020-05-13T00:00:00.012345+02:00",
"end_time": "2020-05-14T23:59:00.012345+02:00",
"reservations": [
{
"resource_type": "physical:host",
"min": 2,
"max": 3,
"hypervisor_properties": "[]",
"resource_properties": "[\"==\", \"$availability_zone\", \"az1\"]",
"allocations": [
{
"id": "1",
"hypervisor_hostname": "32af5a7a-e7a3-4883-a643-828e3f63bf54",
"extra": {
"availability_zone": "az1"
}
},
{
"id": "2",
"hypervisor_hostname": "af69aabd-8386-4053-a6dd-1a983787bd7f",
"extra": {
"availability_zone": "az1"
}
}
]
# TODO(assumptionsandg): "resource_requests" :
{
"vcpu" / "pcpu" : "8"
"memory" : "4096" # MB ?
"storage" : "30" # GB ?
"storage_type" : "SSD"
}
}
]
}
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should be able to have the same request format standardised across all resource providers?

@assumptionsandg I've added some TODOs on what additions we need to the request that blazar sends to the coral-credits API, perhaps @JohnGarbutt has some suggestions on this

"""
# Check request is valid
resource_request = serializers.ConsumerRequest(data=request.data, current_lease_required=False)
resource_request.is_valid(raise_exception=True)

context = resource_request.validated_data['context']
lease = resource_request.validated_data['lease']

# Match the project_id with a ResourceProviderAccount
try:
resource_provider_account = models.ResourceProviderAccount.objects.get(project_id=context['project_id'])
except models.ResourceProviderAccount.DoesNotExist:
return Response({"error": "No matching ResourceProviderAccount found"}, status=status.HTTP_403_FORBIDDEN)

# Find all associated active CreditAllocations
# Make sure we only look for CreditAllocations valid for the current time
now = timezone.now()
credit_allocations = models.CreditAllocation.objects.filter(
account=resource_provider_account.account,
start__lte=now,
end__gte=now
).order_by('-start')

if not credit_allocations.exists():
return Response({"error": "No active CreditAllocation found"}, status=status.HTTP_403_FORBIDDEN)

# Calculate lease duration
lease_start = timezone.make_aware(lease['start_date'])
lease_end = timezone.make_aware(lease['end_time'])
lease_duration = (lease_end - lease_start).total_seconds() / 3600 # Convert to hours

# Check resource credit availability (first check)
for resource_type, amount in lease['reservations']['resource_requests'].items():
# Check resource type requested is valid
resource_class = get_object_or_404(models.ResourceClass, name=resource_type)
# Keep it simple, ust take min for now
# TODO(tylerchristie): check we can allocate max
# CreditAllocationResource is a record of the number of resource_hours available for
# one unit of a ResourceClass, so we multiply lease_duration by units required.
requested_resource_hours = float(amount) * lease['reservations']['min'] * lease_duration

for credit_allocation in credit_allocations:
# We know we only get one result because (allocation,resource_class) together is unique
credit_allocation_resource = models.CreditAllocationResource.objects.filter(
allocation=credit_allocation,
resource_class=resource_class
).first()
if credit_allocation_resource:
if credit_allocation_resource.resource_hours < requested_resource_hours:
return Response(
{"error": f"Insufficient {resource_type} credits available"},
status=status.HTTP_403_FORBIDDEN
)
else:
return Response({"error": f"No credit allocated for resource_type {resource_type}"}, status=status.HTTP_403_FORBIDDEN)

# Account has sufficient credits at time of database query, so we allocate resources
# Create Consumer and ResourceConsumptionRecords
consumer = models.Consumer.objects.create(
consumer_ref=lease.get('lease_name'),
consumer_uuid=lease.get('lease_id'),
resource_provider_account=resource_provider_account,
user_ref=context['user_id'],
start=lease_start,
end=lease_end
)

for reservation in lease['reservations']:
for resource_type, amount in reservation['resource_type'].items():
# TODO(tylerchristie) remove code duplication?
resource_class = models.ResourceClass.objects.get(name=resource_type)
resource_hours = float(amount) * reservation['min'] * lease_duration

models.ResourceConsumptionRecord.objects.create(
consumer=consumer,
resource_class=resource_class,
resource_hours=resource_hours
)

# Final check
for credit_allocation_resource in models.CreditAllocationResource.objects.filter(allocation=credit_allocation):
if credit_allocation_resource.resource_hours < 0:
transaction.set_rollback(True)
return Response(
{"error": f"Insufficient {credit_allocation_resource.resource_class.name} credits after allocation"},
status=status.HTTP_403_FORBIDDEN
)

return Response({"message": "Consumer and resources created successfully"}, status=status.HTTP_204_NO_CONTENT)

def update(self, request, pk=None):
"""
Add a resource request
Expand All @@ -95,6 +239,12 @@ def update(self, request, pk=None):
}
]
}

class ConsumerRequest(serializers.Serializer):
consumer_ref = serializers.CharField(max_length=200)
resource_provider_id = serializers.IntegerField()
start = serializers.DateTimeField()
end = serializers.DateTimeField()
"""
resource_request = serializers.ConsumerRequest(data=request.data)
resource_request.is_valid(raise_exception=True)
Expand Down