Merge pull request #142 from Amsterdam/feature/58866-fix-individual-t…

…ravel-times

Feature/58866 fix individual travel times

src/reistijden_v1/consumer.py

@@ -1,6 +1,7 @@
 import logging
 
 from ingress.consumer.base import BaseConsumer
+from rest_framework.exceptions import ValidationError
 
 from reistijden_v1.parser import ReistijdenParser
 from reistijden_v1.serializers import PublicationSerializer
@@ -14,8 +15,17 @@ class ReistijdenConsumer(BaseConsumer):
     def consume_raw_data(self, raw_data):
         # note that exceptions are not handled here, but must be raised
         # so that the message will be moved to the failed queue.
-        restructured_data = ReistijdenParser(raw_data).restructure_data()
+        try:
+            restructured_data = ReistijdenParser(raw_data).restructure_data()
 
-        publication_serializer = PublicationSerializer(data=restructured_data)
-        publication_serializer.is_valid(raise_exception=True)
-        publication_serializer.save()
+            publication_serializer = PublicationSerializer(data=restructured_data)
+            publication_serializer.is_valid(raise_exception=True)
+            publication_serializer.save()
+        except ValidationError as e:
+            logger.error('Validation Error on consume_raw_data')
+            logger.exception(e)
+            self.on_consume_error(raw_data)
+        except Exception as e:
+            logger.error("Exception called on consume_raw_data")
+            logger.exception(e)
+            self.on_consume_error(raw_data)

src/reistijden_v1/parser.py

@@ -17,7 +17,6 @@ def __init__(self, xml_str):
     def restructure_data(self):
         data_dict = humps.decamelize(xmltodict.parse(self.xml_str.strip()))
         publication_src = data_dict["amsterdam_travel_times"]["payload_publication"]
-
         measurements = []
         if "site_measurements" in publication_src:
             if type(publication_src["site_measurements"]) is list:
@@ -179,21 +178,30 @@ def get_travel_times_from_measurement(self, src_d):
                 travel_times = [self.travel_time_src_to_dict(src_d["travel_time_data"])]
         return travel_times
 
+    ## In Xml for Individual Travel Time the timestamp label is (Camelized) detetection_start/end_time.
+    ## It has been renamed to be more consistent with the timestamp name in publication, used in the model.
+    ## This was decided together with end-user Leon Deckers
     def get_individual_travel_times_from_measurement(self, src_d):
-        individual_travel_times = []
-        if "individual_travel_time_data" in src_d:
-            if type(src_d["individual_travel_time_data"]) is list:
-                individual_travel_times = [
-                    d for d in src_d["individual_travel_time_data"]
-                ]
-            else:
-                individual_travel_times = [src_d["individual_travel_time_data"]]
+
+        individual_travel_times = src_d.get("individual_travel_time_data", [])
+        if not (isinstance(individual_travel_times, list)):
+            individual_travel_times = [individual_travel_times]
+
+        for individual_travel_time in individual_travel_times:
+            self.convert_detection_key(individual_travel_time)
+
         return individual_travel_times
 
-    def get_individual_travel_time(self, src_d):
-        src_d['detection_start_time'] = src_d.pop('start_detection_time', None)
-        src_d['detection_end_time'] = src_d.pop('end_detection_time', None)
-        return src_d
+    def convert_detection_key(self, value):
+        """
+        Convert the names from the detection start/end times to the correct key values
+        detection_start_time -> start_detection_time
+        detection_end_time -> end_detection_time
+        """
+        value['detection_start_time'] = value.pop('start_detection_time', None)
+        value['detection_end_time'] = value.pop('end_detection_time', None)
+
+        return value
 
     def get_traffic_flows_from_measurement(self, src_d):
         traffic_flows = []

src/reistijden_v1/serializers.py

@@ -46,6 +46,13 @@ class Meta:
 
 
 class IndividualTravelTimeSerializer(serializers.ModelSerializer):
+    """
+    Overwrite the detection_start/end_time to be required in the validator
+    """
+
+    detection_start_time = serializers.DateTimeField(required=True)
+    detection_end_time = serializers.DateTimeField(required=True)
+
     class Meta:
         model = IndividualTravelTime
         exclude = ['measurement']
@@ -74,10 +81,10 @@ class Meta:
 
 
 class MeasurementSerializer(serializers.ModelSerializer):
-    measurement_site = MeasurementSiteSerializer()
-    travel_times = TravelTimeSerializer(many=True)
-    individual_travel_times = IndividualTravelTimeSerializer(many=True)
-    traffic_flows = TrafficFlowSerializer(many=True)
+    measurement_site = MeasurementSiteSerializer(required=True)
+    travel_times = TravelTimeSerializer(many=True, required=True)
+    individual_travel_times = IndividualTravelTimeSerializer(many=True, required=True)
+    traffic_flows = TrafficFlowSerializer(many=True, required=True)
 
     class Meta:
         model = Measurement

tests/reistijden_v1/test_parser.py

@@ -0,0 +1,97 @@
+import pytest
+
+from src.reistijden_v1.parser import ReistijdenParser
+
+
+class TestReistijdenParser:
+    @pytest.mark.parametrize(
+        'data, expected_data',
+        [
+            (
+                {
+                    'end_detection_time': 'some_data',
+                    'start_detection_time': 'some_data',
+                },
+                {
+                    'detection_end_time': 'some_data',
+                    'detection_start_time': 'some_data',
+                },
+            ),
+            (
+                {
+                    'some_fake_key': 'some_data',
+                    'start_detection_time': 'some_data',
+                },
+                {
+                    'detection_start_time': 'some_data',
+                    'some_fake_key': 'some_data',
+                    'detection_end_time': None,
+                },
+            ),
+            (
+                {
+                    'end_detection_time': 'some_data',
+                    'some_fake_key': 'some_data',
+                },
+                {
+                    'detection_end_time': 'some_data',
+                    'detection_start_time': None,
+                    'some_fake_key': 'some_data',
+                },
+            ),
+        ],
+    )
+    def test_convert_detection_key(self, data, expected_data):
+        parser = ReistijdenParser('fake_xml')
+        response = parser.convert_detection_key(data)
+        assert response == expected_data
+
+    @pytest.mark.parametrize(
+        'data, expected_data',
+        [
+            (
+                {
+                    'individual_travel_time_data': [
+                        {
+                            'start_detection_time': 'some_data',
+                            'end_detection_time': 'some_data',
+                        },
+                        {
+                            'start_detection_time': 'some_data_2',
+                            'end_detection_time': 'some_data_2',
+                        },
+                    ]
+                },
+                [
+                    {
+                        'detection_start_time': 'some_data',
+                        'detection_end_time': 'some_data',
+                    },
+                    {
+                        'detection_start_time': 'some_data_2',
+                        'detection_end_time': 'some_data_2',
+                    },
+                ],
+            ),
+            (
+                {
+                    'individual_travel_time_data': {
+                        'start_detection_time': 'some_data',
+                        'end_detection_time': 'some_data',
+                    }
+                },
+                [
+                    {
+                        'detection_start_time': 'some_data',
+                        'detection_end_time': 'some_data',
+                    }
+                ],
+            ),
+            ({'no_data': {}}, []),
+        ],
+    )
+    def test_get_individual_travel_times_from_measurement(self, data, expected_data):
+        parser = ReistijdenParser('fake_xml')
+
+        response = parser.get_individual_travel_times_from_measurement(data)
+        assert response == expected_data

tests/api/reistijden_v1/test_reistijden_api.py → tests/reistijden_v1/test_reistijden_api.py

@@ -15,7 +15,7 @@
     TrafficFlowCategoryCount,
     TravelTime,
 )
-from tests.api.reistijden_v1.test_xml import (
+from tests.reistijden_v1.test_xml import (
     TEST_POST_EMPTY,
     TEST_POST_INDIVIDUAL_TRAVEL_TIME,
     TEST_POST_INDIVIDUAL_TRAVEL_TIME_SINGLE_MEASUREMENT,