add convert flights action and add flightnumber endpoint

cdk/lib/constructs/api-lambda-construct.ts

@@ -42,7 +42,7 @@ export class ApiLambdaConstruct extends Construct {
     const lambda = new Function(this, 'ApiLambda', {
       runtime: Runtime.PROVIDED_AL2023,
       architecture: Architecture.ARM_64,
-      memorySize: 1024,
+      memorySize: 2048,
       timeout: Duration.seconds(30),
       code: Code.fromAsset(props.apiLambdaZipPath),
       handler: 'bootstrap',
@@ -84,6 +84,7 @@ export class ApiLambdaConstruct extends Construct {
     }));
 
     props.dataBucket.grantRead(lambda, 'processed/flights/*');
+    props.dataBucket.grantRead(lambda, 'processed/flight_numbers/*');
     props.dataBucket.grantRead(lambda, 'raw/ourairports_data/airports.csv');
     props.dataBucket.grantRead(lambda, 'raw/ourairports_data/countries.csv');
     props.dataBucket.grantRead(lambda, 'raw/ourairports_data/regions.csv');

cdk/lib/constructs/cron-lambda-construct.ts

@@ -47,5 +47,6 @@ export class CronLambdaConstruct extends Construct {
     props.dataBucket.grantWrite(this.lambda, 'raw/LH_Public_Data/*');
     props.dataBucket.grantWrite(this.lambda, 'raw/ourairports_data/*');
     props.dataBucket.grantReadWrite(this.lambda, 'processed/flights/*');
+    props.dataBucket.grantReadWrite(this.lambda, 'processed/flight_numbers/*');
   }
 }

cdk/lib/constructs/sfn-construct.ts

@@ -60,6 +60,22 @@ export class SfnConstruct extends Construct {
         resultPath: '$.convertSchedulesResponse',
         retryOnServiceExceptions: true,
       }))
+      .next(new LambdaInvoke(this, 'ConvertFlightsTask', {
+        lambdaFunction: props.cronLambda,
+        payload: TaskInput.fromObject({
+          'action': 'convert_flights',
+          'params': {
+            'inputBucket': props.dataBucket.bucketName,
+            'inputPrefix': 'processed/flights/',
+            'outputBucket': props.dataBucket.bucketName,
+            'outputPrefix': 'processed/flight_numbers/',
+            'dateRanges': JsonPath.objectAt('$.convertSchedulesResponse.dateRanges'),
+          },
+        }),
+        payloadResponseOnly: true,
+        resultPath: '$.convertNumbersResponse',
+        retryOnServiceExceptions: true,
+      }))
       .toSingleState('ConvertTry', { outputPath: '$[0]' })
       .addCatch(
         this.sendWebhookTask(
@@ -75,9 +91,10 @@ export class SfnConstruct extends Construct {
         props.cronLambda,
         props.webhookUrl,
         JsonPath.format(
-          'FlightSchedules Cron {} succeeded:\n```json\n{}\n```',
+          'FlightSchedules Cron {} succeeded:\nQueried:\n```json\n{}\n```\nTouched:\n```json\n{}\n```',
           JsonPath.stringAt('$.time'),
           JsonPath.jsonToString(JsonPath.objectAt('$.loadSchedulesResponse.loadFlightSchedules.input.dateRanges')),
+          JsonPath.jsonToString(JsonPath.objectAt('$.convertSchedulesResponse.dateRanges')),
         ),
       ))
       .next(success);

go/api/data/handler.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/csv"
 	"errors"
+	"fmt"
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/explore-flights/monorepo/go/common"
@@ -255,6 +256,29 @@ func (h *Handler) Aircraft(ctx context.Context) ([]Aircraft, error) {
 	return result, err
 }
 
+func (h *Handler) FlightSchedule(ctx context.Context, fnRaw string) (*common.FlightSchedule, error) {
+	fn, err := common.ParseFlightNumber(fnRaw)
+	if err != nil {
+		return nil, err
+	}
+
+	fs, err := loadJson[*common.FlightSchedule](
+		ctx,
+		h,
+		"processed/flight_numbers/"+fmt.Sprintf("%s/%d%s.json", fn.Airline, fn.Number, fn.Suffix),
+	)
+
+	if err != nil {
+		if adapt.IsS3NotFound(err) {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+
+	return fs, nil
+}
+
 func (h *Handler) FlightNumber(ctx context.Context, fnRaw, airport string, d common.LocalDate) (*common.Flight, error) {
 	flights, err := loadJson[[]*common.Flight](
 		ctx,

go/api/main.go

@@ -47,6 +47,7 @@ func main() {
 
 	jsonConnEdp := web.NewConnectionsEndpoint(connHandler, "json")
 	pngConnEdp := web.NewConnectionsEndpoint(connHandler, "png")
+	fnEdp := web.NewFlightNumberEndpoint(dataHandler)
 
 	e.POST("/api/connections/json", jsonConnEdp)
 	e.GET("/api/connections/json/:payload", jsonConnEdp)
@@ -61,9 +62,10 @@ func main() {
 	e.GET("/auth/oauth2/login/:issuer", authHandler.Login)
 	e.GET("/auth/oauth2/code/:issuer", authHandler.Code)
 
-	e.GET("/data/airports.json", web.NewAirportsHandler(dataHandler))
-	e.GET("/data/aircraft.json", web.NewAircraftHandler(dataHandler))
-	e.GET("/data/flight/:fn/:airport/:date", web.NewFlightNumberHandler(dataHandler))
+	e.GET("/data/airports.json", web.NewAirportsEndpoint(dataHandler))
+	e.GET("/data/aircraft.json", web.NewAircraftEndpoint(dataHandler))
+	e.GET("/data/flight/:fn", fnEdp)
+	e.GET("/data/flight/:fn/:airport/:date", fnEdp)
 
 	if err := run(ctx, e); err != nil {
 		panic(err)

go/api/web/data.go

@@ -13,64 +13,55 @@ func noCache(c echo.Context) {
 	c.Response().Header().Set(echo.HeaderCacheControl, "private, no-cache, no-store, max-age=0, must-revalidate")
 }
 
-func NewAirportsHandler(dh *data.Handler) echo.HandlerFunc {
+func NewAirportsEndpoint(dh *data.Handler) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		airports, err := dh.Airports(c.Request().Context())
-		if err != nil {
-			noCache(c)
-
-			if errors.Is(err, context.DeadlineExceeded) {
-				return echo.NewHTTPError(http.StatusRequestTimeout, err)
-			}
-
-			return echo.NewHTTPError(http.StatusInternalServerError)
-		}
-
-		return c.JSON(http.StatusOK, airports)
+		return jsonResponse(c, airports, err, func(v data.AirportsResponse) bool { return false })
 	}
 }
 
-func NewAircraftHandler(dh *data.Handler) echo.HandlerFunc {
+func NewAircraftEndpoint(dh *data.Handler) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		aircraft, err := dh.Aircraft(c.Request().Context())
-		if err != nil {
-			noCache(c)
-
-			if errors.Is(err, context.DeadlineExceeded) {
-				return echo.NewHTTPError(http.StatusRequestTimeout, err)
-			}
-
-			return echo.NewHTTPError(http.StatusInternalServerError)
-		}
-
-		return c.JSON(http.StatusOK, aircraft)
+		return jsonResponse(c, aircraft, err, func(v []data.Aircraft) bool { return false })
 	}
 }
 
-func NewFlightNumberHandler(dh *data.Handler) echo.HandlerFunc {
+func NewFlightNumberEndpoint(dh *data.Handler) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		fn := c.Param("fn")
 		airport := c.Param("airport")
-		d, err := common.ParseLocalDate(c.Param("date"))
-		if err != nil {
-			return echo.NewHTTPError(http.StatusBadRequest, err)
-		}
-
-		flight, err := dh.FlightNumber(c.Request().Context(), fn, airport, d)
-		if err != nil {
-			noCache(c)
+		dateRaw := c.Param("date")
 
-			if errors.Is(err, context.DeadlineExceeded) {
-				return echo.NewHTTPError(http.StatusRequestTimeout, err)
+		if airport != "" && dateRaw != "" {
+			d, err := common.ParseLocalDate(dateRaw)
+			if err != nil {
+				return echo.NewHTTPError(http.StatusBadRequest, err)
 			}
 
-			return echo.NewHTTPError(http.StatusInternalServerError)
+			flight, err := dh.FlightNumber(c.Request().Context(), fn, airport, d)
+			return jsonResponse(c, flight, err, func(v *common.Flight) bool { return v == nil })
+		} else {
+			fs, err := dh.FlightSchedule(c.Request().Context(), fn)
+			return jsonResponse(c, fs, err, func(v *common.FlightSchedule) bool { return v == nil })
 		}
+	}
+}
 
-		if flight == nil {
-			return echo.NewHTTPError(http.StatusNotFound)
+func jsonResponse[T any](c echo.Context, v T, err error, isEmpty func(T) bool) error {
+	if err != nil {
+		noCache(c)
+
+		if errors.Is(err, context.DeadlineExceeded) {
+			return echo.NewHTTPError(http.StatusRequestTimeout, err)
 		}
 
-		return c.JSON(http.StatusOK, flight)
+		return echo.NewHTTPError(http.StatusInternalServerError)
+	}
+
+	if isEmpty(v) {
+		return echo.NewHTTPError(http.StatusNotFound)
 	}
+
+	return c.JSON(http.StatusOK, v)
 }

go/common/flight_schedule.go

@@ -1,7 +1,8 @@
 package common
 
 import (
-	"maps"
+	"slices"
+	"time"
 )
 
 type FlightScheduleData struct {
@@ -13,8 +14,6 @@ type FlightScheduleData struct {
 	AircraftOwner                AirlineIdentifier `json:"aircraftOwner"`
 	AircraftType                 string            `json:"aircraftType"`
 	AircraftConfigurationVersion string            `json:"aircraftConfigurationVersion"`
-	Registration                 string            `json:"registration"`
-	DataElements                 map[int]string    `json:"dataElements"`
 	CodeShares                   []FlightNumber    `json:"codeShares"`
 }
 
@@ -27,18 +26,27 @@ func (fsd FlightScheduleData) Equal(other FlightScheduleData) bool {
 		fsd.AircraftOwner == other.AircraftOwner &&
 		fsd.AircraftType == other.AircraftType &&
 		fsd.AircraftConfigurationVersion == other.AircraftConfigurationVersion &&
-		fsd.Registration == other.Registration &&
-		maps.Equal(fsd.DataElements, other.DataElements) &&
 		SliceEqualContent(fsd.CodeShares, other.CodeShares)
 }
 
+type FlightScheduleAlias struct {
+	FlightNumber     FlightNumber `json:"flightNumber"`
+	DepartureTime    OffsetTime   `json:"departureTime"`
+	DepartureAirport string       `json:"departureAirport"`
+}
+
 type FlightScheduleVariant struct {
-	Ranges []LocalDateRange   `json:"ranges"`
-	Data   FlightScheduleData `json:"data"`
+	Ranges LocalDateRanges      `json:"ranges"`
+	Data   *FlightScheduleData  `json:"data,omitempty"`
+	Alias  *FlightScheduleAlias `json:"alias,omitempty"`
 }
 
-func (fsv *FlightScheduleVariant) Expand(d LocalDate) {
+func (fsv *FlightScheduleVariant) DepartureTime() OffsetTime {
+	if fsv.Data != nil {
+		return fsv.Data.DepartureTime
+	}
 
+	return fsv.Alias.DepartureTime
 }
 
 type FlightSchedule struct {
@@ -48,12 +56,35 @@ type FlightSchedule struct {
 	Variants     []*FlightScheduleVariant `json:"variants"`
 }
 
-func (fs *FlightSchedule) DataVariant(fsd FlightScheduleData) *FlightScheduleVariant {
+func (fs *FlightSchedule) RemoveVariants(start, end time.Time) {
+	fs.Variants = slices.DeleteFunc(fs.Variants, func(variant *FlightScheduleVariant) bool {
+		for d := range variant.Ranges.Iter() {
+			t := variant.DepartureTime().Time(d)
+			if t.Compare(start) >= 0 && t.Compare(end) <= 0 {
+				variant.Ranges = variant.Ranges.Remove(d)
+			}
+		}
+
+		return len(variant.Ranges) < 1
+	})
+}
+
+func (fs *FlightSchedule) DataVariant(fsd FlightScheduleData) (*FlightScheduleVariant, bool) {
+	for _, variant := range fs.Variants {
+		if variant.Data != nil && variant.Data.Equal(fsd) {
+			return variant, true
+		}
+	}
+
+	return nil, false
+}
+
+func (fs *FlightSchedule) AliasVariant(fsa FlightScheduleAlias) (*FlightScheduleVariant, bool) {
 	for _, variant := range fs.Variants {
-		if variant.Data.Equal(fsd) {
-			return variant
+		if variant.Alias != nil && *variant.Alias == fsa {
+			return variant, true
 		}
 	}
 
-	return nil
+	return nil, false
 }

go/common/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1
 	github.com/go-jose/go-jose/v4 v4.0.4
 	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/time v0.6.0
 )
 
@@ -20,5 +21,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect
 	github.com/aws/smithy-go v1.20.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/crypto v0.26.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go/common/go.sum

@@ -36,5 +36,7 @@ golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
 golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
 golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=