save flightnumbers metadata with last modified date

go/api/data/handler.go

@@ -18,6 +18,7 @@ import (
 	"io"
 	"iter"
 	"log/slog"
+	"maps"
 	"net/http"
 	"slices"
 	"strconv"
@@ -388,31 +389,39 @@ func (h *Handler) Flight(ctx context.Context, fn common.FlightNumber, departureD
 
 func (h *Handler) FlightNumbers(ctx context.Context, prefix string, limit int) ([]common.FlightNumber, error) {
 	var fns []common.FlightNumber
-	if err := adapt.S3GetJson(ctx, h.s3c, h.bucket, "processed/metadata/flightNumbers.json", &fns); err != nil {
-		return nil, err
-	}
+	{
+		fnsRaw, err := h.FlightNumbersRaw(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		if prefix != "" {
+			maps.DeleteFunc(fnsRaw, func(fn common.FlightNumber, _ time.Time) bool {
+				return !strings.HasPrefix(fn.String(), prefix)
+			})
+		}
 
-	if prefix != "" {
-		fns = slices.DeleteFunc(fns, func(fn common.FlightNumber) bool {
-			return !strings.HasPrefix(fn.String(), prefix)
+		fns = slices.SortedFunc(maps.Keys(fnsRaw), func(a, b common.FlightNumber) int {
+			return cmp.Or(
+				cmp.Compare(a.Airline, b.Airline),
+				cmp.Compare(a.Number, b.Number),
+				cmp.Compare(a.Suffix, b.Suffix),
+			)
 		})
 	}
 
-	slices.SortFunc(fns, func(a, b common.FlightNumber) int {
-		return cmp.Or(
-			cmp.Compare(a.Airline, b.Airline),
-			cmp.Compare(a.Number, b.Number),
-			cmp.Compare(a.Suffix, b.Suffix),
-		)
-	})
-
 	if limit < 1 {
 		limit = len(fns)
 	}
 
 	return fns[:min(limit, len(fns))], nil
 }
 
+func (h *Handler) FlightNumbersRaw(ctx context.Context) (map[common.FlightNumber]time.Time, error) {
+	var fns map[common.FlightNumber]time.Time
+	return fns, adapt.S3GetJson(ctx, h.s3c, h.bucket, "processed/metadata/flightNumbers.json", &fns)
+}
+
 func (h *Handler) Airlines(ctx context.Context, prefix string) ([]common.AirlineIdentifier, error) {
 	var airlines []common.AirlineIdentifier
 	if err := adapt.S3GetJson(ctx, h.s3c, h.bucket, "processed/metadata/airlines.json", &airlines); err != nil {

go/api/web/sitemap.go

@@ -17,17 +17,15 @@ func NewSitemapHandler(dh *data.Handler) echo.HandlerFunc {
 	const ttl = time.Hour * 3
 
 	return func(c echo.Context) error {
-		fns, err := dh.FlightNumbers(c.Request().Context(), "", -1)
+		fns, err := dh.FlightNumbersRaw(c.Request().Context())
 		if err != nil {
 			return echo.NewHTTPError(http.StatusInternalServerError)
 		}
 
 		baseURL := baseUrl(c)
-		now := time.Now()
-
 		res := c.Response()
 		res.Header().Set(echo.HeaderContentType, echo.MIMEApplicationXMLCharsetUTF8)
-		addExpirationHeaders(c, now, ttl)
+		addExpirationHeaders(c, time.Now(), ttl)
 
 		_, err = res.Write([]byte(xml.Header))
 		if err != nil {
@@ -48,22 +46,20 @@ func NewSitemapHandler(dh *data.Handler) echo.HandlerFunc {
 			return echo.NewHTTPError(http.StatusInternalServerError)
 		}
 
-		for _, fn := range fns {
+		for fn, lastMod := range fns {
 			loc := baseURL + "/flight/" + fn.String()
 
-			if err = addSitemapURL(enc, loc, now); err != nil {
+			if err = addSitemapURL(enc, loc, lastMod); err != nil {
 				return echo.NewHTTPError(http.StatusInternalServerError)
 			}
 		}
 
-		err = enc.EncodeToken(xml.EndElement{
+		return enc.EncodeToken(xml.EndElement{
 			Name: xml.Name{
 				Local: "urlset",
 				Space: "http://www.sitemaps.org/schemas/sitemap/0.9",
 			},
 		})
-
-		return nil
 	}
 }
 

go/common/jwks/jwks.go

@@ -49,8 +49,7 @@ func NewVerifier(jwksUri string, opts ...Option) *Verifier {
 }
 
 func (v *Verifier) Key(ctx context.Context, kid string) (jose.JSONWebKey, error) {
-	jwk, ok := v.cache.Load(kid)
-	if ok {
+	if jwk, ok := v.cache.Load(kid); ok {
 		return jwk, nil
 	}
 

go/cron/action/update_metadata.go

@@ -12,6 +12,7 @@ import (
 	"github.com/explore-flights/monorepo/go/common/xiter"
 	"golang.org/x/sync/errgroup"
 	"maps"
+	"time"
 )
 
 type UpdateMetadataParams struct {
@@ -36,7 +37,7 @@ type AddedAndRemoved struct {
 type metadata struct {
 	airports      common.Set[string]
 	airlines      common.Set[common.AirlineIdentifier]
-	flightNumbers common.Set[common.FlightNumber]
+	flightNumbers map[common.FlightNumber]time.Time
 	aircraft      common.Set[string]
 }
 
@@ -121,9 +122,11 @@ func (a *umdAction) worker(ctx context.Context, obj [2]string, md metadata) (met
 	md = a.ensureMetadata(md)
 	for fn, fs := range schedules {
 		md.airlines[fn.Airline] = struct{}{}
-		md.flightNumbers[fn] = struct{}{}
 
 		for _, variant := range fs.Variants {
+			md.flightNumbers[fn] = common.Max(md.flightNumbers[fn], variant.Metadata.RangesUpdateTime)
+			md.flightNumbers[fn] = common.Max(md.flightNumbers[fn], variant.Metadata.DataUpdateTime)
+
 			md.airports[variant.Data.DepartureAirport] = struct{}{}
 			md.airports[variant.Data.ArrivalAirport] = struct{}{}
 			md.aircraft[variant.Data.AircraftType] = struct{}{}
@@ -169,7 +172,7 @@ func (*umdAction) ensureMetadata(md metadata) metadata {
 	}
 
 	if md.flightNumbers == nil {
-		md.flightNumbers = make(common.Set[common.FlightNumber])
+		md.flightNumbers = make(map[common.FlightNumber]time.Time)
 	}
 
 	if md.aircraft == nil {
@@ -196,9 +199,31 @@ func (a *umdAction) updateMetadata(ctx context.Context, bucket, prefix string, m
 	})
 
 	g.Go(func() error {
-		var err error
-		output.FlightNumbers.Added, output.FlightNumbers.Removed, err = updateMetadata(ctx, a.s3c, bucket, prefix, "flightNumbers", md.flightNumbers)
-		return err
+		key := prefix + "flightNumbers.json"
+		{
+			var existing map[common.FlightNumber]time.Time
+			if err := adapt.S3GetJson(ctx, a.s3c, bucket, key, &existing); err != nil && !adapt.IsS3NotFound(err) {
+				return err
+			}
+
+			if existing != nil {
+				for fn := range md.flightNumbers {
+					_, ok := existing[fn]
+					delete(existing, fn)
+
+					if !ok {
+						output.FlightNumbers.Added += 1
+					}
+				}
+
+				output.FlightNumbers.Removed = len(existing)
+			} else {
+				output.FlightNumbers.Added = len(md.flightNumbers)
+				output.FlightNumbers.Removed = 0
+			}
+		}
+
+		return adapt.S3PutJson(ctx, a.s3c, bucket, key, md.flightNumbers)
 	})
 
 	g.Go(func() error {