add cron action loading data from ourairports.com

cdk/lib/constructs/cron-lambda-construct.ts

@@ -45,6 +45,7 @@ export class CronLambdaConstruct extends Construct {
 
     props.dataBucket.grantRead(this.lambda, 'raw/LH_Public_Data/flightschedules/*');
     props.dataBucket.grantWrite(this.lambda, 'raw/LH_Public_Data/*');
+    props.dataBucket.grantWrite(this.lambda, 'raw/ourairports_data/*');
     props.dataBucket.grantWrite(this.lambda, 'processed/flights/*');
   }
 }

go/cron/action/load_flight_schedules.go

@@ -48,7 +48,6 @@ func (a *lfsAction) Handle(ctx context.Context, params LoadFlightSchedulesParams
 
 func (a *lfsAction) loadSingle(ctx context.Context, bucket, prefix string, d common.LocalDate) error {
 	var b bytes.Buffer
-	b.Bytes()
 	err := a.lhc.FlightSchedulesRaw(
 		ctx,
 		[]common.AirlineIdentifier{common.Lufthansa, common.AirDolomiti, common.Swiss, common.Austrian, common.Edelweiss, common.Brussels, common.EurowingsDiscover},

go/cron/action/load_ourairports.go

@@ -0,0 +1,71 @@
+package action
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"golang.org/x/sync/errgroup"
+	"io"
+	"net/http"
+)
+
+type LoadOurAirportsDataParams struct {
+	OutputBucket string   `json:"outputBucket"`
+	OutputPrefix string   `json:"outputPrefix"`
+	Files        []string `json:"files"`
+}
+
+type LoadOurAirportsDataOutput struct {
+}
+
+type loa struct {
+	s3c        *s3.Client
+	httpClient *http.Client
+}
+
+func NewLoadOurAirportsDataAction(s3c *s3.Client, httpClient *http.Client) Action[LoadOurAirportsDataParams, LoadOurAirportsDataOutput] {
+	return &loa{
+		s3c:        s3c,
+		httpClient: cmp.Or(httpClient, http.DefaultClient),
+	}
+}
+
+func (a *loa) Handle(ctx context.Context, params LoadOurAirportsDataParams) (LoadOurAirportsDataOutput, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	g, ctx := errgroup.WithContext(ctx)
+	for _, file := range params.Files {
+		g.Go(func() error {
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://davidmegginson.github.io/ourairports-data/"+file, nil)
+			if err != nil {
+				return err
+			}
+
+			resp, err := a.httpClient.Do(req)
+			if err != nil {
+				return err
+			}
+
+			defer resp.Body.Close()
+
+			var b bytes.Buffer
+			if _, err = io.Copy(&b, resp.Body); err != nil {
+				return err
+			}
+
+			_, err = a.s3c.PutObject(ctx, &s3.PutObjectInput{
+				Bucket:      aws.String(params.OutputBucket),
+				Key:         aws.String(params.OutputPrefix + file),
+				ContentType: aws.String("text/csv"),
+				Body:        bytes.NewReader(b.Bytes()),
+			})
+
+			return err
+		})
+	}
+
+	return LoadOurAirportsDataOutput{}, g.Wait()
+}

go/cron/main.go

@@ -63,6 +63,7 @@ func newHandler(s3c *s3.Client) func(ctx context.Context, event InputEvent) (jso
 	lfsAction := action.NewLoadFlightSchedulesAction(s3c, lhc)
 	cfsAction := action.NewConvertFlightSchedulesAction(s3c)
 	cronAction := action.NewCronAction(lfsAction, cfsAction)
+	loaAction := action.NewLoadOurAirportsDataAction(s3c, nil)
 
 	return func(ctx context.Context, event InputEvent) (json.RawMessage, error) {
 		switch event.Action {
@@ -89,6 +90,9 @@ func newHandler(s3c *s3.Client) func(ctx context.Context, event InputEvent) (jso
 
 		case "cron":
 			return handle(ctx, cronAction, event.Params)
+
+		case "load_our_airports_data":
+			return handle(ctx, loaAction, event.Params)
 		}
 
 		return nil, fmt.Errorf("unsupported action: %v", event.Action)