Add support for S3 index storage backend

Signed-off-by: Brenno Oliveira <[email protected]>

README.md

@@ -217,6 +217,7 @@ Indices for *falcoctl* can be retrieved from various storage backends. The suppo
 | https | https://   | Convenience alias for the HTTP backend.                                                       |
 | gcs   | gs://      | For indices stored as Google Cloud Storage objects. Supports application default credentials. |
 | file  | file://    | For indices stored on the local file system.                                                  |
+| s3    | s3://      | For indices stored as AWS S3 objects. Supports default credentials, IRSA.                     |
 
 
 #### falcoctl index add

go.mod

@@ -4,6 +4,7 @@ go 1.22.5
 
 require (
 	cloud.google.com/go/storage v1.42.0
+	github.com/aws/aws-sdk-go v1.53.10
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cilium/ebpf v0.15.0

pkg/index/fetch/fetch.go

@@ -25,6 +25,7 @@ import (
 	"github.com/falcosecurity/falcoctl/pkg/index/fetch/file"
 	"github.com/falcosecurity/falcoctl/pkg/index/fetch/gcs"
 	"github.com/falcosecurity/falcoctl/pkg/index/fetch/http"
+	"github.com/falcosecurity/falcoctl/pkg/index/fetch/s3"
 	"github.com/falcosecurity/falcoctl/pkg/index/index"
 )
 
@@ -48,12 +49,14 @@ func NewFetcher() *Fetcher {
 			"https": http.Fetch,
 			"gcs":   gcs.Fetch,
 			"file":  file.Fetch,
+			"s3":    s3.Fetch,
 		},
 		schemeDefaultBackends: map[string]string{
 			"http":  "http",
 			"https": "https",
 			"gs":    "gcs",
 			"file":  "file",
+			"s3":    "s3",
 		},
 	}
 }

pkg/index/fetch/s3/doc.go

@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2024 The Falco Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package s3 implements all the logic for fetching indexes from AWS S3.
+package s3

pkg/index/fetch/s3/fetcher.go

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2024 The Falco Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	"github.com/falcosecurity/falcoctl/pkg/index/config"
+)
+
+// Fetch fetches the raw index file from an S3 object.
+func Fetch(ctx context.Context, conf *config.Entry) ([]byte, error) {
+	o, err := s3ObjectFromURI(conf.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a new AWS session
+	sess, err := session.NewSession()
+	if err != nil {
+		return nil, fmt.Errorf("unable to create AWS session: %w", err)
+	}
+
+	svc := s3.New(sess)
+
+	// Get the object from S3
+	res, err := svc.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(o.Bucket),
+		Key:    aws.String(o.Key),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to get S3 object: %w", err)
+	}
+	defer res.Body.Close()
+
+	// Read the object data
+	bytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading S3 object: %w", err)
+	}
+
+	return bytes, nil
+}

pkg/index/fetch/s3/types.go

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2024 The Falco Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+const s3Scheme = "s3"
+
+// s3Object represents an S3 object with its bucket and key.
+type s3Object struct {
+	Bucket string
+	Key    string
+}
+
+// s3ObjectFromURI parses S3 URIs (s3://<bucket>/<object>) and returns a s3Object.
+func s3ObjectFromURI(uri string) (*s3Object, error) {
+	parsedURI, err := url.Parse(uri)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse URI: %w", err)
+	}
+
+	if !strings.EqualFold(parsedURI.Scheme, s3Scheme) {
+		return nil, fmt.Errorf("invalid S3 URI: scheme should be '%s' but got '%s'", s3Scheme, parsedURI.Scheme)
+	}
+
+	if parsedURI.Host == "" {
+		return nil, fmt.Errorf("invalid S3 URI: missing bucket name")
+	}
+
+	if parsedURI.Path == "" {
+		return nil, fmt.Errorf("invalid S3 URI: missing object name")
+	}
+
+	return &s3Object{
+		Bucket: parsedURI.Host,
+		Key:    parsedURI.Path[1:], // Remove the leading slash
+	}, nil
+}