Add back sample queries; s3 request payment is now json; unroll calls…

… to run in the launcher app

.gitignore

@@ -1,7 +1,6 @@
 .vscode/
 __pycache__/
 *.pyc
-/sample_queries
 /requirements.txt
 /pg_data
 /gf

introspector/aws/s3.py

@@ -27,7 +27,7 @@ def _import_bucket(proxy: ServiceProxy, bucket_metadata) -> Dict:
         result[key] = attr_result[key]
       else:
         result[key] = attr_result
-      if key in ('Policy', ):
+      if key in ('Policy', 'RequestPayment'):
         result[key] = json.loads(result[key])
     elif op_name == 'get_bucket_location':
       result['Location'] = {'LocationConstraint': 'us-east-1'}

introspector/queries/0023-aws_s3_bucket.sql

@@ -19,7 +19,7 @@ SELECT
   policy.attr_value::jsonb AS policy,
   policystatus.attr_value::jsonb AS policystatus,
   replication.attr_value::jsonb AS replication,
-  requestpayment.attr_value #>> '{}' AS requestpayment,
+  requestpayment.attr_value::jsonb AS requestpayment,
   tagging.attr_value::jsonb AS tagging,
   versioning.attr_value::jsonb AS versioning,
   website.attr_value::jsonb AS website,

launcher/main.go

@@ -4,13 +4,17 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws/external"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/pkg/errors"
 )
 
 func requireIntrospectorComposition(ctx context.Context, cli *client.Client) types.Container {
@@ -65,8 +69,60 @@ func needsGcpCredential(userCmd []string) bool {
 	return false
 }
 
+func runFileCommand(filename string, rest []string) ([]string, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	bytes, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	cmd := []string{"run", string(bytes)}
+	cmd = append(cmd, rest...)
+	return cmd, nil
+}
+
+func unrollRunCommands(cmd []string) ([][]string, error) {
+	if len(cmd) < 2 || cmd[0] != "run" {
+		return [][]string{cmd}, nil
+	}
+	queryTarget := cmd[1]
+	info, err := os.Stat(queryTarget)
+	if os.IsNotExist(err) {
+		// Let introspector handle whatever
+		return [][]string{cmd}, nil
+	} else if err != nil {
+		return nil, errors.Wrapf(err, "Failed to stat %v", queryTarget)
+	}
+	if info.IsDir() {
+		infos, err := ioutil.ReadDir(queryTarget)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Failed to ReadDir(%v)", queryTarget)
+		}
+		cmds := [][]string{}
+		for _, info := range infos {
+			if strings.HasSuffix(info.Name(), ".sql") {
+				filename := filepath.Join(queryTarget, info.Name())
+				subCommand, err := runFileCommand(filename, cmd[2:])
+				if err != nil {
+					return nil, err
+				}
+				cmds = append(cmds, subCommand)
+			}
+		}
+		return cmds, nil
+	}
+	runCommand, err := runFileCommand(queryTarget, cmd[2:])
+	if err != nil {
+		return nil, err
+	}
+	return [][]string{runCommand}, nil
+}
+
 func cmdPassthrough(ctx context.Context, cli *client.Client, introspector types.Container, userCmd []string) error {
 	var env map[string]string
+
 	cmd := append([]string{"python", "introspector.py"}, userCmd...)
 	if needsAwsCredential((userCmd)) {
 		awsEnv, err := loadAwsCredentials(ctx)
@@ -142,8 +198,14 @@ func main() {
 		panic(err)
 	}
 	introspector := requireIntrospectorComposition(ctx, cli)
-	err = cmdPassthrough(ctx, cli, introspector, cmd)
+	cmds, err := unrollRunCommands(cmd)
 	if err != nil {
 		panic(err)
 	}
+	for _, cmd := range cmds {
+		err = cmdPassthrough(ctx, cli, introspector, cmd)
+		if err != nil {
+			panic(err)
+		}
+	}
 }

migrations/provider/aws/0023-aws_s3_bucket.sql

@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS aws_s3_bucket (
   policy JSONB,
   policystatus JSONB,
   replication JSONB,
-  requestpayment TEXT,
+  requestpayment JSONB,
   tagging JSONB,
   versioning JSONB,
   website JSONB,

sample_queries/aws_ec2_instance_ips.sql

@@ -0,0 +1,6 @@
+SELECT
+  uri,
+  instanceid,
+  publicipaddress
+FROM
+  aws_ec2_instance

sample_queries/aws_function_versions_with_aliases.sql

@@ -0,0 +1,8 @@
+SELECT
+  DISTINCT(FV.version)
+FROM
+  aws_lambda_alias AS A
+  INNER JOIN aws_lambda_alias_functionversion AS AFV
+    ON A._id = AFV.alias_id
+  INNER JOIN aws_lambda_functionversion AS FV
+    ON AFV.functionversion_id = FV._id

sample_queries/aws_owner_pays_buckets.sql

@@ -0,0 +1,8 @@
+SELECT
+  name,
+  uri,
+  requestpayment->>'Payer' AS Payer
+FROM
+  aws_s3_bucket
+WHERE
+  requestpayment->>'Payer' = 'BucketOwner'

sample_queries/aws_policy_documents.sql

@@ -0,0 +1,8 @@
+SELECT
+  P.uri AS PolicyArn,
+  PV.document AS PolicyDocument
+FROM
+  aws_iam_policy AS P
+  INNER JOIN aws_iam_policyversion AS PV
+    ON PV._policy_id = P._id
+    AND PV.isdefaultversion = true

sample_queries/aws_rds_check_if_backups_are_disabled.sql

@@ -0,0 +1,8 @@
+SELECT
+    uri,
+    dbinstanceidentifier,
+    backupretentionperiod
+FROM
+    aws_rds_dbinstance
+WHERE
+    backupretentionperiod = 0

sample_queries/aws_storage_buckets.sql

@@ -0,0 +1,6 @@
+SELECT
+  name,
+  uri,
+  creationdate
+FROM
+  aws_s3_bucket

sample_queries/aws_total_disk_size.sql

@@ -0,0 +1,4 @@
+SELECT
+  SUM(size)
+FROM
+  aws_ec2_volume