-Introduce aws_auth to proto

[poodlewars]

(RBAC is not replaced yet)
-Add STS auth method to codebase
(But no test yet)

Reference Issues/PRs

What does this implement or fix?

Any other comments?

Checklist

Checklist for code changes...

• Have you updated the relevant docstrings, documentation and copyright notice?
• Is this contribution tested against all ArcticDB's features?
• Do all exceptions introduced raise appropriate error messages?
• Are API changes highlighted in the PR description?
• Is the PR labelled as enhancement or bug so it appears in autogenerated release notes?

[phoebusm]

Script to create user and role for testing:

import boto3
import json
import sys
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()

USER_NAME = ""
ROLE_NAME = ""
POLICY_NAME = ""
BUCKET_NAME = ""
PROFILE_NAME = ""
ACCOUNT_ID = boto3.client("sts").get_caller_identity().get("Account")

# Create IAM client
iam_client = boto3.client("iam")

created_resources = {
    "user": None,
    "role": None,
    "policy": None
}

access_key_id = None

def cleanup():
    logger.info("Starting cleanup process...")
    try:
        if created_resources["policy"]:
            iam_client.detach_role_policy(
            RoleName=ROLE_NAME,
            PolicyArn=created_resources["policy"]
        )
        iam_client.delete_policy(
            PolicyArn=created_resources["policy"]
        )
        logger.info(f"Policy {POLICY_NAME} deleted successfully.")
    except Exception as e:
        logger.error(f"Error deleting policy: {e}")

    try:
        if created_resources["role"]:
            iam_client.delete_role(
                RoleName=ROLE_NAME
            )
            logger.info(f"Role {ROLE_NAME} deleted successfully.")
    except Exception as e:
        logger.error(f"Error deleting role: {e}")

    
    try:
        if "access_key" in created_resources:
            iam_client.delete_access_key(
                UserName=USER_NAME,
                AccessKeyId=created_resources["access_key"]
            )
            logger.info(f"Access key id {created_resources['access_key']} deleted successfully.")
    except Exception as e:
        logger.error(f"Error deleting access key id: {e}")

    try:
        if created_resources["user"]:
            # Detach user policies
            policies = iam_client.list_user_policies(UserName=USER_NAME)["PolicyNames"]
            for policy_name in policies:
                iam_client.delete_user_policy(UserName=USER_NAME, PolicyName=policy_name)
                logger.info(f"Detached and deleted inline policy {policy_name} from user {USER_NAME}.")

            # Delete the user
            iam_client.delete_user(
                UserName=USER_NAME
            )
            logger.info(f"User {USER_NAME} deleted successfully.")
    except Exception as e:
        logger.error(f"Error deleting user: {e}")

def main():
    try:
        # Create IAM user
        try:
            iam_client.create_user(UserName=USER_NAME)
            created_resources["user"] = USER_NAME
            logger.info(f"User {USER_NAME} created successfully.")
        except iam_client.exceptions.EntityAlreadyExistsException:
            logger.warning(f"User {USER_NAME} already exists.")
        except Exception as e:
            logger.error(f"Error creating user: {e}")
            cleanup()
            sys.exit(1)

        # Create IAM role
        assume_role_policy_document = {
            "Version": "2012-10-17",
            "Statement": [{
                "Effect": "Allow",
                "Principal": {
                    "Service": "ec2.amazonaws.com",
                    "AWS": ACCOUNT_ID
                },
                "Action": "sts:AssumeRole"
            }
            ]
        }

        try:
            role_response = iam_client.create_role(
                RoleName=ROLE_NAME,
                AssumeRolePolicyDocument=json.dumps(assume_role_policy_document)
            )
            role_arn = role_response["Role"]["Arn"]
            created_resources["role"] = ROLE_NAME
            logger.info(f"Role {ROLE_NAME} created successfully.")
        except iam_client.exceptions.EntityAlreadyExistsException:
            role_arn = f"arn:aws:iam::{ACCOUNT_ID}:role/{ROLE_NAME}"
            logger.warning(f"Role {ROLE_NAME} already exists.")
        except Exception as e:
            logger.error(f"Error creating role: {e}")
            cleanup()
            sys.exit(1)

        # Create a policy for S3 bucket access
        s3_access_policy_document = {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Effect": "Allow",
                    "Action": ["s3:ListBucket"],
                    "Resource": [f"arn:aws:s3:::{BUCKET_NAME}"]
                },
                {
                    "Effect": "Allow",
                    "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListObject"],
                    "Resource": [f"arn:aws:s3:::{BUCKET_NAME}/*"]
                }
            ]
        }

        try:
            policy_response = iam_client.create_policy(
                PolicyName=POLICY_NAME,
                PolicyDocument=json.dumps(s3_access_policy_document)
            )
            policy_arn = policy_response["Policy"]["Arn"]
            created_resources["policy"] = policy_arn
            logger.info(f"Policy {POLICY_NAME} created successfully.")
        except iam_client.exceptions.EntityAlreadyExistsException:
            policy_arn = f"arn:aws:iam::{ACCOUNT_ID}:policy/{POLICY_NAME}"
            created_resources["policy"] = policy_arn
            logger.warning(f"Policy {POLICY_NAME} already exists.")
        except Exception as e:
            logger.error(f"Error creating policy: {e}")
            cleanup()
            sys.exit(1)

        # Attach the policy to the role
        try:
            iam_client.attach_role_policy(
                RoleName=ROLE_NAME,
                PolicyArn=policy_arn
            )
            logger.info(f"Policy {POLICY_NAME} attached to role {ROLE_NAME} successfully.")
        except Exception as e:
            logger.error(f"Error attaching policy to role: {e}")
            cleanup()
            sys.exit(1)

        # Create an inline policy for the user to assume the role
        assume_role_user_policy_document = {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Effect": "Allow",
                    "Action": "sts:AssumeRole",
                    "Resource": f"arn:aws:iam::{ACCOUNT_ID}:role/{ROLE_NAME}",
                }
            ]
        }

        try:
            iam_client.put_user_policy(
                UserName=USER_NAME,
                PolicyName="AssumeRolePolicy",
                PolicyDocument=json.dumps(assume_role_user_policy_document)
            )
            logger.info(f"Inline policy to assume role {ROLE_NAME} attached to user {USER_NAME} successfully.")
        except Exception as e:
            logger.error(f"Error attaching inline policy to user: {e}")
            cleanup()
            sys.exit(1)

        logger.info(f"User {USER_NAME} created with role {ROLE_NAME} to access bucket {BUCKET_NAME}.")

        try:
            access_key_response = iam_client.create_access_key(UserName=USER_NAME)
            access_key_id = access_key_response["AccessKey"]["AccessKeyId"]
            secret_access_key = access_key_response["AccessKey"]["SecretAccessKey"]
            created_resources["access_key"] = access_key_id
            logger.info("Access key created successfully.")
        except Exception as e:
            logger.error(f"Error creating access key: {e}")
            cleanup()
            sys.exit(1)
            
        aws_credentials = f"""
[profile {PROFILE_NAME}]
role_arn = {role_arn}
source_profile = {PROFILE_NAME}_base

[profile {PROFILE_NAME}_base]
aws_access_key_id = {access_key_id}
aws_secret_access_key = {secret_access_key}
        """
        print(aws_credentials)
    finally:
        if "access_key" in created_resources:
            input("Please enter to continue...")
        cleanup()

if __name__ == "__main__":
    main()

While running the script, it will print out a tempoary set of profile as below:

[profile pro]
role_arn = arn:aws:iam::*
source_profile = *

[profile base]
aws_access_key_id = *
aws_secret_access_key = *

Copy the content to .aws/config. Then the bucket can be accessed by ArcticDB:

>>> import arcticdb as adb
>>> arctic = adb.Arctic('s3://s3.eu-north-1.amazonaws.com:BUCKET?aws_auth=2&aws_profile=pro')
>>> arctic.list_libraries()
['test']

The script will automatically clear the username and roles after pressing enter