From 3681da8853ab638804ffe3b401f7f9e12164c95e Mon Sep 17 00:00:00 2001 From: Reuben Cartwright Date: Thu, 4 Jul 2024 15:56:41 +0100 Subject: [PATCH] tools: Port, debug, and document createIoTThings.py The script contains tools used to create, destroy, and manage AWS IoT Things, Policies, Buckets, Roles, and Jobs. It is intended to automate the process of creating an OTA update. The script was previously in a deprecated project. Documentation for the script was also ported to `/docs/components/aws_iot/aws_tool.md``. Modified the aws_tool.md file with the following: 1. Fix paths from total-solutions. 2. Remove deprecated references (e.g. ats.sh) 3. Clarifications added (e.g. where to get AWS API keys). 4. Remove duplicate sections that have been ported already. This commit links to this documentation in the top-level `README.md`, under a section called 'Tools'. Modified createIoTThings.py with the following: 1. comment all functions with Python Docstrings. 2. debug and refactor createIoTThings.py as detailed below. 3. Fix filenames and directories from deprecated total-solutions. Total-solutions saved all credentials to the same .h file, but build.sh in this project takes credentials from separate files. This commit stores generated credentials for new objects in the `credentials` directory by default, with unique file names per Thing. The user can optionally specify the credentials directory. This commit makes it possible to do any operation except for an OTA image update if update-signature.txt is not in the correct directory. A warning is generated if update-signature.txt is not present. This commit improves --help messages for the script. This commit places additional user input checks e.g. permissions_boundary must be of a certain format. Previous error messages were confusing. fix: does not forget to pass role name during role creation. fix: _does_role_exist handles None and "" cases. fix: cleanup after command failure. Do not delete a role if it already existed before the script ran. fix: create-update-only now gets role ARNs correctly. fix: create-policy-only now passes thing name and policy name correctly. fix: create-policy-only does not allow empty policy name or thing name. fix: correct help messages e.g. for create-update-only. Some other minor fixes also exist. This script updates `/applications//configs/aws_config/ aws_clientcredentials.h` once a Thing is created, where `` is specified on the CLI. If the file has been previously modified, the script asks before overwriting. This saves the user modifying the file. Signed-off-by: Reuben Cartwright --- .github/.cSpellWords.txt | 1 + README.md | 6 + docs/components/aws_iot/aws_tool.md | 128 ++ release_changes/202407231054.change | 2 + tools/scripts/createIoTThings.py | 2595 +++++++++++++++++++++++++++ 5 files changed, 2732 insertions(+) create mode 100644 docs/components/aws_iot/aws_tool.md create mode 100644 release_changes/202407231054.change create mode 100644 tools/scripts/createIoTThings.py diff --git a/.github/.cSpellWords.txt b/.github/.cSpellWords.txt index 9ee852e..af0d4cf 100644 --- a/.github/.cSpellWords.txt +++ b/.github/.cSpellWords.txt @@ -13,6 +13,7 @@ ASYM BBOOL blinky Blinky +boto buildx BWCAP CALIB diff --git a/README.md b/README.md index 75384ba..a015db4 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,12 @@ image are protected by the PSA secure services. Implementation of [Arm® Mali™-C55 bare-metal driver](https://gitlab.arm.com/iot/m-class/drivers/isp_mali-c55), that demonstrates the usage of the Mali-C55 ISP with the Corstone M85 processor. +## Tools + +### Python script for automating AWS interactions + +A python script that automates creation, deletion, and listing of AWS Things, Policies, Jobs, Roles, and Buckets is documented [here](docs/components/aws_iot/aws_tool.md). + ## Contributing See [CONTRIBUTING](CONTRIBUTING.md) for more information. diff --git a/docs/components/aws_iot/aws_tool.md b/docs/components/aws_iot/aws_tool.md new file mode 100644 index 0000000..39687c2 --- /dev/null +++ b/docs/components/aws_iot/aws_tool.md @@ -0,0 +1,128 @@ +# Creating an AWS IOT setup with automated scripts + +An automation script is available to make setting up an OTA update easier. It is located at `tools/scripts/createIoTThings.py`. +The below will document how to set up AWS IOT for OTA updates using this script. + +### AWS IOT basic concepts + +In order to communicate through MQTT and/or perform an OTA update, you need to understand a few AWS concepts: + +[Things](https://docs.aws.amazon.com/iot/latest/developerguide/iot-thing-management.html) are endpoints that typically represent IOT devices on the AWS side. You'll bind them to your actual device by using a certificate and RSA key. + +[Policies](https://docs.aws.amazon.com/iot/latest/developerguide/thing-registry.html#attach-thing-principal) are sets of permissions you can attach to AWS entities. Typically to things and roles/users. + +Those two concepts are enough to set up and use MQTT, but we need a few more if we want to perform an OTA update: + +[S3 Buckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Bucket.html) are some cloud storage. In order to run an OTA campaign, we need to store the binary somewhere, and it'll be in a bucket. + +[Roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) are a set of permissions related to a task. As a user might not have the permission directly to perform an update, but they may be allowed to assume a role that does have that permission, thus temporarily gaining access to those permissions. To perform an OTA update you'll need to specify a role that will perform the update. + + > Note: Corporate account will likely have limitations on what roles can do. This is represented by a "permission boundary". Failure to set up this permission boundary when you create the role will prevent you from assuming that role, therefore prevent you from running an OTA update campaign. There may be additional restrictions than only a permission boundary. For instance, the role name may need to have a specific prefix. Please contact your dev ops to learn the limitation you may have regarding creating and assuming roles. + +#### Prerequisites + +To manually do the tasks `tools/scripts/createIoTThings.py` automates, also refer to [Setting up AWS connectivity](../../applications/aws_iot/setting_up_aws_connectivity.md) and [AWS IoT cloud connection](../../applications/aws_iot/aws_iot_cloud_connection.md). + +Install python dependencies. first setting up a virtual environment: +```sh +python -m venv .venv +. .venv/bin/activate +``` +Then, install the dependencies with: +```sh +python -m pip install boto3 click +``` +Configure AWS credentials environment variables on command line, for example: +```sh +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_SESSION_TOKEN= +export AWS_REGION=eu-west-1 +``` +The above keys and tokens can be found on the AWS online portal. They are also called AWS API keys. They need to be reset occasionally as they expire. See [Troubleshooting](#troubleshooting) for more information. + +In order to do an OTA update, you should have built an application that uses AWS, e.g. Keyword Detection. This is because the script assumes that `build/update-signature.txt` file exists when doing an OTA update. +Alternatively, you can provide the directory holding signatures and credentials as a command line argument (see `--build_dir`). Script-generated credentials will be written here. +It is still possible to create Things, Policies, and so on without a previous build existing. + +#### Using the commands + +Running the script with no parameter will give you the list of commands available. Additionally, every command have an `--help` option to list the accepted options. +Overall, there's three types of commands : `create-*`, `list-*` and `delete-*`. +Some commands can work on several objects, like `create-bucket-role-update` that create a bucket, a role, and an ota update in one go; or like `delete-thing -p` that delete a thing and, if that thing is the only object attached to a certificate, will delete the certificate as well. +Most commands will prompt you to enter names (for creating or deleting the object with that name). You may also enter the names directly in the commandline with options like `--thing_name`. +See `--help` for each command. For example: +```sh +python tools/scripts/createIoTThings.py --help +python tools/scripts/createIoTThings.py create-thing-and-policy --help +``` +For examples of how to use each command, see [Creating AWS IoT Firmware update](#creating-aws-iot-firmware-update-job-using-the-automated-script). + +### Create an AWS IOT setup manually + +See the `docs/applications/aws_iot/setting_up_aws_connectivity.md` file for instructions on setting up IoT manually. + +## Creating AWS IoT firmware update job using the automated script + +Please refer to [AWS IOT basic concepts](#aws-iot-basic-concepts) and [Using the commands](#using-the-commands) for the general instructions and limitation about using the script. + +Performing an OTA update will require you to: + * Create a role with the required permissions + * Create a bucket to store the update's binaries. Upload the update's binary. + * Create an AWS thing. Tie the thing to your device by updating the credentials in the source code. + * Create a policy that allows to connect to MQTT and attach it to the thing. + * Finally, create and run the OTA update campaign + +Create a thing, an IOT policy, and attach the two together with: +```sh +python tools/scripts/createIoTThings.py create-thing-and-policy --thing_name --policy_name --target_application +``` +Where `` is one of `keyword_detection`, `object_detection`, `speech_recognition`. +You must specify the `--target_application` argument if creating a Thing. +The script will update your `aws_clientcredential.h` config file automatically. If you have already modified the entries `clientcredentialMQTT_BROKER_ENDPOINT` or `clientcredentialIOT_THING_NAME` in the file, the script will warn you and ask before overwriting. + > Note: You may also create each things and policies individually, but you'll have to make sure to pass the certificate created by the first command to the second. Certificates will be printed upon creation during the first command. Use `--use_existing_certificate_arn ` on the second command. + +This creates and updates the `certificates` directory with the certificate of the newly created thing. +It is the user's responsibility to clean up the `certificates` directory. + +You may now use MQTT to send and receive message for that device. See section [Observing MQTT connectivity](../../applications/aws_iot/aws_iot_cloud_connection.md#observing-mqtt-connectivity) + +You may now rebuild keyword with those certificates: +```sh +./tools/scripts/build.sh keyword-detection --certificate_path certificates/thing_certificate_.pem.crt --private_key_path certificates/thing_private_key_.pem.key --target --inference --audio --toolchain +``` +Next, we'll create the bucket, upload the binary there, create a role capable of running an OTA update, and create the update. All of those with the following command: +```sh +python tools/scripts/createIoTThings.py create-bucket-role-update --thing_name --bucket_name --iam_role_name - --update_name --ota_binary keyword-detection-update_signed.bin --permissions_boundary arn:aws:iam::: +``` +The above assumes you have `keyword-detection-update_signed.bin` saved in the directory specified by the `--ota_binary_build_dir` argument. This argument defaults to the project's root directory. +If you want to run an OTA update for another example application, use another signed binary name. +Buckets have a few rules about their name. You must not use capital case and the name must be unique. Failure to do so will trigger an `InvalidBucketName` or `AccessDenied` error. +The `` you use is often defined by your company's AWS policies. For example, Arm uses the prefix `Proj`. Using the wrong prefix can result in failure to create roles. + +And you can now run keyword and see the OTA update happening. +```sh +./tools/scripts/run.sh keyword-detection --target --audio +``` + +You can clean up everything created here with: +```sh +python tools/scripts/createIoTThings.py delete-ota-update --ota_update_name -f && +python tools/scripts/createIoTThings.py delete-iam-role --iam_role_name - -f +python tools/scripts/createIoTThings.py delete-bucket --bucket_name -f +python tools/scripts/createIoTThings.py delete-policy -p --policy_name +python tools/scripts/createIoTThings.py delete-thing -p --thing_name +``` +- If the `-f` flag is set when deleting an OTA Update, the Update is deleted even if it has not yet finished. +- If the `-f` flag is set when deleting a Role, the Role is deleted even if it is attached to other AWS entities. +- If the `-f` flag is set when deleting a Bucket, the Bucket is deleted even if it is not empty. +- If the `-p` flag is set when deleting a Policy, any certificates attached to the Policy are pruned. Pruning means that if a certificate is attached to the policy and nothing else, this certificate is deleted. +- If the `-p` flag is set when deleting a Thing, certificates attached to the Thing are pruned. + + > Note: Your role is what allow you to interact with the OTA update. It is important you don't delete it before successfully deleting the OTA update or you will lose the permission to delete the update. If such a thing happen, you'll have to recreate the role and manually delete the update. + +## Troubleshooting + +##### 1. My AWS credentials are rejected, despite being accepted earlier. + +AWS credentials such as `AWS_SESSION_TOKEN` occasionally expire. It is the responsibility of the user to ensure their tokens are up to date (by re-`export`ing them). diff --git a/release_changes/202407231054.change b/release_changes/202407231054.change new file mode 100644 index 0000000..0ddcbcc --- /dev/null +++ b/release_changes/202407231054.change @@ -0,0 +1,2 @@ +docs: document createIoTThings.py usage in aws_tools.md. +tools: Add createIoTThings.py script for setting up OTA updates. diff --git a/tools/scripts/createIoTThings.py b/tools/scripts/createIoTThings.py new file mode 100644 index 0000000..72b937f --- /dev/null +++ b/tools/scripts/createIoTThings.py @@ -0,0 +1,2595 @@ +#!/usr/bin/python3 + +# Copyright (c) 2023-2024 Arm Limited. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage can be found at : /docs/components/aws_iot/aws_tool.md + +from enum import Enum +import os +import time +import traceback as tb +import re +from functools import reduce + +# JSON is used for policy creation +import json + +# used to communicate with AWS services. +import boto3 +import botocore + +# used for the CLI, and argument parsing. +import click +import logging + +DEFAULT_LOG_LEVEL = "warning" +# Default path of OTA binary directory is build/ +DEFAULT_BUILD_DIR = "build" +DEFAULT_CREDENTIALS_PATH = "certificates" +DEFAULT_OTA_BINARY = "keyword-detection-update_signed.bin" +# Signature used to sign new firmware (update) image. +# Can be found in the build directory. +# If a file not found error occurs due to this file, +# try to build one of the FRI applications (e.g. keyword-detection) +DEFAULT_OTA_UPDATE_SIGNATURE_FILENAME = "update-signature.txt" + +CREATE_NEW_CERTIFICATE = "" + +OTA_JOB_NAME_PREFIX = "AFR_OTA-" + +for var in [ + "AWS_REGION", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", +]: + if not os.getenv(var): + raise ValueError(var + " is not set in environment") + +AWS_REGION = os.getenv("AWS_REGION") +iot = boto3.client("iot", AWS_REGION) +s3 = boto3.client("s3", AWS_REGION) +iam = boto3.client("iam", AWS_REGION) +sts_client = boto3.client("sts", AWS_REGION) + +validCredentials = False +try: + sts_client.get_caller_identity() + validCredentials = True +except botocore.exceptions.ClientError: + pass +if not validCredentials: + raise ValueError( + "Your AWS client API keys (which include 'AWS_SESSION_TOKEN') " + "are invalid or expired. Try re-exporting them." + ) + + +@click.group() +def cli(): + pass + + +def read_whole_file(path, mode="r"): + with open(path, mode) as fp: + return fp.read() + + +class ApplicationType(Enum): + # These must be the subdirectory for each application + # in the `applications` folder. + KEYWORD_DETECTION = "keyword_detection" + OBJECT_DETECTION = "object_detection" + SPEECH_RECOGNITION = "speech_recognition" + UNDEFINED = "DEFAULT" + + def app_type_from_string(s): + for app in AWS_APPLICATIONS: + if s == app.value: + return app + return ApplicationType.UNDEFINED + + +AWS_APPLICATIONS = [ + ApplicationType.KEYWORD_DETECTION, + ApplicationType.OBJECT_DETECTION, + ApplicationType.SPEECH_RECOGNITION, +] + + +class Flags: + def __init__( + self, + bucket_name="", + role_name="", + build_dir=DEFAULT_BUILD_DIR, + ota_binary=DEFAULT_OTA_BINARY, + ota_update_signature_filename=DEFAULT_OTA_UPDATE_SIGNATURE_FILENAME, + target_application=ApplicationType.UNDEFINED, + ): + """ + Create an object holding metadata needed for OTA with AWS. + + Parameters: + build_dir: directory to where the OtA update binary is. + E.g. "build/" contains "keyword-detection-update_signed.bin" + """ + self.BUILD_DIR = build_dir + self.AWS_ACCOUNT_ID = boto3.client("sts").get_caller_identity().get("Account") + self.AWS_ACCOUNT_ARN = boto3.client("sts").get_caller_identity().get("Arn") + self.THING_NAME = None + self.THING_ARN = None + self.POLICY_NAME = None + self.OTA_BUCKET_NAME = bucket_name + self.OTA_ROLE_NAME = role_name + self.OTA_BINARY = ota_binary + self.OTA_BINARY_FILE_PATH = os.path.join(self.BUILD_DIR, self.OTA_BINARY) + self.UPDATE_ID = None + self.OTA_UPDATE_PROTOCOLS = ["MQTT"] + self.OTA_UPDATE_TARGET_SELECTION = "SNAPSHOT" + self.bucket = None + self.file = None + self.thing = None + self.policy = None + self.keyAndCertificate = None + self.certificate = None + self.privateKey = None + self.publicKey = None + self.endPointAddress = None + self.role = ( + None + if not _does_role_exist(role_name) + else iam.get_role(RoleName=role_name)["Role"] + ) + self.OTA_ROLE_ARN = None if self.role is None else self.role["Arn"] + logging.info("Target application is: " + target_application.value) + # Used to update 'aws_clientcredential.h' + self.targetApplication = target_application + # Used for error handling cleanup. + self.bucketHasBeenCreated = False + self.certificateHasBeenCreated = False + self.roleHasBeenCreated = False + self.updateHasBeenCreated = False + self.policyHasBeenCreated = False + self.thingHasBeenCreated = False + # JSONS + self.POLICY_DOCUMENT = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iot:Publish", + "iot:Receive", + "iot:Subscribe", + "iot:Connect", + ], + "Resource": [ + "arn:aws:iot:" + AWS_REGION + ":" + self.AWS_ACCOUNT_ID + ":*" + ], + } + ], + } + self.ASSUME_ROLE_POLICY_DOCUMENT = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": self.AWS_ACCOUNT_ARN, + "Service": [ + "iot.amazonaws.com", + "s3.amazonaws.com", + "iam.amazonaws.com", + ], + }, + "Action": ["sts:AssumeRole"], + } + ], + } + self.IAM_OTA_PERMISSION = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["iam:GetRole", "iam:PassRole"], + "Resource": "arn:aws:iam::" + + self.AWS_ACCOUNT_ID + + ":role/" + + self.OTA_ROLE_NAME, + } + ], + } + self.S3_OTA_PERMISSION = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObjectVersion", "s3:GetObject", "s3:PutObject"], + "Resource": ["arn:aws:s3:::" + self.OTA_BUCKET_NAME + "/*"], + } + ], + } + signaturePath = os.path.join( + self.BUILD_DIR, + ota_update_signature_filename, + ) + signature = "unavailable" + try: + signature = read_whole_file(signaturePath) + except FileNotFoundError: + pass # this check is done at the CLI stage. + self.OTA_UPDATE_FILES = [ + { + "fileName": "non_secure image", + "fileLocation": { + "s3Location": { + "bucket": self.OTA_BUCKET_NAME, + "key": self.OTA_BINARY, + } + }, + "codeSigning": { + "customCodeSigning": { + "signature": { + "inlineDocument": bytearray( + signature.strip(), + "utf-8", + ) + }, + "certificateChain": {"certificateName": "0"}, + "hashAlgorithm": "SHA256", + "signatureAlgorithm": "RSA", + }, + }, + } + ] + + +def set_log_level(loglevel): + logging.basicConfig(level=loglevel.upper()) + + +class StdCommand(click.core.Command): + def __init__(self, *args, **kwargs): + """ + Defines parsing of command line arguments, e.g. routes + 'delete-thing' or 'delete-policy' arguments to their + respective functions. + See click API for more info. + + Parameters: + *args: usually an empty tuple. + **kwargs (dict): E.g. + {'name': 'delete-thing', + 'callback': , + 'params': [