Skip to content

Commit

Permalink
Backport AWS VPC/Subnet enhancement from main branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@IsaacYangSLA
IsaacYangSLA committed May 8, 2024
1 parent 90cda8c commit def9adf
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 10 deletions.
18 changes: 17 additions & 1 deletion nvflare/dashboard/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,11 @@ def cloud(args):
"t",
exe=True,
)
print(f"Dashboard launch script for cloud is written at {dest}. Now running the script.")
print(f"Dashboard launch script for cloud is written at {dest}. Now running it.")
if args.vpc_id and args.subnet_id:
option = [f"--vpc-id={args.vpc_id}", f"--subnet-id={args.subnet_id}"]
print(f"Option of the script: {option}")
dest = [dest] + option
_ = subprocess.run(dest)
os.remove(dest)

Expand Down Expand Up @@ -191,6 +195,18 @@ def define_dashboard_parser(parser):
parser.add_argument("--cred", help="set credential directly in the form of USER_EMAIL:PASSWORD")
parser.add_argument("-i", "--image", help="set the container image name")
parser.add_argument("--local", action="store_true", help="start dashboard locally without docker image")
parser.add_argument(
"--vpc-id",
type=str,
default="",
help="VPC id for AWS EC2 instance. Applicable to AWS only. Ignored if subnet-id is not specified.",
)
parser.add_argument(
"--subnet-id",
type=str,
default="",
help="Subnet id for AWS EC2 instance. Applicable to AWS only. Ignored if vpc-id is not specified.",
)


def handle_dashboard(args):
Expand Down
132 changes: 123 additions & 9 deletions nvflare/lighter/impl/master_template.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -950,6 +950,14 @@ cloud_script_header: |
image_name=$2
shift
;;
--vpc-id)
vpc_id=$2
shift
;;
--subnet-id)
subnet_id=$2
shift
;;
esac
shift
done
Expand Down Expand Up @@ -1669,6 +1677,13 @@ aws_start_svr_sh: |
check_binary dig "Please install it first."
check_binary jq "Please install it first."
if [ -z ${vpc_id+x} ]
then
using_default_vpc=true
else
using_default_vpc=false
fi
if [ -z ${image_name+x} ]
then
container=false
Expand Down Expand Up @@ -1718,6 +1733,25 @@ aws_start_svr_sh: |
prompt ans "Press ENTER when it's done or no additional dependencies. "
fi
# Check if default VPC exists
if [ $using_default_vpc == true ]
then
echo "Checking if default VPC exists"
found_default_vpc=$(aws ec2 describe-vpcs | jq '.Vpcs[] | select(.IsDefault == true)')
if [ -z "${found_default_vpc}" ]
then
echo "No default VPC found. Please create one before running this script with the following command."
echo "aws ec2 create-default-vpc"
echo "or specify your own vpc and subnet with --vpc-id and --subnet-id"
exit
else
echo "Default VPC found"
fi
else
echo "Please check the vpc-id $vpc_id and subnet-id $subnet_id are correct and they support EC2 with public IP and internet gateway with proper routing."
echo "This script will use the above info to create EC2 instance."
fi
cd $DIR/..
# Generate key pair
Expand All @@ -1730,8 +1764,12 @@ aws_start_svr_sh: |
chmod 400 $KEY_FILE
# Generate Security Group
sg_result=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group")
if [ $using_default_vpc == true ]
then
sg_result=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group")
else
sg_result=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" --vpc-id $vpc_id)
fi
report_status "$?" "Only one NVFL server VM and its security group is allowed. $SECURITY_GROUP exists and thus creating duplicate security group"
sg_id=$(echo $sg_result | jq -r .GroupId)
my_public_ip=$(dig +short myip.opendns.com @resolver1.opendns.com)
Expand All @@ -1749,7 +1787,12 @@ aws_start_svr_sh: |
echo "Creating VM at region $REGION, may take a few minutes."
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id > vm_create.json
if [ $using_default_vpc == true ]
then
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id > vm_create.json
else
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id --subnet-id $subnet_id > vm_create.json
fi
report_status "$?" "creating VM"
instance_id=$(jq -r .Instances[0].InstanceId vm_create.json)
Expand Down Expand Up @@ -1807,6 +1850,13 @@ aws_start_cln_sh: |
check_binary dig "Please install it first."
check_binary jq "Please install it first."
if [ -z ${vpc_id+x} ]
then
using_default_vpc=true
else
using_default_vpc=false
fi
if [ -z ${image_name+x} ]
then
container=false
Expand Down Expand Up @@ -1855,6 +1905,25 @@ aws_start_cln_sh: |
prompt ans "Press ENTER when it's done or no additional dependencies. "
fi
# Check if default VPC exists
if [ $using_default_vpc == true ]
then
echo "Checking if default VPC exists"
found_default_vpc=$(aws ec2 describe-vpcs | jq '.Vpcs[] | select(.IsDefault == true)')
if [ -z "${found_default_vpc}" ]
then
echo "No default VPC found. Please create one before running this script with the following command."
echo "aws ec2 create-default-vpc"
echo "or specify your own vpc and subnet with --vpc-id and --subnet-id"
exit
else
echo "Default VPC found"
fi
else
echo "Please check the vpc-id $vpc_id and subnet-id $subnet_id are correct and they support EC2 with public IP and internet gateway with proper routing."
echo "This script will use the above info to create EC2 instance."
fi
cd $DIR/..
# Generate key pair
Expand All @@ -1868,7 +1937,12 @@ aws_start_cln_sh: |
# Generate Security Group
# Try not reusing existing security group because we have to modify it for our own need.
sg_id=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" | jq -r .GroupId)
if [ $using_default_vpc == true ]
then
sg_id=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" | jq -r .GroupId)
else
sg_id=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" --vpc-id $vpc_id | jq -r .GroupId)
fi
report_status "$?" "creating security group"
my_public_ip=$(dig +short myip.opendns.com @resolver1.opendns.com)
if [ "$?" -eq 0 ] && [[ "$my_public_ip" =~ ^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$ ]]
Expand All @@ -1884,7 +1958,12 @@ aws_start_cln_sh: |
echo "Creating VM at region $REGION, may take a few minutes."
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id > vm_create.json
if [ $using_default_vpc == true ]
then
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id > vm_create.json
else
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id --subnet-id $subnet_id > vm_create.json
fi
report_status "$?" "creating VM"
instance_id=$(jq -r .Instances[0].InstanceId vm_create.json)
Expand Down Expand Up @@ -1949,6 +2028,13 @@ aws_start_dsb_sh: |
check_binary dig "Please install it first."
check_binary jq "Please install it first."
if [ -z ${vpc_id+x} ]
then
using_default_vpc=true
else
using_default_vpc=false
fi
echo "One initial user will be created when starting dashboard."
echo "Please enter the email address for this user."
read email
Expand All @@ -1964,9 +2050,33 @@ aws_start_dsb_sh: |
report_status "$?" "creating key pair"
chmod 400 $KEY_FILE
# Generate Security Group
# Check if default VPC exists
if [ $using_default_vpc == true ]
then
echo "Checking if default VPC exists"
found_default_vpc=$(aws ec2 describe-vpcs | jq '.Vpcs[] | select(.IsDefault == true)')
if [ -z "${found_default_vpc}" ]
then
echo "No default VPC found. Please create one before running this script with the following command."
echo "aws ec2 create-default-vpc"
echo "or specify your own vpc and subnet with --vpc-id and --subnet-id"
exit
else
echo "Default VPC found"
fi
else
echo "Please check the vpc-id $vpc_id and subnet-id $subnet_id are correct and they support EC2 with public IP and internet gateway with proper routing."
echo "This script will use the above info to create EC2 instance."
fi
sg_id=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" | jq -r .GroupId)
# Generate Security Group
# Try not reusing existing security group because we have to modify it for our own need.
if [ $using_default_vpc == true ]
then
sg_id=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" | jq -r .GroupId)
else
sg_id=$(aws ec2 create-security-group --group-name $SECURITY_GROUP --description "NVFlare security group" --vpc-id $vpc_id | jq -r .GroupId)
fi
report_status "$?" "creating security group"
echo "Security group id: ${sg_id}"
my_public_ip=$(dig +short myip.opendns.com @resolver1.opendns.com)
Expand All @@ -1983,8 +2093,12 @@ aws_start_dsb_sh: |
# Start provisioning
echo "Creating VM at region $REGION, may take a few minutes."
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id > vm_create.json
if [ $using_default_vpc == true ]
then
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id > vm_create.json
else
aws ec2 run-instances --region $REGION --image-id $AMI_IMAGE --count 1 --instance-type $EC2_TYPE --key-name $KEY_PAIR --security-group-ids $sg_id --subnet-id $subnet_id > vm_create.json
fi
report_status "$?" "creating VM"
instance_id=$(jq -r .Instances[0].InstanceId vm_create.json)
Expand Down

0 comments on commit def9adf

Please sign in to comment.