diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..603553e --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,4 @@ +# Workflows + +The actions only push the opencap (-dev) image to ECR, not the openpose (-dev) and mmpose (-dev) images. +The mmpose (-dev) image is too big and the action fail. To push them to ECR, do it manually through the makefile by running make build and then make run. \ No newline at end of file diff --git a/.github/workflows/ecr-dev.yml b/.github/workflows/ecr-dev.yml new file mode 100644 index 0000000..ca3a75e --- /dev/null +++ b/.github/workflows/ecr-dev.yml @@ -0,0 +1,70 @@ +# This workflow will build and push a new container image to Amazon ECR, +# and then will deploy a new task definition to Amazon ECS, on every push +# to the master branch. +# +# To use this workflow, you will need to complete the following set-up steps: +# +# 1. Create an ECR repository to store your images. +# For example: `aws ecr create-repository --repository-name my-ecr-repo --region us-east-2`. +# Replace the value of `ECR_REPOSITORY` in the workflow below with your repository's name. +# Replace the value of `aws-region` in the workflow below with your repository's region. +# +# 2. Create an ECS task definition, an ECS cluster, and an ECS service. +# For example, follow the Getting Started guide on the ECS console: +# https://us-east-2.console.aws.amazon.com/ecs/home?region=us-east-2#/firstRun +# Replace the values for `service` and `cluster` in the workflow below with your service and cluster names. +# +# 3. Store your ECS task definition as a JSON file in your repository. +# The format should follow the output of `aws ecs register-task-definition --generate-cli-skeleton`. +# Replace the value of `task-definition` in the workflow below with your JSON file's name. +# Replace the value of `container-name` in the workflow below with the name of the container +# in the `containerDefinitions` section of the task definition. +# +# 4. Store an IAM user access key in GitHub Actions secrets named `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. +# See the documentation for each action used below for the recommended IAM policies for this IAM user, +# and best practices on handling the access key credentials. + +on: + push: + branches: + - dev + +name: Deploy to Amazon ECS + +jobs: + deploy: + name: Deploy OpenCap + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: opencap/opencap-dev + IMAGE_TAG: latest # ${{ github.sha }} + run: | + # Build a docker container and + # push it to ECR so that it can + # be deployed to ECS. + docker build -f docker/Dockerfile -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + + - name: Force deployment + run: | + aws ecs update-service --cluster opencap-processing-cluster-dev --service worker --force-new-deployment diff --git a/Examples/changeSessionMetadata.py b/Examples/changeSessionMetadata.py index eb63f08..eb5425c 100644 --- a/Examples/changeSessionMetadata.py +++ b/Examples/changeSessionMetadata.py @@ -23,6 +23,8 @@ developer use. The available options for metadata are: + - scalingsetup: upright_standing_pose + any_pose - openSimModel: LaiUhlrich2022 LaiUhlrich2022_shoulder - posemodel: openpose @@ -44,12 +46,15 @@ from utils import changeSessionMetadata -session_ids = ['0d46adef-62cb-455f-9ff3-8116717cc2fe'] +session_ids = ["0d46adef-62cb-455f-9ff3-8116717cc2fe"] # Dictionary of metadata fields to change (see sessionMetadata.yaml). -newMetadata = {'openSimModel':'LaiUhlrich2022_shoulder', - 'posemodel':'hrnet', - 'augmentermodel':'v0.3', - 'filterfrequency':15, - 'datasharing':'Share processed data and identified videos'} +newMetadata = { + 'openSimModel':'LaiUhlrich2022_shoulder', + 'posemodel':'hrnet', + 'augmentermodel':'v0.3', + 'filterfrequency':15, + 'datasharing':'Share processed data and identified videos', + 'scalingsetup': 'upright_standing_pose' +} changeSessionMetadata(session_ids,newMetadata) \ No newline at end of file diff --git a/app.py b/app.py index 37460c6..86a2644 100644 --- a/app.py +++ b/app.py @@ -8,22 +8,32 @@ import logging import glob import numpy as np -from utilsAPI import getAPIURL, getWorkerType +from utilsAPI import getAPIURL, getWorkerType, getASInstance, unprotect_current_instance, get_number_of_pending_trials from utilsAuth import getToken -from utils import getDataDirectory, checkTime, checkResourceUsage, sendStatusEmail +from utils import (getDataDirectory, checkTime, checkResourceUsage, + sendStatusEmail, checkForTrialsWithStatus) logging.basicConfig(level=logging.INFO) API_TOKEN = getToken() API_URL = getAPIURL() workerType = getWorkerType() +autoScalingInstance = getASInstance() +logging.info(f"AUTOSCALING TEST INSTANCE: {autoScalingInstance}") # if true, will delete entire data directory when finished with a trial isDocker = True # get start time -t = time.localtime() initialStatusCheck = False +t = time.localtime() + +# For removing AWS machine scale-in protection +t_lastTrial = time.localtime() +justProcessed = True +with_on_prem = True +minutesBeforeRemoveScaleInProtection = 2 +max_on_prem_pending_trials = 5 while True: # Run test trial at a given frequency to check status of machine. Stop machine if fails. @@ -31,6 +41,23 @@ runTestSession(isDocker=isDocker) t = time.localtime() initialStatusCheck = True + + # When using autoscaling, if there are on-prem workers, then we will remove + # the instance scale-in protection if the number of pending trials is below + # a threshold so that the on-prem workers are prioritized. + if with_on_prem: + # Query the number of pending trials + if autoScalingInstance: + pending_trials = get_number_of_pending_trials() + logging.info(f"Number of pending trials: {pending_trials}") + if pending_trials < max_on_prem_pending_trials: + # Remove scale-in protection and sleep in the cycle so that the + # asg will remove that instance from the group. + logging.info("Removing scale-in protection (out loop).") + unprotect_current_instance() + logging.info("Removed scale-in protection (out loop).") + for i in range(3600): + time.sleep(1) # workerType = 'calibration' -> just processes calibration and neutral # workerType = 'all' -> processes all types of trials @@ -47,6 +74,27 @@ if r.status_code == 404: logging.info("...pulling " + workerType + " trials.") time.sleep(1) + + # When using autoscaling, we will remove the instance scale-in protection if it hasn't + # pulled a trial recently and there are no actively recording trials + if (autoScalingInstance and not justProcessed and + checkTime(t_lastTrial, minutesElapsed=minutesBeforeRemoveScaleInProtection)): + if checkForTrialsWithStatus('recording', hours=2/60) == 0: + # Remove scale-in protection and sleep in the cycle so that the + # asg will remove that instance from the group. + logging.info("Removing scale-in protection (in loop).") + unprotect_current_instance() + logging.info("Removed scale-in protection (in loop).") + for i in range(3600): + time.sleep(1) + else: + t_lastTrial = time.localtime() + + # If a trial was just processed, reset the timer. + if autoScalingInstance and justProcessed: + justProcessed = False + t_lastTrial = time.localtime() + continue if np.floor(r.status_code/100) == 5: # 5xx codes are server faults @@ -88,12 +136,12 @@ logging.info("processTrial({},{},trial_type={})".format(trial["session"], trial["id"], trial_type)) try: + # trigger reset of timer for last processed trial processTrial(trial["session"], trial["id"], trial_type=trial_type, isDocker=isDocker) # note a result needs to be posted for the API to know we finished, but we are posting them # automatically thru procesTrial now r = requests.patch(trial_url, data={"status": "done"}, headers = {"Authorization": "Token {}".format(API_TOKEN)}) - logging.info('0.5s pause if need to restart.') time.sleep(0.5) except Exception as e: @@ -106,6 +154,7 @@ message = "A backend OpenCap machine timed out during pose detection. It has been stopped." sendStatusEmail(message=message) raise Exception('Worker failed. Stopped.') + justProcessed = True # Clean data directory if isDocker: diff --git a/docker/Makefile b/docker/Makefile index 0aedb7a..35bfb4e 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,17 +1,38 @@ +BASE_NAME := 660440363484.dkr.ecr.us-west-2.amazonaws.com +REPO_NAME := opencap +PROD_BRANCH := main + +# Determine the branch name +CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) + +# Determine image tag based on branch +ifeq ($(CURRENT_BRANCH),$(PROD_BRANCH)) + OPENCAP_IMAGE_TAG := opencap + OPENPOSE_IMAGE_TAG := openpose + MMPOSE_IMAGE_TAG := mmpose +else + OPENCAP_IMAGE_TAG := opencap-dev + OPENPOSE_IMAGE_TAG := openpose-dev + MMPOSE_IMAGE_TAG := mmpose-dev +endif + + .PHONY: build build: wget -c -O ../mmpose/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth wget -c -O ../mmpose/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth - docker build -t 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap .. -f Dockerfile - docker build -t 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose .. -f openpose/Dockerfile - docker build -t 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose .. -f mmpose/Dockerfile + + docker build -t $(BASE_NAME)/$(REPO_NAME)/$(OPENCAP_IMAGE_TAG) .. -f Dockerfile + docker build -t $(BASE_NAME)/$(REPO_NAME)/$(OPENPOSE_IMAGE_TAG) .. -f openpose/Dockerfile + docker build -t $(BASE_NAME)/$(REPO_NAME)/$(MMPOSE_IMAGE_TAG) .. -f mmpose/Dockerfile .PHONY: push push: aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose + + docker push $(BASE_NAME)/$(REPO_NAME)/$(OPENCAP_IMAGE_TAG) + docker push $(BASE_NAME)/$(REPO_NAME)/$(OPENPOSE_IMAGE_TAG) + docker push $(BASE_NAME)/$(REPO_NAME)/$(MMPOSE_IMAGE_TAG) .PHONY: run run: diff --git a/main.py b/main.py index 08425ed..57ffece 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,9 @@ import yaml import traceback +import logging +logging.basicConfig(level=logging.INFO) + from utils import importMetadata, loadCameraParameters, getVideoExtension from utils import getDataDirectory, getOpenPoseDirectory, getMMposeDirectory from utilsChecker import saveCameraParameters @@ -37,7 +40,8 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], scaleModel=False, bbox_thr=0.8, augmenter_model='v0.3', genericFolderNames=False, offset=True, benchmark=False, dataDir=None, overwriteAugmenterModel=False, - filter_frequency='default', overwriteFilterFrequency=False): + filter_frequency='default', overwriteFilterFrequency=False, + scaling_setup='upright_standing_pose', overwriteScalingSetup=False): # %% High-level settings. # Camera calibration. @@ -109,6 +113,14 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], else: filtFreqs = {'gait':filterfrequency, 'default':filterfrequency} + # If scaling setup defined through web app. + # If overwriteScalingSetup is True, the scaling setup is the one + # passed as an argument to main(). This is useful for local testing. + if 'scalingsetup' in sessionMetadata and not overwriteScalingSetup: + scalingSetup = sessionMetadata['scalingsetup'] + else: + scalingSetup = scaling_setup + # %% Paths to pose detector folder for local testing. if poseDetector == 'OpenPose': poseDetectorDirectory = getOpenPoseDirectory(isDocker) @@ -152,7 +164,10 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], 'poseDetector': poseDetector, 'augmenter_model': augmenterModel, 'imageUpsampleFactor': imageUpsampleFactor, - 'openSimModel': sessionMetadata['openSimModel']} + 'openSimModel': sessionMetadata['openSimModel'], + 'scalingSetup': scalingSetup, + 'filterFrequency': filterfrequency, + } if poseDetector == 'OpenPose': settings['resolutionPoseDetection'] = resolutionPoseDetection elif poseDetector == 'mmpose': @@ -191,7 +206,7 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], # Intrinsics and extrinsics already exist for this session. if os.path.exists( os.path.join(camDir,"cameraIntrinsicsExtrinsics.pickle")): - print("Load extrinsics for {} - already existing".format( + logging.info("Load extrinsics for {} - already existing".format( camName)) CamParams = loadCameraParameters( os.path.join(camDir, "cameraIntrinsicsExtrinsics.pickle")) @@ -199,8 +214,7 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], # Extrinsics do not exist for this session. else: - print("Compute extrinsics for {} - not yet existing".format( - camName)) + logging.info("Compute extrinsics for {} - not yet existing".format(camName)) # Intrinsics ################################################## # Intrinsics directories. intrinsicDir = os.path.join(baseDir, 'CameraIntrinsics', @@ -396,7 +410,7 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], if runMarkerAugmentation: os.makedirs(postAugmentationDir, exist_ok=True) augmenterDir = os.path.join(baseDir, "MarkerAugmenter") - print('Augmenting marker set') + logging.info('Augmenting marker set') try: vertical_offset = augmentTRC( pathOutputFiles[trialName],sessionMetadata['mass_kg'], @@ -441,8 +455,11 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], if scaleModel: os.makedirs(outputScaledModelDir, exist_ok=True) # Path setup file. - genericSetupFile4ScalingName = ( - 'Setup_scaling_RajagopalModified2016_withArms_KA.xml') + if scalingSetup == 'any_pose': + genericSetupFile4ScalingName = 'Setup_scaling_LaiUhlrich2022_any_pose.xml' + else: # by default, use upright_standing_pose + genericSetupFile4ScalingName = 'Setup_scaling_LaiUhlrich2022.xml' + pathGenericSetupFile4Scaling = os.path.join( openSimPipelineDir, 'Scaling', genericSetupFile4ScalingName) # Path model file. @@ -465,11 +482,11 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], thresholdTime=0.1, removeRoot=True) success = True except Exception as e: - print(f"Attempt with thresholdPosition {thresholdPosition} failed: {e}") + logging.info(f"Attempt identifying scaling time range with thresholdPosition {thresholdPosition} failed: {e}") thresholdPosition += increment # Increase the threshold for the next iteration # Run scale tool. - print('Running Scaling') + logging.info('Running Scaling') pathScaledModel = runScaleTool( pathGenericSetupFile4Scaling, pathGenericModel4Scaling, sessionMetadata['mass_kg'], pathTRCFile4Scaling, @@ -507,7 +524,7 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], # Path TRC file. pathTRCFile4IK = pathAugmentedOutputFiles[trialName] # Run IK tool. - print('Running Inverse Kinematics') + logging.info('Running Inverse Kinematics') try: pathOutputIK = runIKTool( pathGenericSetupFile4IK, pathScaledModel, diff --git a/mmpose/loop_mmpose.py b/mmpose/loop_mmpose.py index 4bedc36..6f18a88 100644 --- a/mmpose/loop_mmpose.py +++ b/mmpose/loop_mmpose.py @@ -35,6 +35,7 @@ def checkCudaPyTorch(): if os.path.isfile(video_path): os.remove(video_path) +checkCudaPyTorch() while True: if not os.path.isfile(video_path): time.sleep(0.1) diff --git a/opensimPipeline/Models/RajagopalModified2016_markers_augmenter.xml b/opensimPipeline/Models/LaiUhlrich2022_markers_augmenter.xml similarity index 100% rename from opensimPipeline/Models/RajagopalModified2016_markers_augmenter.xml rename to opensimPipeline/Models/LaiUhlrich2022_markers_augmenter.xml diff --git a/opensimPipeline/Models/RajagopalModified2016_markers_augmenter_shoulder.xml b/opensimPipeline/Models/LaiUhlrich2022_markers_augmenter_shoulder.xml similarity index 100% rename from opensimPipeline/Models/RajagopalModified2016_markers_augmenter_shoulder.xml rename to opensimPipeline/Models/LaiUhlrich2022_markers_augmenter_shoulder.xml diff --git a/opensimPipeline/Models/RajagopalModified2016_markers_mmpose.xml b/opensimPipeline/Models/LaiUhlrich2022_markers_mmpose.xml similarity index 100% rename from opensimPipeline/Models/RajagopalModified2016_markers_mmpose.xml rename to opensimPipeline/Models/LaiUhlrich2022_markers_mmpose.xml diff --git a/opensimPipeline/Models/RajagopalModified2016_markers_mocap.xml b/opensimPipeline/Models/LaiUhlrich2022_markers_mocap.xml similarity index 100% rename from opensimPipeline/Models/RajagopalModified2016_markers_mocap.xml rename to opensimPipeline/Models/LaiUhlrich2022_markers_mocap.xml diff --git a/opensimPipeline/Models/RajagopalModified2016_markers_mocap_shoulder.xml b/opensimPipeline/Models/LaiUhlrich2022_markers_mocap_shoulder.xml similarity index 100% rename from opensimPipeline/Models/RajagopalModified2016_markers_mocap_shoulder.xml rename to opensimPipeline/Models/LaiUhlrich2022_markers_mocap_shoulder.xml diff --git a/opensimPipeline/Models/RajagopalModified2016_markers_openpose.xml b/opensimPipeline/Models/LaiUhlrich2022_markers_openpose.xml similarity index 100% rename from opensimPipeline/Models/RajagopalModified2016_markers_openpose.xml rename to opensimPipeline/Models/LaiUhlrich2022_markers_openpose.xml diff --git a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA.xml b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022.xml similarity index 99% rename from opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA.xml rename to opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022.xml index df14235..df00928 100644 --- a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA.xml +++ b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022.xml @@ -1,6 +1,6 @@ - + 75.337 diff --git a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_Mocap.xml b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_Mocap.xml similarity index 99% rename from opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_Mocap.xml rename to opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_Mocap.xml index cb6a0a5..e621404 100644 --- a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_Mocap.xml +++ b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_Mocap.xml @@ -1,6 +1,6 @@ - + 75.337 diff --git a/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_any_pose.xml b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_any_pose.xml new file mode 100644 index 0000000..c9057fa --- /dev/null +++ b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_any_pose.xml @@ -0,0 +1,639 @@ + + + + + 75.337 + + 1.8899999999999999 + + -1 + + Unassigned + + + + Unassigned + + Unassigned + + + + + true + + measurements + + + + + + true + + + + + + r_calc_study r_toe_study + + + + r_calc_study r_5meta_study + + + + L_calc_study L_toe_study + + + + L_calc_study L_5meta_study + + + + + + + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + + + + + true + + + + + + r_knee_study r_ankle_study + + + + r_mknee_study r_mankle_study + + + + + + + + + + X Y Z + + + + + + + + true + + + + + + RHJC_study r_knee_study + + + + + + + + + + X Y Z + + + + X Y Z + + + + + + + + true + + + + + + L_knee_study L_ankle_study + + + + L_mknee_study L_mankle_study + + + + + + + + + + X Y Z + + + + + + + + true + + + + + + LHJC_study L_knee_study + + + + + + + + + + X Y Z + + + + X Y Z + + + + + + + + true + + + + + + r.ASIS_study r.PSIS_study + + + + L.ASIS_study L.PSIS_study + + + + + + + + + + X Y + + + + + + + + true + + + + + + RHJC_study LHJC_study + + + + + + + + + + Z + + + + + + + + true + + + + + + r_shoulder_study r.PSIS_study + + + + L_shoulder_study L.PSIS_study + + + + C7_study r.PSIS_study + + + + C7_study L.PSIS_study + + + + + + + + + + X Y + + + + X Y + + + + X Y + + + + + + + + true + + + + + + r_shoulder_study L_shoulder_study + + + + + + + + + + Z + + + + Z + + + + Z + + + + + + + + true + + + + + + r_shoulder_study r_lelbow_study + + + + r_shoulder_study r_melbow_study + + + + L_shoulder_study L_lelbow_study + + + + L_shoulder_study L_melbow_study + + + + + + + + + + X Y Z + + + + X Y Z + + + + + + + + true + + + + + + r_lwrist_study r_lelbow_study + + + + r_mwrist_study r_melbow_study + + + + L_lwrist_study L_lelbow_study + + + + L_mwrist_study L_melbow_study + + + + + + + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + X Y Z + + + + + + + + + + + + + + + Unassigned + + 3.41 3.66 + + true + + + + + + + + + true + + + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 10 + + + + true + + 5 + + + + true + + 5 + + + + true + + 20 + + + + true + + 1 + + + + true + + 5 + + + + true + + 5 + + + + true + + 1 + + + + true + + 5 + + + + true + + 5 + + + + + + Unassigned + + Unassigned + + 3.41 3.66 + + + + + + + + -1 + + + diff --git a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_mmpose.xml b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_mmpose.xml similarity index 99% rename from opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_mmpose.xml rename to opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_mmpose.xml index 3451f88..48c0a6f 100644 --- a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_mmpose.xml +++ b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_mmpose.xml @@ -1,6 +1,6 @@ - + 75.337 diff --git a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_openpose.xml b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_openpose.xml similarity index 99% rename from opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_openpose.xml rename to opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_openpose.xml index a176066..9863c0e 100644 --- a/opensimPipeline/Scaling/Setup_scaling_RajagopalModified2016_withArms_KA_openpose.xml +++ b/opensimPipeline/Scaling/Setup_scaling_LaiUhlrich2022_openpose.xml @@ -1,6 +1,6 @@ - + 75.337 diff --git a/requirements.txt b/requirements.txt index e2eff9e..3d62633 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ pingouin==0.5.2 openpyxl ffmpeg-python psutil +boto3 \ No newline at end of file diff --git a/utils.py b/utils.py index c21866d..05aba40 100644 --- a/utils.py +++ b/utils.py @@ -377,7 +377,7 @@ def getMetadataFromServer(session_id,justCheckerParams=False): session_desc["posemodel"] = session['meta']['subject']['posemodel'] except: session_desc["posemodel"] = 'openpose' - # This might happen if openSimModel/augmentermodel/filterfrequency was changed post data collection. + # This might happen if openSimModel/augmentermodel/filterfrequency/scalingsetup was changed post data collection. if 'settings' in session['meta']: try: session_desc["openSimModel"] = session['meta']['settings']['openSimModel'] @@ -393,6 +393,10 @@ def getMetadataFromServer(session_id,justCheckerParams=False): session_desc["filterfrequency"] = float(session_desc["filterfrequency"]) except: session_desc["filterfrequency"] = 'default' + try: + session_desc["scalingsetup"] = session['meta']['settings']['scalingsetup'] + except: + session_desc["scalingsetup"] = 'upright_standing_pose' else: subject_info = getSubjectJson(session['subject']) session_desc["subjectID"] = subject_info['name'] @@ -416,6 +420,10 @@ def getMetadataFromServer(session_id,justCheckerParams=False): session_desc["filterfrequency"] = float(session_desc["filterfrequency"]) except: session_desc["filterfrequency"] = 'default' + try: + session_desc["scalingsetup"] = session['meta']['settings']['scalingsetup'] + except: + session_desc["scalingsetup"] = 'upright_standing_pose' if 'sessionWithCalibration' in session['meta'] and 'checkerboard' not in session['meta']: newSessionId = session['meta']['sessionWithCalibration']['id'] @@ -663,7 +671,7 @@ def changeSessionMetadata(session_ids,newMetaDict): for newMeta in newMetaDict: if not newMeta in addedKey: print("Could not find {} in existing metadata, trying to add it.".format(newMeta)) - settings_fields = ['framerate', 'posemodel', 'openSimModel', 'augmentermodel', 'filterfrequency'] + settings_fields = ['framerate', 'posemodel', 'openSimModel', 'augmentermodel', 'filterfrequency', 'scalingsetup'] if newMeta in settings_fields: if 'settings' not in existingMeta: existingMeta['settings'] = {} @@ -1459,7 +1467,21 @@ def getVideoExtension(pathFileWithoutExtension): # check how much time has passed since last status check def checkTime(t,minutesElapsed=30): t2 = time.localtime() - return (t2.tm_hour - t.tm_hour) * 60 + (t2.tm_min - t.tm_min) >= minutesElapsed + return (t2.tm_hour - t.tm_hour) * 3600 + (t2.tm_min - t.tm_min)*60 + (t2.tm_sec - t.tm_sec) >= minutesElapsed*60 + +# check for trials with certain status +def checkForTrialsWithStatus(status,hours=9999999,relativeTime='newer'): + + # get trials with statusOld + params = {'status':status, + 'hoursSinceUpdate':hours, + 'justNumber':1, + 'relativeTime':relativeTime} + + r = requests.get(API_URL+"trials/get_trials_with_status/",params=params, + headers = {"Authorization": "Token {}".format(API_TOKEN)}).json() + + return r['nTrials'] # send status email def sendStatusEmail(message=None,subject=None): diff --git a/utilsAPI.py b/utilsAPI.py index 860be0d..cf0e837 100644 --- a/utilsAPI.py +++ b/utilsAPI.py @@ -4,7 +4,12 @@ @author: suhlr """ +import os +import boto3 +import requests + from decouple import config +from datetime import datetime, timedelta def getAPIURL(): if 'API_URL' not in globals(): @@ -38,4 +43,84 @@ def getStatusEmails(): emailInfo = None return emailInfo - \ No newline at end of file + +def getASInstance(): + try: + # Check if the ECS_CONTAINER_METADATA_FILE environment variable exists + ecs_metadata_file = os.getenv('ECS_CONTAINER_METADATA_FILE') + if ecs_metadata_file: + if os.path.isfile(ecs_metadata_file): + return True + else: + return False + else: + return False + except Exception as e: + return False + +def get_metric_average(namespace, metric_name, start_time, end_time, period): + """ + Fetch the average value of a specific metric from AWS CloudWatch. + + Parameters: + - namespace (str): The namespace for the metric data. + - metric_name (str): The name of the metric. + - start_time (datetime): Start time for the data retrieval. + - end_time (datetime): End time for the data retrieval. + - period (int): The granularity, in seconds, of the data points returned. + """ + client = boto3.client('cloudwatch', region_name='us-west-2') + response = client.get_metric_statistics( + Namespace=namespace, + MetricName=metric_name, + StartTime=start_time, + EndTime=end_time, + Period=period, + Statistics=['Average'] # Correctly specifying 'Average' here + ) + return response + +def get_number_of_pending_trials(period=60): + # Time range setup for the last 1 minute + end_time = datetime.utcnow() + start_time = end_time - timedelta(minutes=1) + + # Fetch the metric data + namespace = 'Custom/opencap-dev' # or 'Custom/opencap' for production + metric_name = 'opencap_trials_pending' + stats = get_metric_average( + namespace, metric_name, start_time, end_time, period) + + if stats['Datapoints']: + average = stats['Datapoints'][0]['Average'] + else: + # Maybe raise an exception or do nothing to have control-loop retry this call later + return None + + return average + +def get_instance_id(): + """Retrieve the instance ID from EC2 metadata.""" + response = requests.get("http://169.254.169.254/latest/meta-data/instance-id") + return response.text + +def get_auto_scaling_group_name(instance_id): + """Retrieve the Auto Scaling Group name using the instance ID.""" + client = boto3.client('autoscaling', region_name='us-west-2') + response = client.describe_auto_scaling_instances(InstanceIds=[instance_id]) + asg_name = response['AutoScalingInstances'][0]['AutoScalingGroupName'] + return asg_name + +def set_instance_protection(instance_id, asg_name, protect): + """Set or remove instance protection.""" + client = boto3.client('autoscaling', region_name='us-west-2') + client.set_instance_protection( + InstanceIds=[instance_id], + AutoScalingGroupName=asg_name, + ProtectedFromScaleIn=protect + ) + +def unprotect_current_instance(): + instance_id = get_instance_id() + asg_name = get_auto_scaling_group_name(instance_id) + set_instance_protection(instance_id, asg_name, protect=False) diff --git a/utilsAugmenter.py b/utilsAugmenter.py index f10c4b1..c7eee42 100644 --- a/utilsAugmenter.py +++ b/utilsAugmenter.py @@ -45,7 +45,7 @@ def augmentTRC(pathInputTRCFile, subject_mass, subject_height, augmenterModelType_all = [augmenterModelType_lower, augmenterModelType_upper] feature_markers_all = [feature_markers_lower, feature_markers_upper] response_markers_all = [response_markers_lower, response_markers_upper] - print('Using augmenter model: {}'.format(augmenter_model)) + # print('Using augmenter model: {}'.format(augmenter_model)) # %% Process data. # Import TRC file @@ -112,7 +112,7 @@ def augmentTRC(pathInputTRCFile, subject_mass, subject_height, json_file.close() model = tf.keras.models.model_from_json(pretrainedModel_json) model.load_weights(os.path.join(augmenterModelDir, "weights.h5")) - outputs = model.predict(inputs) + outputs = model.predict(inputs, verbose=2) # %% Post-process outputs. # Step 1: Reshape if necessary (eg, LSTM) diff --git a/utilsDetector.py b/utilsDetector.py index 3038654..1e57dde 100644 --- a/utilsDetector.py +++ b/utilsDetector.py @@ -318,7 +318,7 @@ def runMMposeVideo( time.sleep(0.1) # copy /data/output to pathOutputPkl - os.system("cp /data/output_mmpose/* {pathOutputPkl}/".format(pathOutputPkl=pathOutputPkl)) + os.system("cp /data/output_mmpose/* {pathOutputPkl}/".format(pathOutputPkl=pathOutputPkl)) pkl_path_tmp = os.path.join(pathOutputPkl, 'human.pkl') os.rename(pkl_path_tmp, pklPath) diff --git a/utilsMMpose.py b/utilsMMpose.py index 2319c90..0201620 100644 --- a/utilsMMpose.py +++ b/utilsMMpose.py @@ -2,7 +2,7 @@ import pickle import torch -from tqdm import tqdm +# from tqdm import tqdm from mmpose_utils import process_mmdet_results, frame_iter, concat, convert_instance_to_frame try: from mmdet.apis import inference_detector, init_detector @@ -47,7 +47,8 @@ def detection_inference(model_config, model_ckpt, video_path, bbox_path, output = [] nFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - for img in tqdm(frame_iter(cap), total=nFrames): + # for img in tqdm(frame_iter(cap), total=nFrames): + for img in frame_iter(cap): # test a single image, the resulting box is (x1, y1, x2, y2) mmdet_results = inference_detector(det_model, img) @@ -87,7 +88,8 @@ def pose_inference(model_config, model_ckpt, video_path, bbox_path, pkl_path, # run pose inference print("Running pose inference...") instances = [] - for batch in tqdm(dataloader): + # for batch in tqdm(dataloader): + for batch in dataloader: batch['img'] = batch['img'].to(device) batch['img_metas'] = [img_metas[0] for img_metas in batch['img_metas'].data] with torch.no_grad(): @@ -122,7 +124,8 @@ def pose_inference(model_config, model_ckpt, video_path, bbox_path, pkl_path, dataset = model.cfg.data.test.type dataset_info_d = get_dataset_info() dataset_info = DatasetInfo(dataset_info_d[dataset]) - for pose_results, img in tqdm(zip(results, frame_iter(cap))): + # for pose_results, img in tqdm(zip(results, frame_iter(cap))): + for pose_results, img in zip(results, frame_iter(cap)): for instance in pose_results: instance['keypoints'] = instance['preds_with_flip'] vis_img = vis_pose_tracking_result(model, img, pose_results, diff --git a/utilsOpenSim.py b/utilsOpenSim.py index cb32d70..18fa62b 100644 --- a/utilsOpenSim.py +++ b/utilsOpenSim.py @@ -31,16 +31,16 @@ def runScaleTool(pathGenericSetupFile, pathGenericModel, subjectMass, _, setupFileName = os.path.split(pathGenericSetupFile) if 'Lai' in scaledModelName or 'Rajagopal' in scaledModelName: if 'Mocap' in setupFileName: - markerSetFileName = 'RajagopalModified2016_markers_mocap{}.xml'.format(suffix_model) + markerSetFileName = 'LaiUhlrich2022_markers_mocap{}.xml'.format(suffix_model) elif 'openpose' in setupFileName: - markerSetFileName = 'RajagopalModified2016_markers_openpose.xml' + markerSetFileName = 'LaiUhlrich2022_markers_openpose.xml' elif 'mmpose' in setupFileName: - markerSetFileName = 'RajagopalModified2016_markers_mmpose.xml' + markerSetFileName = 'LaiUhlrich2022_markers_mmpose.xml' else: if fixed_markers: - markerSetFileName = 'RajagopalModified2016_markers_augmenter_fixed.xml' + markerSetFileName = 'LaiUhlrich2022_markers_augmenter_fixed.xml' else: - markerSetFileName = 'RajagopalModified2016_markers_augmenter{}.xml'.format(suffix_model) + markerSetFileName = 'LaiUhlrich2022_markers_augmenter{}.xml'.format(suffix_model) elif 'gait2392' in scaledModelName: if 'Mocap' in setupFileName: markerSetFileName = 'gait2392_markers_mocap.xml' diff --git a/utilsServer.py b/utilsServer.py index 9424e1b..865ceaa 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -456,7 +456,7 @@ def runTestSession(pose='all',isDocker=True): try: for trial_id in trialList: trial = getTrialJson(trial_id) - logging.info("Running status check on trial name: " + trial['name'] + "\n\n") + logging.info("Running status check on trial name: " + trial['name'] + "_" + str(trial_id) + "\n\n") processTrial(trial["session"], trial_id, trial_type='static', isDocker=isDocker) except: logging.info("test trial failed. stopping machine.")