Skip to content

Commit

Permalink
Feat 46 hpc endpoint (#54)
Browse files Browse the repository at this point in the history
* feat: adds hpc job submit model

* feat: updates server

* feat: updates unit tests

* feat: adds endpoint to submit hpc configs
  • Loading branch information
jtyoung84 authored Oct 12, 2023
1 parent 1fff303 commit 02cbb82
Show file tree
Hide file tree
Showing 7 changed files with 947 additions and 9 deletions.
1 change: 1 addition & 0 deletions src/aind_data_transfer_service/configs/job_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ def from_csv_row(
)


# Deprecating this class
class HpcJobConfigs(BaseSettings):
"""Class to contain settings for hpc resources"""

Expand Down
47 changes: 46 additions & 1 deletion src/aind_data_transfer_service/hpc/client.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
"""Module to manage connection with hpc cluster"""

from typing import Optional, Union
import json
from typing import List, Optional, Union

import requests
from pydantic import BaseSettings, Field, SecretStr, validator
from requests.models import Response

from aind_data_transfer_service.hpc.models import HpcJobSubmitSettings


class HpcClientConfigs(BaseSettings):
"""Configs needed to connect to the hpc cluster"""
Expand Down Expand Up @@ -100,3 +103,45 @@ def submit_job(self, job_def: dict) -> Response:
url=self._job_submit_url, json=job_def, headers=self.__headers
)
return response

def submit_hpc_job(
self,
script: str,
job: Optional[HpcJobSubmitSettings] = None,
jobs: Optional[List[HpcJobSubmitSettings]] = None,
) -> Response:
"""
Submit a job following the v0.0.36 Slurm rest api job submission guide
Parameters
----------
script : str
Executable script (full contents) to run in batch step
job : Optional[HpcJobSubmitSettings]
v0.0.36_job_properties (Default is None)
jobs : Optional[List[HpcJobSubmitSettings]]
List of properties of an HetJob (Default is None)
Returns
-------
Response
"""
# Assert at least one of job or jobs is defined
assert job is not None or jobs is not None
# Assert not both job and jobs are defined
assert job is None or jobs is None
if job is not None:
job_def = {
"job": json.loads(job.json(exclude_none=True)),
"script": script,
}
else:
job_def = {
"jobs": [json.loads(j.json(exclude_none=True)) for j in jobs],
"script": script,
}

response = requests.post(
url=self._job_submit_url, json=job_def, headers=self.__headers
)
return response
Loading

0 comments on commit 02cbb82

Please sign in to comment.