Skip to content

Commit

Permalink
Feat 43 filter status (#55)
Browse files Browse the repository at this point in the history 
* feat: renders time to users locale

* feat: adds mail_user option
  • Loading branch information
@jtyoung84
jtyoung84 authored Oct 14, 2023
1 parent 8cffe5f commit 60c0dd9
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 56 deletions.
10 changes: 7 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ A mock server will be created. You can then create a mock environment to run uvi
```bash
export HPC_HOST="localhost"
export HPC_PORT=3000
export HPC_USERNAME='username'
export HPC_USERNAME='some.user'
export HPC_PASSWORD='password'
export HPC_TOKEN='some_token'
export HPC_PARTITION='part'
Expand All @@ -45,10 +45,14 @@ export HPC_LOGGING_DIRECTORY='/hpc/logging_dir'
export HPC_AWS_ACCESS_KEY_ID='abc-123'
export HPC_AWS_SECRET_ACCESS_KEY='def-456'
export HPC_AWS_DEFAULT_REGION='us-west-2'
export APP_CSRF_SECRET_KEY='anything'
export APP_SECRET_KEY='anything_again'
export HPC_STAGING_DIRECTORY='/hpc/staging_dir'
export HPC_AWS_PARAM_STORE_NAME='/param/store/name'
export HPC_MINIMUM_CPUS_PER_NODE='8'
export HPC_MEMORY_PER_CPU='8000'
export HPC_NODES='[1,1]'
export HPC_TASKS='1'
export HPC_TIME_LIMIT='360'
export HPC_QOS='dev'
uvicorn aind_data_transfer_service.server:app --host 0.0.0.0 --port 5000
```
You can now access `http://localhost:5000/jobs`.
Expand Down
11 changes: 3 additions & 8 deletions src/aind_data_transfer_service/hpc/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,8 +437,6 @@ def from_upload_job_configs(
"SINGULARITYENV_AWS_SESSION_TOKEN"
] = aws_session_token.get_secret_value()
cls._set_default_val(kwargs, "environment", hpc_env)
# Set default time limit to 3 hours
cls._set_default_val(kwargs, "time_limit", 180)
cls._set_default_val(
kwargs,
"standard_out",
Expand All @@ -449,11 +447,6 @@ def from_upload_job_configs(
"standard_error",
str(logging_directory / (kwargs["name"] + "_error.out")),
)
cls._set_default_val(kwargs, "nodes", [1, 1])
cls._set_default_val(kwargs, "minimum_cpus_per_node", 4)
cls._set_default_val(kwargs, "tasks", 1)
# 8 GB per cpu for 32 GB total memory
cls._set_default_val(kwargs, "memory_per_cpu", 8000)
return cls(**kwargs)

@classmethod
Expand Down Expand Up @@ -629,6 +622,7 @@ class JobStatus(BaseModel):
job_id: Optional[int] = Field(None)
job_state: Optional[str] = Field(None)
name: Optional[str] = Field(None)
comment: Optional[str] = Field(None)
start_time: Optional[datetime] = Field(None)
submit_time: Optional[datetime] = Field(None)

Expand All @@ -644,7 +638,7 @@ def _parse_timestamp(
elif timestamp is None or timestamp == 0:
return None
else:
return datetime.fromtimestamp(timestamp)
return datetime.utcfromtimestamp(timestamp)

@classmethod
def from_hpc_job_status(cls, hpc_job: HpcJobStatusResponse):
Expand All @@ -654,6 +648,7 @@ def from_hpc_job_status(cls, hpc_job: HpcJobStatusResponse):
job_id=hpc_job.job_id,
job_state=hpc_job.job_state,
name=hpc_job.name,
comment=hpc_job.comment,
start_time=hpc_job.start_time,
submit_time=hpc_job.submit_time,
)
Expand Down
46 changes: 35 additions & 11 deletions src/aind_data_transfer_service/server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
# TODO: Add server configs model
# UPLOAD_TEMPLATE_LINK
# HPC_SIF_LOCATION
# HPC_USERNAME
# HPC_LOGGING_DIRECTORY
# HPC_AWS_ACCESS_KEY_ID
# HPC_AWS_SECRET_ACCESS_KEY
Expand All @@ -47,7 +48,9 @@ async def validate_csv(request: Request):
"""Validate a csv file. Return parsed contents as json."""
async with request.form() as form:
content = await form["file"].read()
data = content.decode("utf-8")
# A few csv files created from excel have extra unicode byte chars.
# Adding "utf-8-sig" should remove them.
data = content.decode("utf-8-sig")
csv_reader = csv.DictReader(io.StringIO(data))
basic_jobs = []
errors = []
Expand Down Expand Up @@ -132,7 +135,8 @@ async def submit_basic_jobs(request: Request):
)


async def submit_hpc_jobs(request: Request):
# TODO: Refactor to make less complex
async def submit_hpc_jobs(request: Request): # noqa: C901
"""Post HpcJobSubmitSettings to hpc server to process."""

content = await request.json()
Expand All @@ -147,9 +151,20 @@ async def submit_hpc_jobs(request: Request):
parsing_errors = []
for job in job_configs:
try:
base_script = job.get("script")
# If script is empty, assume that the job type is a basic job
basic_job_name = None
if base_script is None or base_script == "":
base_script = HpcJobSubmitSettings.script_command_str(
sif_loc_str=os.getenv("HPC_SIF_LOCATION")
)
basic_job_name = BasicUploadJobConfigs.parse_raw(
job["upload_job_settings"]
).s3_prefix
upload_job_configs = json.loads(job["upload_job_settings"])
hpc_settings = json.loads(job["hpc_settings"])
base_script = job["script"]
if basic_job_name is not None:
hpc_settings["name"] = basic_job_name
hpc_job = HpcJobSubmitSettings.from_upload_job_configs(
logging_directory=Path(os.getenv("HPC_LOGGING_DIRECTORY")),
aws_secret_access_key=SecretStr(
Expand All @@ -166,14 +181,17 @@ async def submit_hpc_jobs(request: Request):
),
**hpc_settings,
)
script = hpc_job.attach_configs_to_script(
script=base_script,
base_configs=upload_job_configs,
upload_configs_aws_param_store_name=os.getenv(
"HPC_AWS_PARAM_STORE_NAME"
),
staging_directory=os.getenv("HPC_STAGING_DIRECTORY"),
)
if not upload_job_configs:
script = base_script
else:
script = hpc_job.attach_configs_to_script(
script=base_script,
base_configs=upload_job_configs,
upload_configs_aws_param_store_name=os.getenv(
"HPC_AWS_PARAM_STORE_NAME"
),
staging_directory=os.getenv("HPC_STAGING_DIRECTORY"),
)
hpc_jobs.append((hpc_job, script))
except Exception as e:
parsing_errors.append(
Expand Down Expand Up @@ -237,12 +255,18 @@ async def jobs(request: Request):
hpc_client_conf = HpcClientConfigs()
hpc_client = HpcClient(configs=hpc_client_conf)
hpc_partition = os.getenv("HPC_PARTITION")
hpc_qos = os.getenv("HPC_QOS")
response = hpc_client.get_jobs()
if response.status_code == 200:
slurm_jobs = [
HpcJobStatusResponse.parse_obj(job_json)
for job_json in response.json()["jobs"]
if job_json["partition"] == hpc_partition
and job_json["user_name"] == os.getenv("HPC_USERNAME")
and (
hpc_qos is None
or (hpc_qos == "production" and job_json["qos"] == hpc_qos)
)
]
job_status_list = [
JobStatus.from_hpc_job_status(slurm_job).jinja_dict
Expand Down
66 changes: 62 additions & 4 deletions src/aind_data_transfer_service/templates/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@
<meta charset="UTF-8">
<title>{% block title %} {% endblock %} AIND Data Transfer Service</title>
<style>
table {
fieldset {
display:inline
}
table {
font-family: arial, sans-serif;
border-collapse: collapse;
width: 75%;
Expand All @@ -22,9 +25,30 @@
<nav>
<a href="/">Submit Jobs</a> |
<a href="/jobs">Job Status</a> |
<a href= "{{ upload_template_link }}" >Upload Template</a>
<a href= "{{ upload_template_link }}" >Job Submit Template</a>
</nav>
<h2>Submit Jobs</h2>
<div>
<fieldset>
<legend>Mail Notifications (optional)</legend><br>
<div>
<label for="email" title="Optionally provide an allen institute email address to receive upload job status notifications">Allen Institute email:</label>
<input type="email" id="email" pattern=".+@alleninstitute\.org" size="30" placeholder="@alleninstitute.org" /><br><br>
</div>
<div>
<input type="checkbox" id="begin" name="begin" />
<label for="begin">BEGIN</label> |
<input type="checkbox" id="end" name="end" />
<label for="end">END</label> |
<input type="checkbox" id="fail" name="fail" checked />
<label for="fail">FAIL</label> |
<input type="checkbox" id="requeue" name="requeue" />
<label for="requeue">REQUEUE</label> |
<input type="checkbox" id="all" name="all" />
<label for="all">ALL</label>
</div>
</fieldset>
</div><br><br>
<form id="preview_form" method="post" enctype="multipart/form-data">
<label for="file">Please select a CSV file:</label>
<input type="file" id="file" name="file"><br><br>
Expand Down Expand Up @@ -104,10 +128,44 @@ <h2>Submit Jobs</h2>
});
submitJobs = function() {
if(jobs.length > 0 && parsing_errors.length == 0){
let mail_user = $("#email").val();
let hpc_settings = {};
if (mail_user !== "" && mail_user !== undefined) {
let mail_type = [];
hpc_settings["mail_user"] = mail_user;
hpc_settings["comment"] = mail_user;
if ($("#all").is(":checked")) {
mail_type = ["ALL"];
} else {
if ($("#begin").is(":checked")) {
mail_type.push("BEGIN");
};
if ($("#end").is(":checked")) {
mail_type.push("END");
};
if ($("#fail").is(":checked")) {
mail_type.push("FAIL");
};
if ($("#requeue").is(":checked")) {
mail_type.push("REQUEUE");
};
};
if (mail_type.length == 0) {
hpc_settings["mail_type"] = "NONE";
} else {
hpc_settings["mail_type"] = mail_type.join()
};
};
let hpc_jobs = []
for (row = 0; row < jobs.length; row++) {
let job = jobs[row];
hpc_job = {"upload_job_settings": job, "hpc_settings": JSON.stringify(hpc_settings), "script": ""};
hpc_jobs.push(hpc_job);
};
$.ajax({
url: "/api/submit_basic_jobs",
url: "/api/submit_hpc_jobs",
type: "POST",
data: JSON.stringify({"jobs": jobs}),
data: JSON.stringify({"jobs": hpc_jobs}),
contentType: 'application/json; charset=utf-8',
beforeSend: function() {
$("#message").html("Submitting jobs. Please don't refresh or re-submit...");
Expand Down
24 changes: 19 additions & 5 deletions src/aind_data_transfer_service/templates/job_status.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<html>
<head>
<meta charset="UTF-8">
<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.3/moment.min.js"></script>
<title>{% block title %} {% endblock %} AIND Data Transfer Service Jobs</title>
<style>
table {
Expand All @@ -25,9 +26,9 @@
<nav>
<a href="/">Submit Jobs</a> |
<a href="/jobs">Job Status</a> |
<a href= "{{ upload_template_link }}" >Upload Template</a>
<a href= "{{ upload_template_link }}" >Job Submit Template</a>
</nav>
<h2>Jobs Currently Running: {{num_of_jobs}}</h2>
<h2>Jobs Submitted: {{num_of_jobs}}</h2>
<table>
<tr>
<th>Asset Name</th>
Expand All @@ -36,17 +37,30 @@ <h2>Jobs Currently Running: {{num_of_jobs}}</h2>
<th>Submit Time</th>
<th>Start Time</th>
<th>End time</th>
<th>Comment</th>
</tr>
{% for job_status in job_status_list %}
<tr>
<td>{{job_status.name}}</td>
<td>{{job_status.job_id}}</td>
<td>{{job_status.job_state}}</td>
<td>{{job_status.submit_time}}</td>
<td>{{job_status.start_time}}</td>
<td>{{job_status.end_time}}</td>
<td class="datetime_to_be_adjusted">{{job_status.submit_time}}</td>
<td class="datetime_to_be_adjusted">{{job_status.start_time}}</td>
<td class="datetime_to_be_adjusted">{{job_status.end_time}}</td>
<td>{{job_status.comment}}</td>
</tr>
{% endfor %}
</table>
<script>
window.onload = function() {
document.querySelectorAll(".datetime_to_be_adjusted").forEach(function(el){
if (el.innerHTML !== "") {
var utcTime = moment.utc(el.innerText); // This is the time in UTC
utcTime.local(); // Switch to using the browser's local timezone
el.innerText = utcTime.format('YYYY-MM-DD h:mm:ss a'); // Write the local time back to the element
};
});
}
</script>
</body>
</html>
Loading

0 comments on commit 60c0dd9

Please sign in to comment.