Skip to content

Commit

Permalink
fix: finish job status (#863)
Browse files Browse the repository at this point in the history
  • Loading branch information
@SunnyCapt @khyurri
SunnyCapt authored and khyurri committed Jun 4, 2024
1 parent 0d26672 commit c23c52b
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 5 deletions.
16 changes: 13 additions & 3 deletions jobs/jobs/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,11 +298,21 @@ async def change_job(
setattr(new_job_params, field, value)

is_job_changed = False
if job_to_change.type == schemas.JobType.ExtractionWithAnnotationJob:

if (
job_to_change.type == schemas.JobType.ExtractionWithAnnotationJob
and new_job_params.status == schemas.Status.finished
):
new_job_params_dict = new_job_params.dict(
exclude_none=True, exclude_unset=True, exclude_defaults=True
)
if (
job_to_change.mode == schemas.JobMode.Automatic
and new_job_params.status == schemas.Status.finished
job_to_change.status == schemas.Status.in_progress
and len(new_job_params_dict) == 1
):
# detect finishing all validation tasks from annotation
db_service.change_job(db, job_to_change, new_job_params)
elif job_to_change.mode == schemas.JobMode.Automatic:
new_job_params.mode = schemas.JobMode.Manual
new_job_params.status = schemas.Status.ready_for_annotation
if job_to_change.start_manual_job_automatically:
Expand Down
4 changes: 2 additions & 2 deletions jobs/jobs/pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ class PipelineFile:
input: PipelineFileInput
input_path: str
pages: List[int]
revision: str = None
revision: Optional[str] = None
output_path: Optional[str] = None
s3_signed_url: Optional[str] = None
annotation_id: Optional[str] = None
file_id: Optional[str] = None


@dataclass
Expand Down
1 change: 1 addition & 0 deletions jobs/jobs/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ def generate_file_data(
"file": f"{file_data['path']}",
"bucket": file_data["bucket"],
"pages": pages,
"file_id": file_data["id"],
"output_path": f"runs/{job_id}/{file_data['id']}",
}
if batch_id:
Expand Down
32 changes: 32 additions & 0 deletions jobs/tests/test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -678,95 +678,111 @@ def test_convert_files_data_for_inference_without_output_bucket(
{
"bucket": "bucket11",
"file": "files/1/1.pdf",
"file_id": 1,
"output_path": "runs/11/1/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 1,
"file": "files/1/1.pdf",
"output_path": "runs/11/1/2",
"pages": [6, 7, 8, 9, 10],
},
{
"bucket": "bucket11",
"file_id": 2,
"file": "files/2/2.pdf",
"output_path": "runs/11/2",
"pages": [1, 2],
},
{
"bucket": "bucket11",
"file_id": 3,
"file": "files/3/3.pdf",
"output_path": "runs/11/3/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 3,
"file": "files/3/3.pdf",
"output_path": "runs/11/3/2",
"pages": [6],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/2",
"pages": [6, 7, 8, 9, 10],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/3",
"pages": [11, 12, 13, 14, 15],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/4",
"pages": [16, 17, 18, 19, 20],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/5",
"pages": [21, 22, 23, 24, 25],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/6",
"pages": [26, 27, 28, 29, 30],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_path": "runs/11/4/7",
"pages": [31, 32],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_path": "runs/11/5/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_path": "runs/11/5/2",
"pages": [6, 7, 8, 9, 10],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_path": "runs/11/5/3",
"pages": [11, 12, 13, 14, 15],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_path": "runs/11/5/4",
"pages": [16, 17, 18, 19, 20],
Expand All @@ -786,111 +802,127 @@ def test_convert_files_data_for_inference_with_completley_another_output_bucket(
expected_result = [
{
"bucket": "bucket11",
"file_id": 1,
"file": "files/1/1.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/1/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 1,
"file": "files/1/1.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/1/2",
"pages": [6, 7, 8, 9, 10],
},
{
"bucket": "bucket11",
"file_id": 2,
"file": "files/2/2.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/2",
"pages": [1, 2],
},
{
"bucket": "bucket11",
"file_id": 3,
"file": "files/3/3.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/3/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 3,
"file": "files/3/3.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/3/2",
"pages": [6],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/2",
"pages": [6, 7, 8, 9, 10],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/3",
"pages": [11, 12, 13, 14, 15],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/4",
"pages": [16, 17, 18, 19, 20],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/5",
"pages": [21, 22, 23, 24, 25],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/6",
"pages": [26, 27, 28, 29, 30],
},
{
"bucket": "bucket11",
"file_id": 4,
"file": "files/4/4.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/4/7",
"pages": [31, 32],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/5/1",
"pages": [1, 2, 3, 4, 5],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/5/2",
"pages": [6, 7, 8, 9, 10],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/5/3",
"pages": [11, 12, 13, 14, 15],
},
{
"bucket": "bucket11",
"file_id": 5,
"file": "files/5/5.pdf",
"output_bucket": "another_bucket",
"output_path": "runs/11/5/4",
Expand Down

0 comments on commit c23c52b

Please sign in to comment.