Skip to content

Commit

Permalink
feat(LAB-3105): add tests on llm_v1 format
Browse files Browse the repository at this point in the history
  • Loading branch information
@FannyGaudin
FannyGaudin committed Oct 18, 2024
1 parent eb4e85b commit aeccef6
Show file tree
Hide file tree
Showing 11 changed files with 539 additions and 160 deletions.
1 change: 1 addition & 0 deletions src/kili/entrypoints/cli/project/export.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ def export_labels(
asset_filter_kwargs=None,
normalized_coordinates=normalized_coordinates,
label_type_in=None,
include_sent_back_labels=None,
)
except NoCompatibleJobError as excp:
print(str(excp))
11 changes: 0 additions & 11 deletions src/kili/llm/services/export/dynamic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,6 @@
"modelName",
]

ASSET_NEEDED_FIELDS = [
"assetProjectModels.id",
"assetProjectModels.projectModelId",
"assetProjectModels.configuration",
"content",
"externalId",
"jsonMetadata",
*(f"labels.{field}" for field in LABELS_NEEDED_FIELDS),
"status",
]


class LLMDynamicExporter:
"""Handle exports of LLM_RLHF projects."""
Expand Down
2 changes: 1 addition & 1 deletion src/kili/presentation/client/label.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1138,7 +1138,7 @@ def export_labels(
asset_filter_kwargs: Optional[Dict[str, object]] = None,
normalized_coordinates: Optional[bool] = None,
label_type_in: Optional[List[str]] = None,
include_sent_back_labels: Optional[bool] = True,
include_sent_back_labels: Optional[bool] = None,
) -> Optional[List[Dict[str, Union[List[str], str]]]]:
# pylint: disable=line-too-long
"""Export the project labels with the requested format into the requested output path.
Expand Down
10 changes: 8 additions & 2 deletions src/kili/services/export/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,17 @@ def export_labels( # pylint: disable=too-many-arguments, too-many-locals
asset_filter_kwargs: Optional[Dict[str, object]],
normalized_coordinates: Optional[bool],
label_type_in: Optional[List[str]],
include_sent_back_labels: Optional[bool] = True,
include_sent_back_labels: Optional[bool],
) -> Optional[List[Dict[str, Union[List[str], str]]]]:
"""Export the selected assets into the required format, and save it into a file archive."""
kili.kili_api_gateway.get_project(project_id, ["id"])

include_sent_back_labels = (
include_sent_back_labels
if include_sent_back_labels is not None
else (label_format != "llm_v1")
)

export_params = ExportParams(
assets_ids=asset_ids,
project_id=project_id,
Expand All @@ -57,7 +63,7 @@ def export_labels( # pylint: disable=too-many-arguments, too-many-locals
asset_filter_kwargs=asset_filter_kwargs,
normalized_coordinates=normalized_coordinates,
label_type_in=label_type_in,
include_sent_back_labels=include_sent_back_labels if label_format != "llm_v1" else False,
include_sent_back_labels=include_sent_back_labels,
)

logger = get_logger(log_level)
Expand Down
5 changes: 3 additions & 2 deletions src/kili/services/export/format/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,8 +289,9 @@ def preprocess_assets(self, assets: List[Dict]) -> List[Dict]:
lambda label: label["isSentBackToQueue"] is False, labels_of_asset
)
)
asset["labels"] = labels_of_asset
assets_in_format.append(asset)
if len(labels_of_asset) > 0:
asset["labels"] = labels_of_asset
assets_in_format.append(asset)
if "latestLabel" in asset:
label = asset["latestLabel"]
if label is not None:
Expand Down
9 changes: 7 additions & 2 deletions src/kili/services/export/format/llm/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def process_and_save(
def process(self, assets: List[Dict]) -> List[Dict[str, Union[List[str], str]]]:
"""LLM specific process."""
warnings.warn(
"Exporting llm labels with `kili.export` is deprecated."
"Exporting llm labels with `kili.export_labels` is deprecated."
" Please use `kili.llm.export` instead.",
DeprecationWarning,
stacklevel=2,
Expand Down Expand Up @@ -97,6 +97,8 @@ def _process_llm_dynamic_v1(self, assets: List[Dict]) -> List[Dict[str, Union[Li

def _process_llm_v1(self, assets: List[Dict]) -> List[Dict[str, Union[List[str], str]]]:
result = []
if len(assets) == 0:
return result
for asset in assets:
result.append(
{
Expand Down Expand Up @@ -263,7 +265,10 @@ def _format_raw_data(
"id": _safe_pop(chat_items_ids),
"chat_id": chat_id,
"model": models[index_completion]
if (index == len(prompts) - 1 or all_model_keys)
if (
(index == len(prompts) - 1 or all_model_keys)
and len(models) > index_completion
)
else None,
}
)
Expand Down
145 changes: 3 additions & 142 deletions tests/unit/llm/services/export/test_static.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,148 +2,9 @@
import tempfile

from kili.llm.presentation.client.llm import LlmClientMethods

mock_json_interface = {
"jobs": {
"CLASSIFICATION_JOB": {
"content": {
"categories": {
"A_BETTER_THAN_B": {
"children": [],
"name": "A better than B",
"id": "category1",
},
"B_BETTER_THAN_A": {
"children": [],
"name": "B better than A",
"id": "category2",
},
"TIE": {"children": [], "name": "Tie", "id": "category3"},
},
"input": "radio",
},
"instruction": "Compare",
"mlTask": "CLASSIFICATION",
"required": 0,
"isChild": False,
"isNew": False,
},
"TRANSCRIPTION_JOB": {
"content": {"input": "markdown"},
"instruction": "",
"mlTask": "TRANSCRIPTION",
"required": 0,
"isChild": False,
"isNew": False,
},
}
}

mock_fetch_assets = [
{
"labels": [
{
"author": {
"id": "user-1",
"email": "[email protected]",
"firstname": "Test",
"lastname": "Admin",
},
"jsonResponse": {
"CLASSIFICATION_JOB": {"categories": [{"name": "A_BETTER_THAN_B"}]}
},
"createdAt": "2024-08-05T13:03:00.051Z",
"isLatestLabelForUser": True,
"isSentBackToQueue": False,
"labelType": "DEFAULT",
"modelName": None,
}
],
"content": "https://storage.googleapis.com/label-public-staging/demo-projects/LLM/01.json",
"externalId": "asset#0",
"jsonMetadata": {},
"status": "LABELED",
},
{
"labels": [
{
"author": {
"id": "user-1",
"email": "[email protected]",
"firstname": "Test",
"lastname": "Admin",
},
"jsonResponse": {
"CLASSIFICATION_JOB": {"categories": [{"name": "B_BETTER_THAN_A"}]}
},
"createdAt": "2024-08-05T13:03:03.061Z",
"isLatestLabelForUser": True,
"isSentBackToQueue": False,
"labelType": "DEFAULT",
"modelName": None,
}
],
"content": "https://storage.googleapis.com/label-public-staging/demo-projects/LLM/02.json",
"externalId": "asset#1",
"jsonMetadata": {},
"status": "LABELED",
},
{
"labels": [
{
"author": {
"id": "user-1",
"email": "[email protected]",
"firstname": "Test",
"lastname": "Admin",
},
"jsonResponse": {
"CLASSIFICATION_JOB": {"categories": [{"name": "TIE"}]},
"TRANSCRIPTION_JOB": {"text": "There is only some formatting changes\n"},
},
"createdAt": "2024-08-05T13:03:16.028Z",
"isLatestLabelForUser": True,
"isSentBackToQueue": True,
"labelType": "DEFAULT",
"modelName": None,
}
],
"content": "https://storage.googleapis.com/label-public-staging/demo-projects/LLM/03.json",
"externalId": "asset#2",
"jsonMetadata": {},
"status": "LABELED",
},
]

mock_raw_asset_content = """{
"prompts": [
{
"prompt": "BLABLABLA",
"completions": [
{
"content": "response A1"
},
{
"content": "response B1"
}
]
},
{
"prompt": "BLIBLIBLI",
"completions": [
{
"content": "response A2"
},
{
"content": "response B2"
}
]
}
],
"type": "markdown",
"version": "0.1"
}
"""
from tests.unit.services.export.fakes.llm_json_interface import mock_json_interface
from tests.unit.services.export.fakes.llm_project_assets import mock_fetch_assets
from tests.unit.services.export.fakes.llm_raw_asset_content import mock_raw_asset_content

expected_export = [
{
Expand Down
35 changes: 35 additions & 0 deletions tests/unit/services/export/fakes/llm_json_interface.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
mock_json_interface = {
"jobs": {
"CLASSIFICATION_JOB": {
"content": {
"categories": {
"A_BETTER_THAN_B": {
"children": [],
"name": "A better than B",
"id": "category1",
},
"B_BETTER_THAN_A": {
"children": [],
"name": "B better than A",
"id": "category2",
},
"TIE": {"children": [], "name": "Tie", "id": "category3"},
},
"input": "radio",
},
"instruction": "Compare",
"mlTask": "CLASSIFICATION",
"required": 0,
"isChild": False,
"isNew": False,
},
"TRANSCRIPTION_JOB": {
"content": {"input": "markdown"},
"instruction": "",
"mlTask": "TRANSCRIPTION",
"required": 0,
"isChild": False,
"isNew": False,
},
}
}
75 changes: 75 additions & 0 deletions tests/unit/services/export/fakes/llm_project_assets.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
mock_fetch_assets = [
{
"labels": [
{
"author": {
"id": "user-1",
"email": "[email protected]",
"firstname": "Test",
"lastname": "Admin",
},
"jsonResponse": {
"CLASSIFICATION_JOB": {"categories": [{"name": "A_BETTER_THAN_B"}]}
},
"createdAt": "2024-08-05T13:03:00.051Z",
"isLatestLabelForUser": True,
"isSentBackToQueue": False,
"labelType": "DEFAULT",
"modelName": None,
}
],
"content": "https://storage.googleapis.com/label-public-staging/demo-projects/LLM/01.json",
"externalId": "asset#0",
"jsonMetadata": {},
"status": "LABELED",
},
{
"labels": [
{
"author": {
"id": "user-1",
"email": "[email protected]",
"firstname": "Test",
"lastname": "Admin",
},
"jsonResponse": {
"CLASSIFICATION_JOB": {"categories": [{"name": "B_BETTER_THAN_A"}]}
},
"createdAt": "2024-08-05T13:03:03.061Z",
"isLatestLabelForUser": True,
"isSentBackToQueue": False,
"labelType": "DEFAULT",
"modelName": None,
}
],
"content": "https://storage.googleapis.com/label-public-staging/demo-projects/LLM/02.json",
"externalId": "asset#1",
"jsonMetadata": {},
"status": "LABELED",
},
{
"labels": [
{
"author": {
"id": "user-1",
"email": "[email protected]",
"firstname": "Test",
"lastname": "Admin",
},
"jsonResponse": {
"CLASSIFICATION_JOB": {"categories": [{"name": "TIE"}]},
"TRANSCRIPTION_JOB": {"text": "There is only some formatting changes\n"},
},
"createdAt": "2024-08-05T13:03:16.028Z",
"isLatestLabelForUser": True,
"isSentBackToQueue": True,
"labelType": "DEFAULT",
"modelName": None,
}
],
"content": "https://storage.googleapis.com/label-public-staging/demo-projects/LLM/03.json",
"externalId": "asset#2",
"jsonMetadata": {},
"status": "LABELED",
},
]
29 changes: 29 additions & 0 deletions tests/unit/services/export/fakes/llm_raw_asset_content.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
mock_raw_asset_content = """{
"prompts": [
{
"prompt": "BLABLABLA",
"completions": [
{
"content": "response A1"
},
{
"content": "response B1"
}
]
},
{
"prompt": "BLIBLIBLI",
"completions": [
{
"content": "response A2"
},
{
"content": "response B2"
}
]
}
],
"type": "markdown",
"version": "0.1"
}
"""
Loading

0 comments on commit aeccef6

Please sign in to comment.