Skip to content

Commit

Permalink
Merge pull request #33 from lehors/mof-1.1
Browse files Browse the repository at this point in the history
Implement MOF 1.1 relaxed requirements
  • Loading branch information
lehors authored Oct 21, 2024
2 parents 69e46c1 + f0cd390 commit d12ce68
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 18 deletions.
10 changes: 5 additions & 5 deletions web/modules/mof/config/install/mof.settings.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ components:
name: Model parameters (Final)
description: "Trained model parameters, weights and biases"
tooltip: "The license used for the final model parameters, 'License not specified', or 'Component not included'."
content_type: data
content_type: [data, code]
class: 3
weight: 10
required: true
Expand Down Expand Up @@ -94,7 +94,7 @@ components:
name: Evaluation results
description: "The results from evaluating the model"
tooltip: "The license used for evaluation results, 'License not specified', or 'Component not included'."
content_type: document
content_type: [document, code]
class: 3
weight: 101
required: true
Expand All @@ -115,7 +115,7 @@ components:
name: Model card
description: "Model details including performance metrics, intended use, and limitations"
tooltip: "The license used for the model card, 'License not specified', or 'Component not included'."
content_type: document
content_type: [document, code]
class: 3
weight: 70
required: true
Expand All @@ -124,7 +124,7 @@ components:
name: Data card
description: "Documentation for datasets including source, characteristics, and preprocessing details"
tooltip: "The license used for the data card, 'License not specified', or 'Component not included'."
content_type: document
content_type: [document, code]
class: 3
weight: 80
required: true
Expand All @@ -133,7 +133,7 @@ components:
name: Technical report
description: "Technical report detailing capabilities and usage instructions for the model"
tooltip: "The license used for the technical report, 'License not specified', or 'Component not included'."
content_type: document
content_type: [document, code]
class: 3
weight: 90
required: true
Expand Down
70 changes: 58 additions & 12 deletions web/modules/mof/src/ModelEvaluator.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ final class ModelEvaluator implements ModelEvaluatorInterface {

use StringTranslationTrait;

// If any of these component IDs have a
// open source license then the model qualifies for class 3.
const CLASS_3_CIDS = [10, 11, 12, 13, 14];

/** @var \Drupal\mof\Entity\Model. */
private $model;

Expand Down Expand Up @@ -142,17 +146,13 @@ public function generateBadge(bool $mini = FALSE): array {
}

$evals = $this->evaluate();
$qualified = FALSE;
$in_progress = FALSE;
$qualified = $in_progress = FALSE;

for ($i = 3, $j = 3; $i >= 1; $i--, $j--) {
$progress = $this->getProgress($i);

if ($evals[$i]['conditional'] === TRUE) {
$status = $this->t('Conditional');
$text_color = '#fff';
$background_color = '#4c1';
}
else if ($progress === 100.00) {
// under MOF 1.1 Conditional is a Pass
if ($progress === 100.00 || $evals[$i]['conditional'] === TRUE) {
$status = $this->t('Qualified');
$text_color = '#fff';
$background_color = '#4c1';
Expand Down Expand Up @@ -211,12 +211,58 @@ public function getTotalProgress(): float {
}

/**
* Class 3 has a conditional pass if these components have an open source license.
* Class 3 has a conditional pass if these components have an open source license but we will
* inform the user that a type-appropriate license should be used.
*
* - `Model parameters (Final)` (10)
* - `Technical report` (11)
* - `Evaluation results` (12)
* - `Model card` (13)
* - `Data card` (14)
*
* @return array
* An array of translatable strings.
*/
public function getConditionalMessage(): array {
$messages = [
$this->t('This model has an open source license on the following components, it should be using a type-appropriate license:'),
];

$component_messages = [
10 => $this->t('Model parameters (Final) of type data.'),
11 => $this->t('Technical report of type documentation.'),
12 => $this->t('Evaluation results of type documentation.'),
13 => $this->t('Model card of type documentation.'),
14 => $this->t('Data card of type documentation.'),
];

foreach (self::CLASS_3_CIDS as $cid) {
if (isset($component_messages[$cid]) && $this->isOpenSourceLicense($cid)) {
$messages[] = $component_messages[$cid];
}
}

return $messages;
}

/**
* Determine if a model has a conditional pass.
* @return bool
*/
private function hasConditionalPass(): bool {
$pass = array_filter(self::CLASS_3_CIDS, fn($cid) => $this->isOpenSourceLicense($cid));
return !empty($pass);
}

/**
* Determine if a component is using an open source license.
*
* @param int $cid Component ID.
* @return bool
*/
private function isOpenSourceLicense(int $cid): bool {
$licenses = $this->model->getLicenses();
return isset($licenses[10]) && $this->licenseHandler->isOpenSource($licenses[10]['license']);
return isset($licenses[$cid]) && $this->licenseHandler->isOpenSource($licenses[$cid]['license']);
}

/**
Expand Down Expand Up @@ -331,9 +377,9 @@ private function getInvalid(array $required): array {
if (in_array($cid, $required)) {
$cid = (int)$cid;

// Special case for component 10: Model parameters (Final).
// Special case for class 3 components.
// Conditional passes are valid.
if ($cid === 10 && $this->hasConditionalPass()) {
if (in_array($cid, self::CLASS_3_CIDS) && $this->hasConditionalPass()) {
continue;
}

Expand Down
8 changes: 7 additions & 1 deletion web/modules/mof/src/ModelViewBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,13 @@ public function build(array $build) {
}

if ($evaluation[3]['conditional']) {
$this->messenger->addMessage($this->t('This model conditionally meets Class III because it has an open source license for Model Parameters (Final)'));
$list = ['#theme' => 'item_list', '#items' => []];

$message = $this->modelEvaluator->getConditionalMessage();
$this->messenger->addMessage(array_shift($message));

$list['#items'] = $message;
$this->messenger->addMessage($list);
}

if ($this->session->get('model_evaluation') === TRUE) {
Expand Down

0 comments on commit d12ce68

Please sign in to comment.