Skip to content

Commit

Permalink
Model can be selected for each question
Browse files Browse the repository at this point in the history
  • Loading branch information
marcusgreen committed May 16, 2024
1 parent f875a5b commit f9e78f6
Show file tree
Hide file tree
Showing 13 changed files with 89 additions and 17 deletions.
2 changes: 1 addition & 1 deletion classes/external.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
* External
*
* @package qtype_aitext
* @author Justin Hunt - poodll.com
* @copyright Justin Hunt - poodll.com
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
Expand Down
3 changes: 2 additions & 1 deletion db/install.xml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
<FIELD NAME="aiprompt" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="The prompt to be sent to the LLM for getting an answer/feedback"/>
<FIELD NAME="markscheme" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="Instructions on how responses are to be marked."/>
<FIELD NAME="sampleanswer" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="A sample answer for testing the prompt and AI response"/>
</FIELDS>
<FIELD NAME="model" TYPE="char" LENGTH="60" NOTNULL="false" SEQUENCE="false" COMMENT="The model in the LLM, e.g. gpt-4 or llama3"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
<KEY NAME="questionid" TYPE="foreign-unique" FIELDS="questionid" REFTABLE="question" REFFIELDS="id"/>
Expand Down
16 changes: 16 additions & 0 deletions db/upgrade.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
* AI Text question type upgrade code.
*
* @package qtype_aitext
* @copyright Marcus Green 2024
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/

Expand Down Expand Up @@ -46,5 +47,20 @@ function xmldb_qtype_aitext_upgrade($oldversion) {

}

if ($oldversion < 2024051100) {

// Define field model to be added to qtype_aitext.
$table = new xmldb_table('qtype_aitext');
$field = new xmldb_field('model', XMLDB_TYPE_CHAR, '60', null, null, null, null, 'sampleanswer');

// Conditionally launch add field model.
if (!$dbman->field_exists($table, $field)) {
$dbman->add_field($table, $field);
}

// Aitext savepoint reached.
upgrade_plugin_savepoint(true, 2024051100, 'qtype', 'aitext');
}

return true;
}
11 changes: 10 additions & 1 deletion edit_aitext_form.php
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,16 @@ protected function definition_inner($mform) {
$mform->setType('markscheme', PARAM_RAW);
$mform->setDefault('markscheme', get_config('qtype_aitext', 'defaultmarksscheme'));
$mform->addHelpButton('markscheme', 'markscheme', 'qtype_aitext');
$models = explode(",", get_config('tool_aiconnect', 'model'));
if (count($models) > 1 ) {
$models = array_combine($models, $models);
$mform->addElement('select', 'model', get_string('model','qtype_aitext'), $models);

} else {
$mform->addElement('hidden', 'model', $models[0]);
}
$mform->setType('model', PARAM_RAW);
// The question_edit_form that this class extends expects a general feedback field.
$mform->addElement('html', '<div class="hidden">');
$mform->addElement('editor', 'generalfeedback', get_string('generalfeedback', 'question')
, ['rows' => 10], $this->editoroptions);
Expand All @@ -67,12 +76,12 @@ protected function definition_inner($mform) {
$mform->addElement('textarea', 'sampleanswer', get_string('sampleanswer', 'qtype_aitext'),
['maxlen' => 50, 'rows' => 6, 'size' => 30]);
$mform->setType('sampleanswer', PARAM_RAW);
$mform->setDefault('sampleanswer', '');
$mform->addHelpButton('sampleanswer', 'sampleanswer', 'qtype_aitext');
$mform->addElement('static', 'sampleanswereval', '', '<a class="qtype_aitext_sampleanswerbtn btn btn-secondary"
id="id_sampleanswerbtn">'
. get_string('sampleanswerevaluate', 'qtype_aitext') . '</a>' .
'<div class="qtype_aitext_sampleanswereval" id="id_sampleanswereval"></div>');

$mform->addElement('header', 'responseoptions', get_string('responseoptions', 'qtype_aitext'));
$mform->setExpanded('responseoptions');

Expand Down
1 change: 1 addition & 0 deletions lang/en/qtype_aitext.php
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
$string['minwordlimit'] = 'Minimum word limit';
$string['minwordlimit_help'] = 'If the response requires that students enter text, this is the minimum number of words that each student will be allowed to submit.';
$string['minwordlimitboundary'] = 'This question requires a response of at least {$a->limit} words and you are attempting to submit {$a->count} words. Please expand your response and try again.';
$string['model'] = 'Model';
$string['nlines'] = '{$a} lines';
$string['prompt'] = 'Prompt';
$string['prompt_setting'] = 'Wrapper text for the prompt set to the AI System, [responsetext] is whatever the student typed as an answer. The ai prompt value from the question will be appended to this';
Expand Down
18 changes: 16 additions & 2 deletions question.php
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ class qtype_aitext_question extends question_graded_automatically_with_countback
/** @var int indicates whether the maximum number of words required */
public $maxwordlimit;

/**
* LLM Model, will vary between AI systems, e.g. gpt4 or llama3
* @var stream_set_blocking
*/
public $model;


/**
* used in the question editing interface
Expand Down Expand Up @@ -146,7 +152,7 @@ public function grade_response(array $response) : array {
$grade = [0 => 0, question_state::$needsgrading];
return $grade;
}
$ai = new ai\ai();
$ai = new ai\ai($this->model);
if (is_array($response)) {
$fullaiprompt = $this->build_full_ai_prompt($response['answer'], $this->aiprompt,
$this->defaultmark, $this->markscheme);
Expand All @@ -173,8 +179,16 @@ public function grade_response(array $response) : array {
return $grade;
}

/**
* Used by prompttester in the editing form
*
* @param string $response
* @param string $aiprompt
* @param number $defaultmark
* @param string $markscheme
* @return void
*/
public function build_full_ai_prompt($response, $aiprompt, $defaultmark, $markscheme) {

$responsetext = strip_tags($response);
$responsetext = '[['.$responsetext.']]';
$prompt = get_config('qtype_aitext', 'prompt');
Expand Down
7 changes: 6 additions & 1 deletion questiontype.php
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ public function save_defaults_for_new_questions(stdClass $fromform): void {
$this->set_default_value('responseformat', $fromform->responseformat);
$this->set_default_value('responsefieldlines', $fromform->responsefieldlines);
$this->set_default_value('markscheme', $fromform->markscheme);
$this->set_default_value('sampleanswer', $fromform->sampleanswer);

}
/**
* Write the question data from the editing form to the database
Expand All @@ -97,6 +99,7 @@ public function save_question_options($formdata) {
$options->aiprompt = $formdata->aiprompt;
$options->markscheme = $formdata->markscheme;
$options->sampleanswer = $formdata->sampleanswer;
$options->model = trim($formdata->model);
$options->responseformat = $formdata->responseformat;
$options->responsefieldlines = $formdata->responsefieldlines;
$options->minwordlimit = isset($formdata->minwordenabled) ? $formdata->minwordlimit : null;
Expand Down Expand Up @@ -144,6 +147,7 @@ protected function initialise_question_instance(question_definition $question, $
$question->aiprompt = $questiondata->options->aiprompt;
$question->markscheme = $questiondata->options->markscheme;
$question->sampleanswer = $questiondata->options->sampleanswer;
$question->model = $questiondata->options->model;
}
/**
* Delete a question from the database
Expand Down Expand Up @@ -249,7 +253,8 @@ public function extra_question_fields() {
'responsetemplateformat',
'aiprompt',
'markscheme',
'sampleanswer'
'sampleanswer',
'model',
];
}
/**
Expand Down
3 changes: 2 additions & 1 deletion renderer.php
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,10 @@ public function feedback(question_attempt $qa, question_display_options $options
// Get data written in the question.php grade_response method.
// This probably should be retrieved by an api call.
$comment = $qa->get_current_manual_comment();
xdebug_break();
if ($this->page->pagetype == 'question-bank-previewquestion-preview') {
$this->page->requires->js_call_amd('qtype_aitext/showprompt', 'init', []);
if ($comment[0] > '') {
$this->page->requires->js_call_amd('qtype_aitext/showprompt', 'init', []);
$prompt = $qa->get_last_qt_var('-aiprompt');
$showprompt = '<br/><button id=showprompt class="rounded">'. get_string('showprompt', 'qtype_aitext').'</button>';
$showprompt .= '<div id="fullprompt" class="hidden">'.$prompt .'</div>';
Expand Down
18 changes: 15 additions & 3 deletions tests/behat/add.feature
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Feature: Test creating an AIText question
I need to be able to create an aitext question

Background:

Given the following "users" exist:
| username |
| teacher |
Expand All @@ -14,18 +15,22 @@ Feature: Test creating an AIText question
And the following "course enrolments" exist:
| user | course | role |
| teacher | C1 | editingteacher |

And the following config values are set as admin:
| model | gpt-4,gpt-4o | tool_aiconnect |
@javascript
Scenario: Create an AI text question with Response format set to 'HTML editor'
When I am on the "Course 1" "core_question > course question bank" page logged in as teacher
# id_sampleanswer because it is collapsed when the form is opened for editing.
And I add a "AI Text" question filling the form with:
| Question name | aitext-001 |
| Question text | Write an aitext with 500 words. |
| General feedback | This is general feedback |
| Response format | HTML editor |
| AI Prompt | Evaluate this |
| Mark scheme | Give one mark if correct |
| id_sampleanswer | sample answer |

Then I should see "aitext-001"
Then I should see "aitext-001"

Scenario: Create an AI Text question with Response format set to 'HTML editor with the file picker'
When I am on the "Course 1" "core_question > course question bank" page logged in as teacher
Expand All @@ -35,9 +40,12 @@ Feature: Test creating an AIText question
| General feedback | This is general feedback |
| AI Prompt | Evaluate this |
| Mark scheme | Give one mark if correct |
| id_sampleanswer | sample answer |
| Model | gpt-4|

Then I should see "aitext-002"


Then I should see "aitext-002"
@javascript
Scenario: Create an AI Text question for testing some default options
When I am on the "Course 1" "core_question > course question bank" page logged in as teacher
Expand All @@ -46,6 +54,10 @@ Feature: Test creating an AIText question
| Question text | Write an aitext with 500 words. |
| General feedback | This is general feedback |
| id_responsefieldlines | 15 |
| id_sampleanswer | sample answer |
| Model | gpt-4o|


Then I should see "aitext-003"
# Checking that the next new question form displays user preferences settings.
And I press "Create a new question ..."
Expand Down
6 changes: 6 additions & 0 deletions tests/behat/backup_and_restore.feature
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ Feature: Test duplicating a quiz containing an aitext question
| aitext-001 | 1 |
| aitext-002 | 1 |

# Without this it will show the pending progress bar and the back
# to course button introduced in Moodle 4.3
# https://docs.moodle.org/403/en/Course_backup#Asynchronous_course_backups
And the following config values are set as admin:
| enableasyncbackup | 0 |

@javascript
Scenario: Backup and restore a course containing 3 aitext questions
When I am on the "Course 1" course page logged in as admin
Expand Down
9 changes: 5 additions & 4 deletions tests/behat/preview.feature
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ Feature: Preview aitext questions

Background:
Given the following "users" exist:
| username |
| teacher |
| username | firstname | lastname | email |
| teacher | user | user | teacher@example.org |
And the following "courses" exist:
| fullname | shortname | category |
| Course 1 | C1 | 0 |
Expand All @@ -22,14 +22,15 @@ Feature: Preview aitext questions
| Test questions | aitext | aitext-001 | editor |
| Test questions | aitext | aitext-002 | plain |

@javascript @_switch_window
@javascript
Scenario: Preview an aitext question that uses the HTML editor.
# Testing with the HTML editor is a legacy of the essay fork
# as aitext strips html before sending it may be redundant
When I am on the "aitext-001" "core_question > preview" page logged in as teacher
And I expand all fieldsets
And I set the field "How questions behave" to "Immediate feedback"
# And I press "Start again with these options"
And I press "saverestart"

And I should see "Please write a story about a frog."

@javascript @_switch_window
Expand Down
10 changes: 8 additions & 2 deletions tests/helper.php
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ public static function make_aitext_question(array $options) {
'questiontext' => $options['questiontext'] ?? '',
'aiprompt' => $options['aiprompt'] ?? 0,
'markscheme' => $options['markscheme'] ?? 0,

'sampleanswer' => $options['sampleanswer'] ?? 0,
'model' => $options['model'] ?? '',
];

$type = 'aitext';
Expand All @@ -73,6 +74,8 @@ protected function initialise_aitext_question() {
$q->responsefieldlines = 10;
$q->minwordlimit = null;
$q->maxwordlimit = null;
$q->sampleanswer = '';
$q->model = 'gpt4';
$q->graderinfo = '';
$q->graderinfoformat = FORMAT_HTML;
$q->qtype = question_bank::get_qtype('aitext');
Expand Down Expand Up @@ -111,6 +114,8 @@ public function get_aitext_question_form_data_editor() {
$fromform->status = \core_question\local\bank\question_version_status::QUESTION_STATUS_READY;
$fromform->aiprompt = 'A prompt for the LLM';
$fromform->markscheme = 'Give one mark if the answer is correct';
$fromform->sampleanswer = '';
$fromform->model = 'gpt-4';
return $fromform;
}

Expand Down Expand Up @@ -147,7 +152,8 @@ public function get_aitext_question_form_data_plain() {
$fromform->graderinfo = array('text' => '', 'format' => FORMAT_HTML);
$fromform->responsetemplate = array('text' => '', 'format' => FORMAT_HTML);
$fromform->status = \core_question\local\bank\question_version_status::QUESTION_STATUS_READY;

$fromform->sampleanswer = '';
$fromform->model = 'gpt-4';
return $fromform;
}

Expand Down
2 changes: 1 addition & 1 deletion version.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
defined('MOODLE_INTERNAL') || die();

$plugin->component = 'qtype_aitext';
$plugin->version = 2024050300;
$plugin->version = 2024051100;
$plugin->requires = 2020110900;
$plugin->release = '0.01';
$plugin->maturity = MATURITY_BETA;
Expand Down

0 comments on commit f9e78f6

Please sign in to comment.