Merge branch 'selectmodel'

backup/moodle2/backup_qtype_aitext_plugin.class.php

@@ -49,7 +49,7 @@ protected function define_question_plugin_structure() {
 
         $aitext = new backup_nested_element('aitext', ['id'], [
                 'aiprompt', 'markscheme', 'sampleanswer', 'responseformat', 'responsefieldlines', 'minwordlimit', 'maxwordlimit',
-                'graderinfo', 'graderinfoformat', 'responsetemplate',
+                'graderinfo', 'graderinfoformat', 'responsetemplate', 'model',
                 'responsetemplateformat', 'maxbytes']);
 
         // Now the own qtype tree.

backup/moodle2/restore_qtype_aitext_plugin.class.php

@@ -127,6 +127,7 @@ protected function after_execute_question() {
             $defaultoptions->attachments = 0;
             $defaultoptions->attachmentsrequired = 0;
             $defaultoptions->graderinfo = '';
+            $defaultoptions->model = '';
             $defaultoptions->graderinfoformat = FORMAT_HTML;
             $defaultoptions->responsetemplate = '';
             $defaultoptions->responsetemplateformat = FORMAT_HTML;

changelog.md

@@ -5,3 +5,8 @@ to allw the testing of prompts from within the question editing form.
 
 Refined the default prompt settings to ensure Ollama/mistral returns a number when grading
 
+The model field in the plugins settings can now accept a comma separated list. If there
+is more than one model the question editor form will show a dropdown with available models. The selected model is written to the database and will be used as part of the connection to the AI system.
+
+Add model name to disclaimer if configured in settings as [[model]]
+

classes/external.php

@@ -18,7 +18,7 @@
  * External
  *
  * @package    qtype_aitext
- * @author     Justin Hunt - poodll.com
+ * @copyright  Justin Hunt - poodll.com
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 defined('MOODLE_INTERNAL') || die();

db/install.xml

@@ -20,7 +20,8 @@
         <FIELD NAME="aiprompt" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="The prompt to be sent to the LLM for getting an answer/feedback"/>
         <FIELD NAME="markscheme" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="Instructions on how responses are to be marked."/>
         <FIELD NAME="sampleanswer" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="A sample answer for testing the prompt and AI response"/>
-      </FIELDS>
+        <FIELD NAME="model" TYPE="char" LENGTH="60" NOTNULL="false" SEQUENCE="false" COMMENT="The model in the LLM, e.g. gpt-4 or llama3"/>
+</FIELDS>
       <KEYS>
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
         <KEY NAME="questionid" TYPE="foreign-unique" FIELDS="questionid" REFTABLE="question" REFFIELDS="id"/>

db/upgrade.php

@@ -18,6 +18,7 @@
  * AI Text question type upgrade code.
  *
  * @package    qtype_aitext
+ * @copyright  Marcus Green 2024
  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
@@ -46,5 +47,20 @@ function xmldb_qtype_aitext_upgrade($oldversion) {
 
     }
 
+    if ($oldversion < 2024051100) {
+
+        // Define field model to be added to qtype_aitext.
+        $table = new xmldb_table('qtype_aitext');
+        $field = new xmldb_field('model', XMLDB_TYPE_CHAR, '60', null, null, null, null, 'sampleanswer');
+
+        // Conditionally launch add field model.
+        if (!$dbman->field_exists($table, $field)) {
+            $dbman->add_field($table, $field);
+        }
+
+        // Aitext savepoint reached.
+        upgrade_plugin_savepoint(true, 2024051100, 'qtype', 'aitext');
+    }
+
     return true;
 }

edit_aitext_form.php

@@ -57,7 +57,16 @@ protected function definition_inner($mform) {
         $mform->setType('markscheme', PARAM_RAW);
         $mform->setDefault('markscheme', get_config('qtype_aitext', 'defaultmarksscheme'));
         $mform->addHelpButton('markscheme', 'markscheme', 'qtype_aitext');
+        $models = explode(",", get_config('tool_aiconnect', 'model'));
+        if (count($models) > 1 ) {
+            $models = array_combine($models, $models);
+            $mform->addElement('select', 'model', get_string('model', 'qtype_aitext'), $models);
 
+        } else {
+            $mform->addElement('hidden', 'model', $models[0]);
+        }
+        $mform->setType('model', PARAM_RAW);
+        // The question_edit_form that this class extends expects a general feedback field.
         $mform->addElement('html', '<div class="hidden">');
         $mform->addElement('editor', 'generalfeedback', get_string('generalfeedback', 'question')
         , ['rows' => 10], $this->editoroptions);
@@ -67,12 +76,12 @@ protected function definition_inner($mform) {
         $mform->addElement('textarea', 'sampleanswer', get_string('sampleanswer', 'qtype_aitext'),
             ['maxlen' => 50, 'rows' => 6, 'size' => 30]);
         $mform->setType('sampleanswer', PARAM_RAW);
+        $mform->setDefault('sampleanswer', '');
         $mform->addHelpButton('sampleanswer', 'sampleanswer', 'qtype_aitext');
         $mform->addElement('static', 'sampleanswereval', '',  '<a class="qtype_aitext_sampleanswerbtn btn btn-secondary"
                 id="id_sampleanswerbtn">'
             . get_string('sampleanswerevaluate', 'qtype_aitext') . '</a>' .
              '<div class="qtype_aitext_sampleanswereval" id="id_sampleanswereval"></div>');
-
         $mform->addElement('header', 'responseoptions', get_string('responseoptions', 'qtype_aitext'));
         $mform->setExpanded('responseoptions');
 

lang/en/qtype_aitext.php

@@ -60,6 +60,7 @@
 $string['minwordlimit'] = 'Minimum word limit';
 $string['minwordlimit_help'] = 'If the response requires that students enter text, this is the minimum number of words that each student will be allowed to submit.';
 $string['minwordlimitboundary'] = 'This question requires a response of at least {$a->limit} words and you are attempting to submit {$a->count} words. Please expand your response and try again.';
+$string['model'] = 'Model';
 $string['nlines'] = '{$a} lines';
 $string['prompt'] = 'Prompt';
 $string['prompt_setting'] = 'Wrapper text for the prompt set to the AI System, [responsetext] is whatever the student typed as an answer. The ai prompt value from the question will be appended to this';

question.php

@@ -55,6 +55,12 @@ class qtype_aitext_question extends question_graded_automatically_with_countback
     /** @var int indicates whether the maximum number of words required */
     public $maxwordlimit;
 
+    /**
+     * LLM Model, will vary between AI systems, e.g. gpt4 or llama3
+     * @var stream_set_blocking
+     */
+    public $model;
+
 
     /**
      * used in the question editing interface
@@ -146,7 +152,7 @@ public function grade_response(array $response) : array {
             $grade = [0 => 0, question_state::$needsgrading];
             return $grade;
         }
-        $ai = new ai\ai();
+        $ai = new ai\ai($this->model);
         if (is_array($response)) {
             $fullaiprompt = $this->build_full_ai_prompt($response['answer'], $this->aiprompt,
                  $this->defaultmark, $this->markscheme);
@@ -173,8 +179,16 @@ public function grade_response(array $response) : array {
         return $grade;
     }
 
+    /**
+     * Used by prompttester in the editing form
+     *
+     * @param string $response
+     * @param string $aiprompt
+     * @param number $defaultmark
+     * @param string $markscheme
+     * @return void
+     */
     public function build_full_ai_prompt($response, $aiprompt, $defaultmark, $markscheme) {
-
         $responsetext = strip_tags($response);
             $responsetext = '[['.$responsetext.']]';
             $prompt = get_config('qtype_aitext', 'prompt');
@@ -211,7 +225,9 @@ public function process_feedback(string $feedback) {
         if (json_last_error() === JSON_ERROR_NONE) {
             $contentobject->feedback = trim($contentobject->feedback);
             $contentobject->feedback = preg_replace(array('/\[\[/', '/\]\]/'), '"', $contentobject->feedback);
-            $contentobject->feedback .= ' '.$this->llm_translate(get_config('qtype_aitext', 'disclaimer'));
+            $disclaimer = get_config('qtype_aitext', 'disclaimer');
+            $disclaimer = str_replace("[[model]]", $this->model, $disclaimer);
+            $contentobject->feedback .= ' '.$this->llm_translate($disclaimer);
         } else {
             $contentobject = (object) [
                                         "feedback" => $feedback,

questiontype.php

@@ -78,6 +78,8 @@ public function save_defaults_for_new_questions(stdClass $fromform): void {
         $this->set_default_value('responseformat', $fromform->responseformat);
         $this->set_default_value('responsefieldlines', $fromform->responsefieldlines);
         $this->set_default_value('markscheme', $fromform->markscheme);
+        $this->set_default_value('sampleanswer', $fromform->sampleanswer);
+
     }
     /**
      * Write the question data from the editing form to the database
@@ -97,6 +99,7 @@ public function save_question_options($formdata) {
         $options->aiprompt = $formdata->aiprompt;
         $options->markscheme = $formdata->markscheme;
         $options->sampleanswer = $formdata->sampleanswer;
+        $options->model = trim($formdata->model);
         $options->responseformat = $formdata->responseformat;
         $options->responsefieldlines = $formdata->responsefieldlines;
         $options->minwordlimit = isset($formdata->minwordenabled) ? $formdata->minwordlimit : null;
@@ -144,6 +147,7 @@ protected function initialise_question_instance(question_definition $question, $
         $question->aiprompt = $questiondata->options->aiprompt;
         $question->markscheme = $questiondata->options->markscheme;
         $question->sampleanswer = $questiondata->options->sampleanswer;
+        $question->model = $questiondata->options->model;
     }
     /**
      * Delete a question from the database
@@ -249,7 +253,8 @@ public function extra_question_fields() {
             'responsetemplateformat',
             'aiprompt',
             'markscheme',
-            'sampleanswer'
+            'sampleanswer',
+            'model',
         ];
     }
     /**

renderer.php

@@ -124,8 +124,8 @@ public function feedback(question_attempt $qa, question_display_options $options
         // This probably should be retrieved by an api call.
         $comment = $qa->get_current_manual_comment();
         if ($this->page->pagetype == 'question-bank-previewquestion-preview') {
-            $this->page->requires->js_call_amd('qtype_aitext/showprompt', 'init', []);
             if ($comment[0] > '') {
+                $this->page->requires->js_call_amd('qtype_aitext/showprompt', 'init', []);
                 $prompt = $qa->get_last_qt_var('-aiprompt');
                 $showprompt = '<br/><button  id=showprompt class="rounded">'. get_string('showprompt', 'qtype_aitext').'</button>';
                 $showprompt .= '<div id="fullprompt" class="hidden">'.$prompt .'</div>';

settings.php

@@ -27,7 +27,7 @@
     $settings->add(new admin_setting_configtext('qtype_aitext/disclaimer',
             new lang_string('disclaimer', 'qtype_aitext'),
             new lang_string('disclaimer_setting', 'qtype_aitext'),
-            "(Response provided by ChatGPT)"));
+            "(Response provided by [[model]])"));
 
     $settings->add(new admin_setting_configtextarea('qtype_aitext/defaultprompt',
         new lang_string('defaultprompt', 'qtype_aitext'),
@@ -42,7 +42,7 @@
         'qtype_aitext/disclaimer',
         new lang_string('disclaimer', 'qtype_aitext'),
         new lang_string('disclaimer_setting', 'qtype_aitext'),
-        '(Response provided by ChatGPT)'
+        '(Response provided by [[model]])'
         ));
     $settings->add(new admin_setting_configtextarea(
         'qtype_aitext/prompt',

tests/behat/add.feature

@@ -5,6 +5,7 @@ Feature: Test creating an AIText question
     I need to be able to create an aitext question
 
   Background:
+
     Given the following "users" exist:
         | username |
         | teacher  |
@@ -14,18 +15,22 @@ Feature: Test creating an AIText question
     And the following "course enrolments" exist:
         | user    | course | role           |
         | teacher | C1     | editingteacher |
-
+    And the following config values are set as admin:
+      | model | gpt-4,gpt-4o  | tool_aiconnect |
+  @javascript
   Scenario: Create an AI text question with Response format set to 'HTML editor'
     When I am on the "Course 1" "core_question > course question bank" page logged in as teacher
+    # id_sampleanswer because it is collapsed when the form is opened for editing.
     And I add a "AI Text" question filling the form with:
         | Question name    | aitext-001                      |
         | Question text    | Write an aitext with 500 words. |
         | General feedback | This is general feedback        |
         | Response format  | HTML editor                     |
         | AI Prompt        | Evaluate this                   |
         | Mark scheme      | Give one mark if correct        |
+        | id_sampleanswer  | sample  answer                  |
 
-    Then I should see "aitext-001"
+  Then I should see "aitext-001"
 
   Scenario: Create an AI Text question with Response format set to 'HTML editor with the file picker'
     When I am on the "Course 1" "core_question > course question bank" page logged in as teacher
@@ -35,9 +40,12 @@ Feature: Test creating an AIText question
         | General feedback | This is general feedback        |
         | AI Prompt        | Evaluate this                   |
         | Mark scheme      | Give one mark if correct        |
+        | id_sampleanswer  | sample  answer                  |
+        | Model | gpt-4|
 
-    Then I should see "aitext-002"
 
+
+    Then I should see "aitext-002"
   @javascript
   Scenario: Create an AI Text question for testing some default options
     When I am on the "Course 1" "core_question > course question bank" page logged in as teacher
@@ -46,6 +54,10 @@ Feature: Test creating an AIText question
         | Question text         | Write an aitext with 500 words. |
         | General feedback      | This is general feedback        |
         | id_responsefieldlines | 15                              |
+        | id_sampleanswer         | sample  answer                |
+        | Model | gpt-4o|
+
+
     Then I should see "aitext-003"
     # Checking that the next new question form displays user preferences settings.
     And I press "Create a new question ..."

tests/behat/backup_and_restore.feature

@@ -22,6 +22,12 @@ Feature: Test duplicating a quiz containing an aitext question
         | aitext-001 | 1 |
         | aitext-002 | 1 |
 
+    # Without this it will show the pending progress bar and the back
+    # to course button introduced in Moodle 4.3
+    # https://docs.moodle.org/403/en/Course_backup#Asynchronous_course_backups
+    And the following config values are set as admin:
+      | enableasyncbackup | 0 |
+
   @javascript
   Scenario: Backup and restore a course containing 3 aitext questions
     When I am on the "Course 1" course page logged in as admin

tests/behat/preview.feature

@@ -6,8 +6,8 @@ Feature: Preview aitext questions
 
   Background:
     Given the following "users" exist:
-        | username |
-        | teacher  |
+        | username | firstname | lastname | email               |
+        | teacher  | user      | user | teacher@example.org |
     And the following "courses" exist:
         | fullname | shortname | category |
         | Course 1 | C1        | 0        |
@@ -22,14 +22,15 @@ Feature: Preview aitext questions
         | Test questions   | aitext | aitext-001 | editor   |
         | Test questions   | aitext | aitext-002 | plain    |
 
-  @javascript @_switch_window
+  @javascript
   Scenario: Preview an aitext question that uses the HTML editor.
+  # Testing with the HTML editor is a legacy of the essay fork
+  # as aitext strips html before sending it may be redundant
     When I am on the "aitext-001" "core_question > preview" page logged in as teacher
     And I expand all fieldsets
     And I set the field "How questions behave" to "Immediate feedback"
     # And I press "Start again with these options"
     And I press "saverestart"
-
     And I should see "Please write a story about a frog."
 
   @javascript @_switch_window

tests/helper.php

@@ -44,16 +44,14 @@ public function get_test_questions() {
      * @return qtype_aitext_question
      */
     public static function make_aitext_question(array $options) {
-        $optionsparam = [
-            'questiontext' => $options['questiontext'] ?? '',
-            'aiprompt' => $options['aiprompt'] ?? 0,
-            'markscheme' => $options['markscheme'] ?? 0,
-
-        ];
-
-        $type = 'aitext';
-        question_bank::load_question_definition_classes($type);
+        question_bank::load_question_definition_classes('aitext');
         $question = new qtype_aitext_question();
+        $question->questiontext = $options['questiontext'] ?? '';
+        $question->model = $options['model'] ?? '';
+        $question->sampleanswer = $options['sampleanswer'] ?? '';
+        $question->markscheme = $options['markscheme'] ?? '';
+        $question->aiprompt = $options['aiprompt'] ?? '';
+
         test_question_maker::initialise_a_question($question);
         $question->qtype = question_bank::get_qtype('aitext');
         return $question;
@@ -73,6 +71,8 @@ protected function initialise_aitext_question() {
         $q->responsefieldlines = 10;
         $q->minwordlimit = null;
         $q->maxwordlimit = null;
+        $q->sampleanswer = '';
+        $q->model = 'gpt4';
         $q->graderinfo = '';
         $q->graderinfoformat = FORMAT_HTML;
         $q->qtype = question_bank::get_qtype('aitext');
@@ -111,6 +111,8 @@ public function get_aitext_question_form_data_editor() {
         $fromform->status = \core_question\local\bank\question_version_status::QUESTION_STATUS_READY;
         $fromform->aiprompt = 'A prompt for the LLM';
         $fromform->markscheme = 'Give one mark if the answer is correct';
+        $fromform->sampleanswer = '';
+        $fromform->model = 'gpt-4';
         return $fromform;
     }
 
@@ -147,7 +149,8 @@ public function get_aitext_question_form_data_plain() {
         $fromform->graderinfo = array('text' => '', 'format' => FORMAT_HTML);
         $fromform->responsetemplate = array('text' => '', 'format' => FORMAT_HTML);
         $fromform->status = \core_question\local\bank\question_version_status::QUESTION_STATUS_READY;
-
+        $fromform->sampleanswer = '';
+        $fromform->model = 'gpt-4';
         return $fromform;
     }
 

tests/question_test.php

@@ -67,12 +67,13 @@ public function test_get_question_summary() {
     public function test_get_feedback() {
         // Create the aitext question under test.
         $questiontext = 'AI question text';
-        $aitext = qtype_aitext_test_helper::make_aitext_question(['questiontext' => $questiontext]);
+        $aitext = qtype_aitext_test_helper::make_aitext_question(['questiontext' => $questiontext, 'model' => 'llama3']);
         $testdata = [
                 "feedback" => "Feedback text",
                 "marks" => 0
                 ];
         $goodjson = json_encode($testdata);
+
         $feedback = $aitext->process_feedback($goodjson);
         $this->assertIsObject($feedback);
         $badjson = 'Some random string'. $goodjson;