MBS-9476: Implement Vertex AI access for Gemini

classes/base_instance.php

@@ -522,7 +522,7 @@ final public function store_formdata(stdClass $data): void {
         if (!empty($data->endpoint)) {
             $this->set_endpoint(trim($data->endpoint));
         }
-        $this->set_apikey(trim($data->apikey));
+        $this->set_apikey(!empty($data->apikey) ? trim($data->apikey) : '');
         $this->set_connector($data->connector);
         $this->set_tenant(trim($data->tenant));
         if (empty($data->model)) {

classes/local/aitool_option_vertexai.php

@@ -35,7 +35,6 @@ class aitool_option_vertexai {
      * @param \MoodleQuickForm $mform the mform object
      */
     public static function extend_form_definition(\MoodleQuickForm $mform): void {
-        $mform->freeze('apikey');
         $mform->freeze('endpoint');
         $mform->addElement('textarea', 'serviceaccountjson',
                 get_string('serviceaccountjson', 'local_ai_manager'), ['rows' => '20']);
@@ -54,7 +53,7 @@ public static function add_vertexai_to_form_data(string $serviceaccountjson): st
     }
 
     /**
-     * Extract the service account JSON and calculate the new endpoint from the form data submitted by the form.
+     * Extract the service account JSON and calculate the new base endpoint from the form data submitted by the form.
      *
      * @param stdClass $data the form data after submission
      * @return array array of the service account JSON and the calculated endpoint
@@ -64,10 +63,10 @@ public static function extract_vertexai_to_store(stdClass $data): array {
         $serviceaccountinfo = json_decode($serviceaccountjson);
         $projectid = $serviceaccountinfo->project_id;
 
-        $endpoint = 'https://europe-west3-aiplatform.googleapis.com/v1/projects/' . $projectid
+        $baseendpoint = 'https://europe-west3-aiplatform.googleapis.com/v1/projects/' . $projectid
                 . '/locations/europe-west3/publishers/google/models/'
-                . $data->model . ':predict';
-        return [$serviceaccountjson, $endpoint];
+                . $data->model;
+        return [$serviceaccountjson, $baseendpoint];
     }
 
     /**
@@ -84,10 +83,15 @@ public static function validate_vertexai(array $data): array {
         }
 
         $serviceaccountinfo = json_decode(trim($data['serviceaccountjson']));
-        foreach (['private_key_id', 'private_key', 'client_email'] as $field) {
-            if (!property_exists($serviceaccountinfo, $field)) {
-                $errors['serviceaccountjson'] = get_string('error_vertexai_serviceaccountjsoninvalid', 'local_ai_manager', $field);
-                break;
+        if (is_null($serviceaccountinfo)) {
+            $errors['serviceaccountjson'] = get_string('error_vertexai_serviceaccountjsoninvalid', 'local_ai_manager');
+        } else {
+            foreach (['private_key_id', 'private_key', 'client_email'] as $field) {
+                if (!property_exists($serviceaccountinfo, $field)) {
+                    $errors['serviceaccountjson'] =
+                            get_string('error_vertexai_serviceaccountjsoninvalidmissing', 'local_ai_manager', $field);
+                    break;
+                }
             }
         }
 

classes/local/aitool_option_vertexai_authhandler.php

@@ -21,7 +21,12 @@
 use Psr\Http\Client\ClientExceptionInterface;
 
 /**
- * Class responsible for handling authentication with Vertex AI using Google's OAuth mechanism.
+ * Helper class for providing the necessary extension functions to implement the authentication with Google OAuth for an AI tool.
+ *
+ * @package    local_ai_manager
+ * @copyright  2024 ISB Bayern
+ * @author     Philipp Memmel
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 class aitool_option_vertexai_authhandler {
 
@@ -101,6 +106,7 @@ public function retrieve_access_token(): array {
      * in the cache.
      *
      * @return string the access token as string, empty if no
+     * @throws \moodle_exception if there is an error retrieving the access token.
      */
     public function get_access_token(): string {
         $clock = \core\di::get(\core\clock::class);
@@ -140,4 +146,23 @@ public function clear_access_token(): void {
         $authcache->delete($this->instanceid);
     }
 
+    /**
+     * Regenerate a token if necessary.
+     *
+     * To check if it's necessary you need to pass over a request_response object that contains an answer from the API that you
+     * tried to access with an access token before. If the request response shows that the token was expired, it will be
+     * regenerated. You then can get it by calling {@see self::get_access_token()}.
+     *
+     * @param request_response $requestresponse the request_response object of the request before
+     */
+    public function is_expired_accesstoken_reason_for_failing(request_response $requestresponse): bool {
+        if ($requestresponse->get_code() !== 401) {
+            return false;
+        }
+        // We need to reset the stream, so we can again read it.
+        $requestresponse->get_response()->rewind();
+        $content = json_decode($requestresponse->get_response()->getContents(), true);
+        return !empty(array_filter($content['error']['details'], fn($details) => $details['reason'] === 'ACCESS_TOKEN_EXPIRED'));
+    }
+
 }

db/upgrade.php

@@ -163,5 +163,20 @@ function xmldb_local_ai_manager_upgrade($oldversion) {
         upgrade_plugin_savepoint(true, 2024110501, 'local', 'ai_manager');
     }
 
+    if ($oldversion < 2024120200) {
+
+        $rs = $DB->get_recordset('local_ai_manager_instance', ['connector' => 'gemini']);
+        foreach ($rs as $record) {
+            $record->customfield2 = 'googleai';
+            $record->model = str_replace('-latest', '', $record->model);
+            $record->endpoint = 'https://generativelanguage.googleapis.com/v1beta/models/' . $record->model . ':generateContent';
+            $DB->update_record('local_ai_manager_instance', $record);
+        }
+        $rs->close();
+
+        // AI manager savepoint reached.
+        upgrade_plugin_savepoint(true, 2024120200, 'local', 'ai_manager');
+    }
+
     return true;
 }

lang/de/local_ai_manager.php

@@ -83,7 +83,8 @@
 $string['error_userlocked'] = 'Ihr Nutzer wurde von Ihrem Tenant-Manager gesperrt.';
 $string['error_usernotconfirmed'] = 'Für die Nutzung müssen die Nutzungsbedingungen akzeptiert werden.';
 $string['error_vertexai_serviceaccountjsonempty'] = 'Sie müssen den Inhalt der JSON-Datei, die Sie beim Anlegen des Service-Accounts in Ihrer Google-Cloud-Console heruntergeladen haben, einfügen.';
-$string['error_vertexai_serviceaccountjsoninvalid'] = 'Ungültiges Format. Es fehlt der Eintrag "{$a}".';
+$string['error_vertexai_serviceaccountjsoninvalid'] = 'Ungültiges Format. Es muss sich um gültiges JSON handeln.';
+$string['error_vertexai_serviceaccountjsoninvalidmissing'] = 'Ungültiges Format. Es fehlt der Eintrag "{$a}".';
 $string['exception_curl'] = 'Es ist ein Fehler bei der Verbindung zum externen KI-Tool aufgetreten.';
 $string['exception_curl28'] = 'Die API hat zu lange gebraucht, um Ihre Anfrage zu verarbeiten, oder konnte nicht in angemessener Zeit erreicht werden.';
 $string['exception_default'] = 'Ein allgemeiner Fehler ist aufgetreten, während versucht wurde, die Anfrage an das KI-Tool zu senden.';

lang/en/local_ai_manager.php

@@ -83,7 +83,8 @@
 $string['error_userlocked'] = 'Your user has been locked by your tenant manager.';
 $string['error_usernotconfirmed'] = 'You have not accepted the terms of use yet.';
 $string['error_vertexai_serviceaccountjsonempty'] = 'You need to paste the content of the JSON file that you downloaded when creating the service account in your Google Cloud Console.';
-$string['error_vertexai_serviceaccountjsoninvalid'] = 'Invalid format. The entry "{$a}" is missing.';
+$string['error_vertexai_serviceaccountjsoninvalid'] = 'Invalid format. Has to be valid JSON.';
+$string['error_vertexai_serviceaccountjsoninvalidmissing'] = 'Invalid format. The entry "{$a}" is missing.';
 $string['exception_curl'] = 'A connection error to the external API endpoint has occurred';
 $string['exception_curl28'] = 'The API took too long to process your request or could not be reached in a reasonable time';
 $string['exception_default'] = 'A general error occurred while trying to send the request to the AI tool';

tools/gemini/classes/connector.php

@@ -16,7 +16,9 @@
 
 namespace aitool_gemini;
 
+use local_ai_manager\local\aitool_option_vertexai_authhandler;
 use local_ai_manager\local\prompt_response;
+use local_ai_manager\local\request_response;
 use local_ai_manager\local\unit;
 use local_ai_manager\local\usage;
 use Psr\Http\Message\StreamInterface;
@@ -31,15 +33,18 @@
  */
 class connector extends \local_ai_manager\base_connector {
 
+    /** @var string The access token to use for authentication against the Google API endpoint. */
+    private string $accesstoken = '';
+
     #[\Override]
     public function get_models_by_purpose(): array {
-        $textmodels = ['gemini-1.0-pro-latest', 'gemini-1.0-pro-vision-latest', 'gemini-1.5-flash-latest', 'gemini-1.5-pro-latest'];
+        $textmodels = ['gemini-1.0-pro', 'gemini-1.0-pro-vision', 'gemini-1.5-flash', 'gemini-1.5-pro'];
         return [
                 'chat' => $textmodels,
                 'feedback' => $textmodels,
                 'singleprompt' => $textmodels,
                 'translate' => $textmodels,
-                'itt' => ['gemini-1.5-pro-latest', 'gemini-1.5-flash-latest'],
+                'itt' => ['gemini-1.5-pro', 'gemini-1.5-flash'],
         ];
     }
 
@@ -148,12 +153,41 @@ public function has_customvalue2(): bool {
     protected function get_headers(): array {
         $headers = parent::get_headers();
         if (in_array('Authorization', array_keys($headers))) {
-            unset($headers['Authorization']);
-            $headers['x-goog-api-key'] = $this->get_api_key();
+            if ($this->instance->get_customfield2() === \aitool_gemini\instance::GOOGLE_BACKEND_GOOGLEAI) {
+                unset($headers['Authorization']);
+                $headers['x-goog-api-key'] = $this->get_api_key();
+            } else {
+                $headers['Authorization'] = 'Bearer ' . $this->accesstoken;
+            }
         }
         return $headers;
     }
 
+    #[\Override]
+    public function make_request(array $data): request_response {
+        if ($this->instance->get_customfield2() === instance::GOOGLE_BACKEND_VERTEXAI) {
+            $vertexaiauthhandler =
+                    new aitool_option_vertexai_authhandler($this->instance->get_id(), $this->instance->get_customfield3());
+            try {
+                $this->accesstoken = $vertexaiauthhandler->get_access_token();
+            } catch (\moodle_exception $exception) {
+                return request_response::create_from_error(0, $exception->getMessage(), $exception->getTraceAsString());
+            }
+            $requestresponse = parent::make_request($data);
+            // We keep track of the time the cached access token expires. However, due latency, different clocks
+            // on different servers etc. we could end up sending a request with an actually expired access token.
+            // In this case we refresh our access token and re-submit the request ONE TIME.
+            if ($vertexaiauthhandler->is_expired_accesstoken_reason_for_failing($requestresponse)) {
+                try {
+                    $vertexaiauthhandler->refresh_access_token();
+                } catch (\moodle_exception $exception) {
+                    return request_response::create_from_error(0, $exception->getMessage(), $exception->getTraceAsString());
+                }
+            }
+        }
+        return parent::make_request($data);
+    }
+
     #[\Override]
     public function allowed_mimetypes(): array {
         // We use the inline_data for sending data to the API, so we basically support every format.

tools/gemini/classes/instance.php

@@ -18,6 +18,7 @@
 
 use local_ai_manager\base_instance;
 use local_ai_manager\local\aitool_option_temperature;
+use local_ai_manager\local\aitool_option_vertexai;
 use stdClass;
 
 /**
@@ -30,18 +31,37 @@
  */
 class instance extends base_instance {
 
+    /** @var string Constant for declaring that Gemini should be used over the open Google AI API. */
+    const GOOGLE_BACKEND_GOOGLEAI = 'googleai';
+
+    /** @var string Constant for declaring that Gemini should be used over Vertex AI. */
+    const GOOGLE_BACKEND_VERTEXAI = 'vertexai';
+
     #[\Override]
     protected function extend_form_definition(\MoodleQuickForm $mform): void {
-        $mform->setDefault('endpoint', 'https://generativelanguage.googleapis.com/v1beta/models');
-        $mform->freeze('endpoint');
+        $mform->addElement('select', 'googlebackend', get_string('googlebackend', 'aitool_gemini'),
+                [
+                        self::GOOGLE_BACKEND_GOOGLEAI => get_string('googlebackendgoogleai', 'aitool_gemini'),
+                        self::GOOGLE_BACKEND_VERTEXAI => get_string('googlebackendvertexai', 'aitool_gemini'),
+                ]);
+        aitool_option_vertexai::extend_form_definition($mform);
+        $mform->hideIf('serviceaccountjson', 'googlebackend', 'neq', 'vertexai');
+        $mform->hideIf('apikey', 'googlebackend', 'eq', 'vertexai');
 
         aitool_option_temperature::extend_form_definition($mform);
     }
 
     #[\Override]
     protected function get_extended_formdata(): stdClass {
-        $temperature = $this->get_customfield1();
         $data = new stdClass();
+        if ($this->get_customfield2() === self::GOOGLE_BACKEND_VERTEXAI) {
+            $data->googlebackend = self::GOOGLE_BACKEND_VERTEXAI;
+            $vertexaidata = aitool_option_vertexai::add_vertexai_to_form_data($this->get_customfield3());
+            foreach ($vertexaidata as $key => $value) {
+                $data->{$key} = $value;
+            }
+        }
+        $temperature = $this->get_customfield1();
         $temperaturedata = aitool_option_temperature::add_temperature_to_form_data($temperature);
         foreach ($temperaturedata as $key => $value) {
             $data->{$key} = $value;
@@ -53,11 +73,26 @@ protected function get_extended_formdata(): stdClass {
     protected function extend_store_formdata(stdClass $data): void {
         $temperature = aitool_option_temperature::extract_temperature_to_store($data);
         $this->set_customfield1($temperature);
+        $this->set_customfield2($data->googlebackend);
+        if ($data->googlebackend === self::GOOGLE_BACKEND_VERTEXAI) {
+            [$serviceaccountjson, $baseendpoint] = aitool_option_vertexai::extract_vertexai_to_store($data);
+            $this->set_customfield3($serviceaccountjson);
+            $this->set_endpoint($baseendpoint . ':generateContent');
+        } else {
+            $this->set_endpoint('https://generativelanguage.googleapis.com/v1beta/models/' . $this->get_model() .
+                    ':generateContent');
+        }
     }
 
     #[\Override]
     protected function extend_validation(array $data, array $files): array {
-        return aitool_option_temperature::validate_temperature($data);
+        $errors = [];
+        if ($data['googlebackend'] === self::GOOGLE_BACKEND_VERTEXAI) {
+            $errors = array_merge($errors, aitool_option_vertexai::validate_vertexai($data));
+        }
+        $errors = array_merge($errors, aitool_option_temperature::validate_temperature($data));
+        return $errors;
+
     }
 
     /**
@@ -68,9 +103,4 @@ protected function extend_validation(array $data, array $files): array {
     public function get_temperature(): float {
         return floatval($this->get_customfield1());
     }
-
-    #[\Override]
-    public function get_endpoint(): string {
-        return parent::get_endpoint() . '/' . $this->get_model() . ':generateContent';
-    }
 }

tools/gemini/lang/de/aitool_gemini.php

@@ -24,5 +24,8 @@
  */
 
 $string['adddescription'] = 'Gemini, das KI-Modell von Google DeepMind, kombiniert fortschrittliche Sprachmodelle (LLMs) mit Tools und multimodalen Fähigkeiten.';
+$string['googlebackend'] = 'Zu nutzendes Google-Backend';
+$string['googlebackendgoogleai'] = 'Google AI';
+$string['googlebackendvertexai'] = 'Vertex AI';
 $string['pluginname'] = 'Gemini';
 $string['privacy:metadata'] = 'Das Subplugin des ai_manager Plugins "Gemini" speichert keine personenbezogenen Daten.';

tools/gemini/lang/en/aitool_gemini.php

@@ -24,5 +24,8 @@
  */
 
 $string['adddescription'] = 'Gemini, Google DeepMind\'s AI model, combines advanced language models (LLMs) with tools and multimodal capabilities.';
+$string['googlebackend'] = 'Google Backend to use';
+$string['googlebackendgoogleai'] = 'Google AI';
+$string['googlebackendvertexai'] = 'Vertex AI';
 $string['pluginname'] = 'Gemini';
 $string['privacy:metadata'] = 'The local ai_manager tool subplugin "Gemini" does not store any personal data.';

tools/gemini/version.php

@@ -24,7 +24,7 @@
  */
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version  = 2023121205;
+$plugin->version  = 2024120300;
 $plugin->requires = 2023042403;
 $plugin->release = '0.0.1';
 $plugin->component = 'aitool_gemini';

tools/googlesynthesize/classes/instance.php

@@ -17,6 +17,8 @@
 namespace aitool_googlesynthesize;
 
 use local_ai_manager\base_instance;
+use local_ai_manager\local\aitool_option_temperature;
+use local_ai_manager\local\aitool_option_vertexai;
 
 /**
  * Instance class for the connector instance of aitool_googlesynthesize.