From 2294fe48c3d04985494c21fff96ad18e2ae9b8f2 Mon Sep 17 00:00:00 2001 From: Jen-Chieh Shen Date: Thu, 16 Mar 2023 23:47:30 -0700 Subject: [PATCH] feat: Expose all parameters (#7) * feat: Expose all parameters * doc * note * progress * fix compile * Progress * document it * Wrap --- README.md | 70 ++++++------- openai-completion.el | 189 ++++++++++++----------------------- openai-edit.el | 60 ++++------- openai-embedding.el | 27 ++--- openai-engine.el | 22 +++-- openai-file.el | 63 ++++++++---- openai-fine-tune.el | 230 +++++++++++++++---------------------------- openai-image.el | 147 +++++++++++++++------------ openai-model.el | 22 +++-- openai-moderation.el | 28 +++--- openai.el | 25 +++-- 11 files changed, 395 insertions(+), 488 deletions(-) diff --git a/README.md b/README.md index 4ea07cb..e65140c 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,9 @@ interactable commands you can use, and those are mostly examples.* - [OpenAI.el](#openaiel) - [📚 Documentation](#📚-documentation) - [🔨 Usage](#🔨-usage) - - [The simplest example](#the-simplest-example) - - [📝 Customization](#📝-customization) + - [🔰 The simplest example](#🔰-the-simplest-example) + - [📨 Sending Request](#📨-sending-request) - [📢 API functions](#📢-api-functions) - - [🖥 Setting Model](#🖥-setting-model) - [🔗 References](#🔗-references) - [Contribute](#contribute) @@ -39,7 +38,19 @@ You will need to set up your API key before you can use this library. (setq openai-key "[YOUR API KEY]") ``` -### The simplest example +For requests that need your user identifier, + +```elisp +(setq openai-user "[YOUR USER UID]") +``` + +> 💡 Tip +> +> The two variables `openai-key` and `openai-user` are the default values for +> sending requests! However, you can still overwrite the value by passing the +> keywords `:key` and `:user`! + +### 🔰 The simplest example Here is the simplest example that teaches you how to use this library. This is a function with a `query` and a callback function. @@ -50,37 +61,25 @@ a function with a `query` and a callback function. (message "%s" data))) ``` -### 📝 Customization +### 📨 Sending Request -Most arguments are extracted (excepts the required one) as global variables. -For example, one variable `openai-completon-n` is defined in `openai-completion.el` -file. That variable is used for the completion request, for more information see -https://beta.openai.com/docs/api-reference/completions. The naming convention is -by the following pattern: +Most arguments are exposed in the argument list (excepts the required one). -``` -[PACKAGE NAME]-[API TYPE]-[NAME OF THE ARGUMENT] -``` +For example, the request function `openai-completion` accepts argument +`max-tokens`. By seeing OpenAI's references page: -For example: +> `max_tokens` integer Optional Defaults to 16 +> +> The maximum number of tokens to generate in the completion. +> +> The token count of your prompt plus `max_tokens` cannot exceed the model's +> context length. Most models have a context length of 2048 tokens (except for +> the newest models, which support 4096). ```elisp -(setq openai-edit-temperature 1) -``` - -- `openai` - is the package name -- `edit` - is the api type, see [OpenAI API reference](https://platform.openai.com/docs/api-reference/introduction) -- `temperature` - is the argument for the [Edit](https://platform.openai.com/docs/api-reference/edits) request. - -You can change the model for a single request without changing its global value. - -```elisp -(let ((openai-edit-model "text-davinci-edit-001") ; use another model for this request, - (openai-edit-n 3)) ; and i want three outcomes - (openai-edit-create "What day of the wek is it?" - "Fix the spelling mistakes" - (lambda (data) - (message "%s" data)))) +(openai-completion ... + ... + :max-tokens 4069) ; max out tokens! ``` ### 📢 API functions @@ -101,17 +100,6 @@ For example: - `file` - is the api type, see [OpenAI API reference](https://platform.openai.com/docs/api-reference/introduction) - `list` - is the request name -### 🖥 Setting Model - -You can also choose which model you want to use by going to the -[api](https://api.openai.com/v1/models) website and looking at the id's. -For code usage you probably want something that starts with `code-` whereas -with more text related files you'll likely want something starting with `text-`. - -```elisp -(setq openai-completion-model "NAME-HERE") -``` - ## 🔗 References - [CodeGPT](https://marketplace.visualstudio.com/items?itemName=timkmecl.codegpt3) diff --git a/openai-completion.el b/openai-completion.el index 5f5f21b..64f0179 100644 --- a/openai-completion.el +++ b/openai-completion.el @@ -28,139 +28,62 @@ (require 'openai) -(defcustom openai-completon-model "text-davinci-003" - "ID of the model to use. - -You can use the List models API to see all of your available models." - :type 'string - :group 'openai) - -(defcustom openai-completon-suffix nil - "The suffix that comes after a completion of inserted text." - :type 'string - :group 'openai) - -(defcustom openai-completon-max-tokens 4000 - "The maximum number of tokens to generate in the completion. - -The token count of your prompt plus max_tokens cannot exceed the model's context -length. Most models have a context length of 2048 tokens (except for the newest -models, which support 4096)." - :type 'integer - :group 'openai) - -(defcustom openai-completon-temperature 1.0 - "What sampling temperature to use. - -Higher values means the model will take more risks. Try 0.9 for more creative -applications, and 0 (argmax sampling) for ones with a well-defined answer." - :type 'number - :group 'openai) - -(defcustom openai-completon-top-p 1.0 - "An alternative to sampling with temperature, called nucleus sampling, where -the model considers the results of the tokens with top_p probability mass. -So 0.1 means only the tokens comprising the top 10% probability mass are -considered. - -We generally recommend altering this or `temperature' but not both." - :type 'number - :group 'openai) - -(defcustom openai-completon-n 1 - "How many completions to generate for each prompt." - :type 'integer - :group 'openai) - -(defcustom openai-completon-stream nil - "Whether to stream back partial progress. - -If set, tokens will be sent as data-only server-sent events as they become -available, with the stream terminated by a data: [DONE] message." - :type 'boolean - :group 'openai) - -(defcustom openai-completon-logprobs nil - "Include the log probabilities on the logprobs most likely tokens, as well the -chosen tokens. For example, if logprobs is 5, the API will return a list of the -5 most likely tokens. The API will always return the logprob of the sampled -token, so there may be up to logprobs+1 elements in the response. - -The maximum value for logprobs is 5." - :type 'integer - :group 'openai) - -(defcustom openai-completon-echo nil - "Echo back the prompt in addition to the completion." - :type 'boolean - :group 'openai) - -(defcustom openai-completon-stop nil - "Up to 4 sequences where the API will stop generating further tokens. -The returned text will not contain the stop sequence." - :type 'string - :group 'openai) - -(defcustom openai-completon-presence-penalty 0 - "Number between -2.0 and 2.0. Positive values penalize new tokens based on -whether they appear in the text so far, increasing the model's likelihood to -talk about new topics." - :type 'number - :group 'openai) - -(defcustom openai-completon-frequency-penalty 0 - "Number between -2.0 and 2.0. - -Positive values penalize new tokens based on their existing frequency in the -text so far, decreasing the model's likelihood to repeat the same line verbatim." - :type 'number - :group 'openai) - -(defcustom openai-completon-best-of 1 - "Generates best_of completions server-side and returns the \"best\" (the one -with the highest log probability per token). Results cannot be streamed. - -When used with `n', `best_of' controls the number of candidate completions and -`n' specifies how many to return – `best_of' must be greater than `n'." - :type 'integer - :group 'openai) - -(defcustom openai-completon-logit-bias nil - "Modify the likelihood of specified tokens appearing in the completion." - :type 'list - :group 'openai) - ;; ;;; API ;;;###autoload -(defun openai-completion (query callback) - "Query OpenAI with QUERY. - -Argument CALLBACK is a function received one argument which is the JSON data." +(cl-defun openai-completion ( prompt callback + &key + (key openai-key) + (model "text-davinci-003") + suffix + max-tokens + temperature + top-p + n + stream + logprobs + echo + stop + presence-penalty + frequency-penalty + best-of + logit-bias + (user openai-user)) + "Send completion request. + +Arguments PROMPT and CALLBACK are required for this type of request. PROMPT is +either the question or instruction to OpenAI. CALLBACK is the execuation after +request is made. + +Arguments KEY and USER are global options; however, you can overwrite the value +by passing it in. + +The rest of the arugments are optional, please see OpenAI API reference page +for more information. Arguments here refer to MODEL, SUFFIX, MAX-TOKENS, +TEMPERATURE, TOP-P, N, STREAM, LOGPROBS, ECHO, STOP, PRESENCE-PENALTY, +FREQUENCY-PENALTY, BEST-OF, and LOGIT-BIAS." (openai-request "https://api.openai.com/v1/completions" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("model" . ,openai-completon-model) - ("prompt" . ,query) - ("suffix" . ,openai-completon-suffix) - ("max_tokens" . ,openai-completon-max-tokens) - ("temperature" . ,openai-completon-temperature) - ("top_p" . ,openai-completon-top-p) - ("n" . ,openai-completon-n) - ;;("stream" . ,(if openai-completon-stream "true" "false")) - ("logprobs" . ,openai-completon-logprobs) - ;;("echo" . ,(if openai-completon-echo "true" "false")) - ;;("stop" . ,openai-completon-stop) - ;;("presence_penalty" . ,openai-completon-presence-penalty) - ;;("frequency_penalty" . ,openai-completon-frequency-penalty) - ;;("best_of" . ,openai-completon-best-of) - ;;("logit_bias" . ,(if (listp openai-completon-logit-bias) - ;; (json-encode openai-completon-logit-bias) - ;; openai-completon-logit-bias)) - ("user" . ,openai-user))) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("model" . ,model) + ("prompt" . ,prompt) + ("suffix" . ,suffix) + ("max_tokens" . ,max-tokens) + ("temperature" . ,temperature) + ("top-p" . ,top-p) + ("n" . ,n) + ("stream" . ,stream) + ("logprobs" . ,logprobs) + ("echo" . ,echo) + ("stop" . ,stop) + ("presence_penalty" . ,presence-penalty) + ("frequency_penalty" . ,frequency-penalty) + ("best_of" . ,best-of) + ("logit_bias" . ,logit-bias) + ("user" . ,user))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) @@ -169,6 +92,16 @@ Argument CALLBACK is a function received one argument which is the JSON data." ;; ;;; Application +(defcustom openai-completion-max-tokens 4000 + "The maximum number of tokens to generate in the completion." + :type 'integer + :group 'openai) + +(defcustom openai-completion-temperature 1.0 + "What sampling temperature to use." + :type 'number + :group 'openai) + ;;;###autoload (defun openai-completion-select-insert (start end) "Send the region to OpenAI and insert the result to the next paragraph. @@ -197,7 +130,9 @@ START and END are selected region boundaries." (fill-region original-point (point)) ;; Highlight the region! (call-interactively #'set-mark-command) - (goto-char (1+ original-point)))))))) + (goto-char (1+ original-point))))) + :max-tokens openai-completion-max-tokens + :temperature openai-completion-temperature))) ;;;###autoload (defun openai-completion-buffer-insert () diff --git a/openai-edit.el b/openai-edit.el index 1a4dc81..4ecb843 100644 --- a/openai-edit.el +++ b/openai-edit.el @@ -29,59 +29,39 @@ (require 'openai) -(defcustom openai-edit-model "text-davinci-edit-001" - "ID of the model to use. - -You can use the `text-davinci-edit-001' or `code-davinci-edit-001' model with -this endpoint." - :type 'string - :group 'openai) - -(defcustom openai-edit-n 1 - "How many edits to generate for the input and instruction." - :type 'integer - :group 'openai) - -(defcustom openai-edit-temperature 1 - "What sampling temperature to use, between 0 and 2. Higher values like 0.8 -will make the output more random, while lower values like 0.2 will make it more -focused and deterministic. - -We generally recommend altering this or `top_p' but not both." - :type 'integer - :group 'openai) - -(defcustom openai-edit-top-p 1.0 - "An alternative to sampling with temperature, called nucleus sampling, where -the model considers the results of the tokens with top_p probability mass. -So 0.1 means only the tokens comprising the top 10% probability mass are -considered. - -We generally recommend altering this or `temperature' but not both." - :type 'number - :group 'openai) - ;; ;;; API -(defun openai-edit-create (input instruction callback) +(cl-defun openai-edit-create ( input instruction callback + &key + (key openai-key) + (model "text-davinci-edit-001") + temperature + top-p + n) "Creates a new edit for the provided input, instruction, and parameters. The INPUT is text to use as a starting point for the edit. The INSTRUCTION that tells the model how to edit the prompt. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global options; however, you can overwrite the value by passing +it in. + +The rest of the arugments are optional, please see OpenAI API reference page +for more information. Arguments here refer to TEMPERATURE, TOP-P, and N." (openai-request "https://api.openai.com/v1/edits" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("model" . ,openai-edit-model) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("model" . ,model) ("input" . ,input) ("instruction" . ,instruction) - ("temperature" . ,openai-edit-temperature) - ("top_p" . ,openai-edit-top-p) - ("n" . ,openai-edit-n))) + ("temperature" . ,temperature) + ("top_p" . ,top-p) + ("n" . ,n))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) diff --git a/openai-embedding.el b/openai-embedding.el index 9e30c90..a0e924d 100644 --- a/openai-embedding.el +++ b/openai-embedding.el @@ -29,15 +29,14 @@ (require 'openai) -(defcustom openai-embedding-model "text-embedding-ada-002" - "ID of the model to use." - :type 'string - :group 'openai) - ;; ;;; API -(defun openai-embedding-create (input callback) +(cl-defun openai-embedding-create ( input callback + &key + (key openai-key) + (model "text-embedding-ada-002") + (user openai-user)) "Creates an embedding vector representing the input text. INPUT text to get embeddings for, encoded as a string or array of tokens. @@ -45,15 +44,21 @@ To get embeddings for multiple inputs in a single request, pass an array of strings or array of token arrays. Each input must not exceed 8192 tokens in length. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY and USER are global options; however, you can overwrite the value +by passing it in. + +The rest of the arugments are optional, please see OpenAI API reference page +for more information. Arguments here refer to MODEL." (openai-request "https://api.openai.com/v1/embeddings" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("model" . ,openai-embedding-model) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("model" . ,model) ("input" . ,input) - ("user" . ,openai-user))) + ("user" . ,user))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) diff --git a/openai-engine.el b/openai-engine.el index ee412da..bb16df5 100644 --- a/openai-engine.el +++ b/openai-engine.el @@ -34,31 +34,41 @@ ;; ;;; API -(defun openai-engine-list (callback) +(cl-defun openai-engine-list ( callback + &key + (key openai-key)) "Lists the currently available (non-finetuned) models, and provides basic information about each one such as the owner and availability. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request "https://api.openai.com/v1/engines" :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-engine-retrieve (engine-id callback) +(cl-defun openai-engine-retrieve ( engine-id callback + &key + (key openai-key)) "Retrieves a model instance, providing basic information about it such as the owner and availability. The argument ENGINE-ID is the engine to use for this request. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request (format "https://api.openai.com/v1/engines/%s" engine-id) :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) diff --git a/openai-file.el b/openai-file.el index 5bbd425..44c7989 100644 --- a/openai-file.el +++ b/openai-file.el @@ -32,20 +32,27 @@ ;; ;;; API -(defun openai-file-list (callback) +(cl-defun openai-file-list ( callback + &key + (key openai-key)) "Return a list of files that belong to the user's organization. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request "https://api.openai.com/v1/files" :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-file-upload (file purpose callback) +(cl-defun openai-file-upload ( file purpose callback + &key + (key openai-key)) "Upload a file that contain document(s) to be used across various endpoints/features. @@ -59,12 +66,15 @@ The argument PURPOSE is the intended purpose of the uploaded documents. Use \"fine-tune\" for Fine-tuning. This allows us to validate the format of the uploaded file. -Argument CALLBACK is function with data pass in." +Argument CALLBACK is function with data pass in. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request "https://api.openai.com/v1/files" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode `(("file" . ,file) ("purpose" . ,purpose))) :parser 'json-read @@ -72,51 +82,66 @@ Argument CALLBACK is function with data pass in." (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-file-delete (file-id callback) +(cl-defun openai-file-delete ( file-id callback + &key + (key openai-key)) "Delete a file. The arument FILE-ID is id of the file to use for this request. -Argument CALLBACK is function with data pass in." +Argument CALLBACK is function with data pass in. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request "https://api.openai.com/v1/files" :type "DELETE" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode `(("file_id" . ,file-id))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-file-retrieve (file-id callback) +(cl-defun openai-file-retrieve ( file-id callback + &key + (key openai-key)) "Return information about a specific file. The arument FILE-ID is id of the file to use for this request. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request (format "https://api.openai.com/v1/files/%s" file-id) :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode `(("file_id" . ,file-id))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-file-retrieve-content (file-id callback) +(cl-defun openai-file-retrieve-content ( file-id callback + &key + (key openai-key)) "Return the contents of the specified file The arument FILE-ID is id of the file to use for this request. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in." (openai-request (format "https://api.openai.com/v1/files/%s/content" file-id) :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode `(("file_id" . ,file-id))) :parser 'json-read :success (cl-function diff --git a/openai-fine-tune.el b/openai-fine-tune.el index dbabcb8..ae66f27 100644 --- a/openai-fine-tune.el +++ b/openai-fine-tune.el @@ -28,113 +28,23 @@ (require 'openai) -(defcustom openai-fine-tune-model "curie" - "The name of the base model to fine-tune. - -You can select one of \"ada\", \"babbage\", \"curie\", \"davinci\", or a -fine-tuned model created after 2022-04-21. To learn more about these models, -see the Models documentation." - :type 'string - :group 'openai) - -(defcustom openai-fine-tune-validation-file "" - "The ID of an uploaded file that contains validation data." - :type 'string - :group 'openai) - -(defcustom openai-fine-tune-n-epochs 4 - "The number of epochs to train the model for. - -An epoch refers to one full cycle through the training dataset." - :type 'integer - :group 'openai) - -(defcustom openai-fine-tune-batch-size nil - "The batch size to use for training. - -The batch size is the number of training examples used to train a single forward -and backward pass. - -By default, the batch size will be dynamically configured to be ~0.2% of the -number of examples in the training set, capped at 256 - in general, we've found -that larger batch sizes tend to work better for larger datasets." - :type 'integer - :group 'openai) - -(defcustom openai-fine-tune-learning-rate-multiplier nil - "The learning rate multiplier to use for training. - -The fine-tuning learning rate is the original learning rate used for pretraining -multiplied by this value. - -By default, the learning rate multiplier is the 0.05, 0.1, or 0.2 depending on -final `batch_size' (larger learning rates tend to perform better with larger -batch sizes). We recommend experimenting with values in the range 0.02 to 0.2 to -see what produces the best results." - :type 'integer - :group 'openai) - -(defcustom openai-fine-tune-prompt-loss-weight 0.01 - "The weight to use for loss on the prompt tokens. This controls how much the -model tries to learn to generate the prompt (as compared to the completion which -always has a weight of 1.0), and can add a stabilizing effect to training when -completions are short. - -If prompts are extremely long (relative to completions), it may make sense to -reduce this weight so as to avoid over-prioritizing learning the prompt." - :type 'number - :group 'openai) - -(defcustom openai-fine-tune-compute-classification-metrics 0.01 - "If set, we calculate classification-specific metrics such as accuracy and F-1 -score using the validation set at the end of every epoch. - -In order to compute classification metrics, you must provide a `validation_file'. -Additionally, you must specify `classification_n_classes' for multiclass -classification or `classification_positive_class' for binary classification." - :type 'boolean - :group 'openai) - -(defcustom openai-fine-tune-classification-n-classes nil - "The number of classes in a classification task. - -This parameter is required for multiclass classification." - :type 'integer - :group 'openai) - -(defcustom openai-fine-tune-classification-positive-class nil - "The positive class in binary classification. - -This parameter is needed to generate precision, recall, and F1 metrics when -doing binary classification." - :type 'string - :group 'openai) - -(defcustom openai-fine-tune-classification-betas nil - "If this is provided, we calculate F-beta scores at the specified beta values. - -The F-beta score is a generalization of F-1 score. This is only used for binary -classification. - -With a beta of 1 (i.e. the F-1 score), precision and recall are given the same -weight. A larger beta score puts more weight on recall and less on precision. -A smaller beta score puts more weight on precision and less on recall." - :type 'list - :group 'openai) - -(defcustom openai-fine-tune-suffix nil - "A string of up to 40 characters that will be added to your fine-tuned model -name. - -For example, a suffix of \"custom-model-name\" would produce a model name like -`ada:ft-your-org:custom-model-name-2022-02-15-04-21-04'." - :type 'string - :group 'openai) - ;; ;;; API -(defun openai-fine-tune-create (training-file callback) +(cl-defun openai-fine-tune-create ( training-file callback + &key + (key openai-key) + (model "curie") + validation-file + n-epochs + batch-size + learning-rate-multiplier + prompt-loss-weight + compute-classification-metrics + classification-n-classes + classification-positive-class + classification-betas + suffix) "Creates a job that fine-tunes a specified model from a given dataset. Response includes details of the enqueued job including job status and the name @@ -143,43 +53,56 @@ of the fine-tuned models once complete. The argument TRAINING-FILE is the ID of an uploaded file that contains training data. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global option; however, you can overwrite the value by passing +it in. + +The rest of the arugments are optional, please see OpenAI API reference page +for more information. Arguments here refer to MODEL, VALIDATION-FILE, N-EPOCHS, +BATCH-SIZE, LEARNING-RATE-MULTIPLIER, PROMPT-LOSS-WEIGHT, +COMPUTE-CLASSIFICATION-METRICS, CLASSIFICATION-N-CLASSES, +CLASSIFICATION-POSITIVE-CLASS, CLASSIFICATION-BETAS, and SUFFIX" (openai-request "https://api.openai.com/v1/fine-tunes" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("model" . ,openai-fine-tune-model) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("model" . ,model) ("training_file" . ,training-file) - ("validation_file" . ,openai-fine-tune-validation-file) - ("n_epochs" . ,openai-fine-tune-n-epochs) - ("batch_size" . ,openai-fine-tune-batch-size) - ("learning_rate_multiplier" . ,openai-fine-tune-learning-rate-multiplier) - ("prompt_loss_weight" . ,openai-fine-tune-prompt-loss-weight) - ("compute_classification_metrics" . ,openai-fine-tune-compute-classification-metrics) - ("classification_n_classes" . ,openai-fine-tune-classification-n-classes) - ("classification_positive_class" . ,openai-fine-tune-classification-positive-class) - ("classification_betas" . ,openai-fine-tune-classification-betas) - ("suffix" . ,openai-fine-tune-suffix))) + ("validation_file" . ,validation-file) + ("n_epochs" . ,n-epochs) + ("batch_size" . ,batch-size) + ("learning_rate_multiplier" . ,learning-rate-multiplier) + ("prompt_loss_weight" . ,prompt-loss-weight) + ("compute_classification_metrics" . ,compute-classification-metrics) + ("classification_n_classes" . ,classification-n-classes) + ("classification_positive_class" . ,classification-positive-class) + ("classification_betas" . ,classification-betas) + ("suffix" . ,suffix))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-fine-tune-list (callback) +(cl-defun openai-fine-tune-list ( callback + &key + (key openai-key)) "List your organization's fine-tuning jobs. The argument CALLBACK is execuated after request is made." (openai-request "https://api.openai.com/v1/fine-tunes" :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-fine-tune-retrieve (fine-tune-id callback) +(cl-defun openai-fine-tune-retrieve ( fine-tune-id callback + &key + (key openai-key)) "Gets info about the fine-tune job. The FINE-TUNE-ID of the fine-tune job. @@ -188,13 +111,15 @@ The argument CALLBACK is execuated after request is made." (openai-request (format "https://api.openai.com/v1/fine-tunes/%s" fine-tune-id) :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-fine-tune-cancel (fine-tune-id callback) +(cl-defun openai-fine-tune-cancel ( fine-tune-id callback + &key + (key openai-key)) "Immediately cancel a fine-tune job. The FINE-TUNE-ID of the fine-tune job to cancel. @@ -203,13 +128,15 @@ The argument CALLBACK is execuated after request is made." (openai-request (format "https://api.openai.com/v1/fine-tunes/%s/cancel" fine-tune-id) :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-fine-tune-list-events (fine-tune-id callback) +(cl-defun openai-fine-tune-list-events ( fine-tune-id callback + &key + (key openai-key)) "Get fine-grained status updates for a fine-tune job. The FINE-TUNE-ID of the fine-tune job to get events for. @@ -218,13 +145,15 @@ The argument CALLBACK is execuated after request is made." (openai-request (format "https://api.openai.com/v1/fine-tunes/%s/events" fine-tune-id) :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-fine-tune-list-events (model callback) +(cl-defun openai-fine-tune-delete ( model callback + &key + (key openai-key)) "Delete a fine-tuned model. You must have the Owner role in your organization. The MODEL to delete. @@ -233,7 +162,7 @@ The argument CALLBACK is execuated after request is made." (openai-request (format "https://api.openai.com/v1/models/%s" model) :type "DELETE" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) @@ -266,28 +195,29 @@ The argument CALLBACK is execuated after request is made." "List fine-tuning jobs." (interactive) (setq openai-fine-tune-entries nil) ; reset - (openai-file-list (lambda (data) - (let ((id 0)) - (let-alist data - (mapc (lambda (fine-tune) - (let-alist fine-tune - (push (list (number-to-string id) - (vector .id - .object - .model - .created_at - .fine_tuned_model - .hyperparams - .organization_id - .result_files - .status - .validation_files - .training_files - .updated_at)) - openai-fine-tune-entries)) - (cl-incf id)) - .data))) - (openai-fine-tune-goto-ui)))) + (openai-fine-tune-list + (lambda (data) + (let ((id 0)) + (let-alist data + (mapc (lambda (fine-tune) + (let-alist fine-tune + (push (list (number-to-string id) + (vector .id + .object + .model + .created_at + .fine_tuned_model + .hyperparams + .organization_id + .result_files + .status + .validation_files + .training_files + .updated_at)) + openai-fine-tune-entries)) + (cl-incf id)) + .data))) + (openai-fine-tune-goto-ui)))) (provide 'openai-fine-tune) ;;; openai-fine-tune.el ends here diff --git a/openai-image.el b/openai-image.el index edd1ef7..1e2dfe2 100644 --- a/openai-image.el +++ b/openai-image.el @@ -28,76 +28,69 @@ (require 'openai) -(defcustom openai-image-n 1 - "The number of images to generate. Must be between 1 and 10." - :type 'integer - :group 'openai) - -(defcustom openai-image-size "1024x1024" - "The size of the generated images. - -Must be one of `256x256', `512x512', or `1024x1024'." - :type 'string - :group 'openai) - -(defcustom openai-image-response-format "url" - "The format in which the generated images are returned. - -Must be one of `url' or `b64_json'." - :type 'string - :group 'openai) - -(defcustom openai-image-mask nil - "An additional image whose fully transparent areas (e.g. where alpha is zero) -indicate where image should be edited. - -Must be a valid PNG file, less than 4MB, and have the same dimensions as image." - :type 'string - :group 'openai) - ;; ;;; API -(defun openai-image (query callback) - "Create image with QUERY. +(cl-defun openai-image ( prompt callback + &key + (key openai-key) + n + size + response-format + (user openai-user)) + "Send create image request. Argument CALLBACK is function with data pass in." (openai-request "https://api.openai.com/v1/images/generations" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("prompt" . ,query) - ("n" . ,openai-image-n) - ("size" . ,openai-image-size) - ("response_format" . ,openai-image-response-format) - ("user" . ,openai-user))) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("prompt" . ,prompt) + ("n" . ,n) + ("size" . ,size) + ("response_format" . ,response-format) + ("user" . ,user))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-image-edit (query callback) - "Create an edited or extended image given an original image and a QUERY. +(cl-defun openai-image-edit ( prompt callback + &key + (key openai-key) + mask + n + size + response-format + (user openai-user)) + "Create an edited or extended image given an original image. Argument CALLBACK is function with data pass in." (openai-request "https://api.openai.com/v1/images/edits" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("prompt" . ,query) - ("mask" . ,openai-image-mask) - ("n" . ,openai-image-n) - ("size" . ,openai-image-size) - ("response_format" . ,openai-image-response-format) - ("user" . ,openai-user))) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("prompt" . ,prompt) + ("mask" . ,mask) + ("n" . ,n) + ("size" . ,size) + ("response_format" . ,response-format) + ("user" . ,user))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-image-variation (image callback) +(cl-defun openai-image-variation ( image callback + &key + (key openai-key) + mask + n + size + response-format + (user openai-user)) "Create an edited or extended image given an original IMAGE. Argument CALLBACK is function with data pass in, and the argument IMAGE must be @@ -108,14 +101,14 @@ the mask." (openai-request "https://api.openai.com/v1/images/variations" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode `(("image" . ,image) - ("mask" . ,openai-image-mask) - ("n" . ,openai-image-n) - ("size" . ,openai-image-size) - ("response_format" . ,openai-image-response-format) - ("user" . ,openai-user))) + ("mask" . ,mask) + ("n" . ,n) + ("size" . ,size) + ("response_format" . ,response-format) + ("user" . ,user))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) @@ -132,6 +125,25 @@ the mask." ;; ;;; Application +(defcustom openai-image-n 1 + "The number of images to generate. Must be between 1 and 10." + :type 'integer + :group 'openai) + +(defcustom openai-image-size "1024x1024" + "The size of the generated images. + +Must be one of `256x256', `512x512', or `1024x1024'." + :type 'string + :group 'openai) + +(defcustom openai-image-response-format "url" + "The format in which the generated images are returned. + +Must be one of `url' or `b64_json'." + :type 'string + :group 'openai) + (defvar openai-image-entries nil "Async images entries.") @@ -142,11 +154,11 @@ the mask." nil) ;;;###autoload -(defun openai-image-prompt (query) - "Prompt to ask for image QUERY, and display result in a buffer." +(defun openai-image-prompt (prompt) + "Use PROMPT to ask for image, and display result in a buffer." (interactive (list (read-string "Describe image: "))) (setq openai-image-entries nil) - (openai-image query + (openai-image prompt (lambda (data) (let ((id 0)) (let-alist data @@ -157,14 +169,17 @@ the mask." openai-image-entries) (cl-incf id))) .data))) - (openai-image-goto-ui)))) + (openai-image-goto-ui) + :size openai-image-size + :n openai-image-n + :response-format openai-image-response-format))) ;;;###autoload -(defun openai-image-edit-prompt (query) - "Prompt to ask for image QUERY, and display result in a buffer." +(defun openai-image-edit-prompt (prompt) + "Use PROMPT to ask for image, and display result in a buffer." (interactive (list (read-string "Describe image: "))) (setq openai-image-entries nil) - (openai-image-edit query + (openai-image-edit prompt (lambda (data) (let ((id 0)) (let-alist data @@ -175,7 +190,10 @@ the mask." openai-image-entries) (cl-incf id))) .data))) - (openai-image-goto-ui)))) + (openai-image-goto-ui)) + :size openai-image-size + :n openai-image-n + :response-format openai-image-response-format)) ;;;###autoload (defun openai-image-variation-prompt (image) @@ -194,7 +212,10 @@ the mask." openai-image-entries) (cl-incf id))) .data))) - (openai-image-goto-ui)))) + (openai-image-goto-ui)) + :size openai-image-size + :n openai-image-n + :response-format openai-image-response-format)) (provide 'openai-image) ;;; openai-image.el ends here diff --git a/openai-model.el b/openai-model.el index f9f9a79..10e8710 100644 --- a/openai-model.el +++ b/openai-model.el @@ -29,23 +29,33 @@ ;; ;;; API -(defun openai-models (callback) - "Return models data and execute the CALLBACK." +(cl-defun openai-models ( callback + &key + (key openai-key)) + "Return models data and execute the CALLBACK. + +Arguments KEY is global options; however, you can overwrite the value by passing +it in." (openai-request "https://api.openai.com/v1/models" :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) (funcall callback data))))) -(defun openai-model (model callback) - "Return MODEL data and execute the CALLBACK." +(cl-defun openai-model ( model callback + &key + (key openai-key)) + "Return MODEL data and execute the CALLBACK. + +Arguments KEY is global options; however, you can overwrite the value by passing +it in." (openai-request (format "https://api.openai.com/v1/models/%s" model) :type "GET" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) + ("Authorization" . ,(concat "Bearer " key))) :parser 'json-read :success (cl-function (lambda (&key data &allow-other-keys) diff --git a/openai-moderation.el b/openai-moderation.el index da3aa64..d36fff6 100644 --- a/openai-moderation.el +++ b/openai-moderation.el @@ -29,33 +29,27 @@ (require 'openai) -(defcustom openai-moderation-model "text-moderation-latest" - "Two content moderations models are available: `text-moderation-stable' and -`text-moderation-latest'. - -The default is text-moderation-latest which will be automatically upgraded over -time. This ensures you are always using our most accurate model. If you use -`text-moderation-stable', we will provide advanced notice before updating the -model. Accuracy of `text-moderation-stable' may be slightly lower than for -`text-moderation-latest'." - :type 'string - :group 'openai) - ;; ;;; API -(defun openai-moderation-create (input callback) +(cl-defun openai-moderation-create ( input callback + &key + (key openai-key) + (model "text-moderation-latest")) "Classifies if text violates OpenAI's Content Policy. Argument INPUT is the text to classify. -The argument CALLBACK is execuated after request is made." +The argument CALLBACK is execuated after request is made. + +Arguments KEY is global options; however, you can overwrite the value by passing +it in." (openai-request "https://api.openai.com/v1/embeddings" :type "POST" :headers `(("Content-Type" . "application/json") - ("Authorization" . ,(concat "Bearer " openai-key))) - :data (json-encode - `(("model" . ,openai-moderation-model) + ("Authorization" . ,(concat "Bearer " key))) + :data (openai--json-encode + `(("model" . ,model) ("input" . ,input))) :parser 'json-read :success (cl-function diff --git a/openai.el b/openai.el index 574131f..0307828 100644 --- a/openai.el +++ b/openai.el @@ -35,6 +35,7 @@ (require 'let-alist) (require 'pcase) (require 'pp) +(require 'json) (require 'request) (require 'tblui) @@ -45,16 +46,24 @@ :group 'comm :link '(url-link :tag "Repository" "https://github.com/emacs-openai/openai")) -(defcustom openai-key "" - "Generated API key." - :type 'list - :group 'openai) +;; +;;; Request + +(defvar openai-key "" + "Generated API key.") -(defcustom openai-user "" +(defvar openai-user "" "A unique identifier representing your end-user, which can help OpenAI to -monitor and detect abuse." - :type 'string - :group 'openai) +monitor and detect abuse.") + +(defun openai--json-encode (object) + "Wrapper for function `json-encode' but it removes `nil' value before +constructing JSON data. + +The argument OBJECT is an alist that can be construct to JSON data; see function +`json-encode' for the detials." + (let ((object (cl-remove-if-not (lambda (pair) (cdr pair)) object))) + (json-encode object))) (defun openai--handle-error (response) "Handle error status code from the RESPONSE.