From 5b0485460c5827f2f2801dc5f3394e3635f7503a Mon Sep 17 00:00:00 2001 From: Alexey Date: Mon, 8 Jul 2024 13:51:50 -0500 Subject: [PATCH 1/2] Better support for stop sequences Show the stop reason in summary Add options stop= to client that accepts str or list[str] Add use examples --- 00_core.ipynb | 85 ++++++++++++++++++++++++++++++++++++++++++-- claudette/_modidx.py | 26 -------------- claudette/core.py | 34 ++++++++++-------- settings.ini | 6 ++-- 4 files changed, 105 insertions(+), 46 deletions(-) diff --git a/00_core.ipynb b/00_core.ipynb index 8eb760e..8b20a18 100644 --- a/00_core.ipynb +++ b/00_core.ipynb @@ -875,7 +875,7 @@ { "cell_type": "code", "execution_count": null, - "id": "be54737f", + "id": "8015d3f3", "metadata": {}, "outputs": [], "source": [ @@ -888,6 +888,8 @@ " blk.text = prefill + (blk.text or '')\n", " self.result = r\n", " self.use += r.usage\n", + " self.stop_reason = r.stop_reason\n", + " self.stop_sequence = r.stop_sequence\n", " return r" ] }, @@ -958,7 +960,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5fb96c43", + "id": "835638bb", "metadata": {}, "outputs": [], "source": [ @@ -972,10 +974,14 @@ " maxtok=4096, # Maximum tokens\n", " prefill='', # Optional prefill to pass to Claude as start of its response\n", " stream:bool=False, # Stream response?\n", + " stop=None, # Stop sequence\n", " **kwargs):\n", " \"Make a call to Claude.\"\n", " pref = [prefill.strip()] if prefill else []\n", " if not isinstance(msgs,list): msgs = [msgs]\n", + " if stop is not None:\n", + " if not isinstance(stop, (list)): stop = [stop]\n", + " kwargs[\"stop_sequences\"] = stop\n", " msgs = mk_msgs(msgs+pref)\n", " if stream: return self._stream(msgs, prefill=prefill, max_tokens=maxtok, system=sp, temperature=temp, **kwargs)\n", " res = self.c.messages.create(\n", @@ -1224,6 +1230,79 @@ "c.use" ] }, + { + "cell_type": "markdown", + "id": "bd556e49", + "metadata": {}, + "source": [ + "Pass a stop seauence if you want claude to stop generating text when it encounters it.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12b17591", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "1, 2, 3, 4,\n", + "\n", + "
\n", + "\n", + "- id: msg_01YGeu3qQNoNwafG66kB8V51\n", + "- content: [{'text': '1, 2, 3, 4, ', 'type': 'text'}]\n", + "- model: claude-3-haiku-20240307\n", + "- role: assistant\n", + "- stop_reason: stop_sequence\n", + "- stop_sequence: 5\n", + "- type: message\n", + "- usage: {'input_tokens': 15, 'output_tokens': 14}\n", + "\n", + "
" + ], + "text/plain": [ + "Message(id='msg_01YGeu3qQNoNwafG66kB8V51', content=[TextBlock(text='1, 2, 3, 4, ', type='text')], model='claude-3-haiku-20240307', role='assistant', stop_reason='stop_sequence', stop_sequence='5', type='message', usage=In: 15; Out: 14; Total: 29)" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c(\"Count from 1 to 10\", stop=\"5\")" + ] + }, + { + "cell_type": "markdown", + "id": "cdbd3df3", + "metadata": {}, + "source": [ + "This also works with streaming, and you can pass more than one stop sequence:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff50577d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1, stop_sequence 2\n" + ] + } + ], + "source": [ + "for o in c(\"Count from 1 to 10\", stop=[\"2\", \"yellow\"], stream=True):\n", + " print(o, end='')\n", + "print(c.stop_reason, c.stop_sequence)" + ] + }, { "cell_type": "markdown", "id": "1a7cdbc6", @@ -1442,7 +1521,7 @@ " func = getattr(obj, fc.name, None)\n", " if not func: func = ns[fc.name]\n", " res = func(**fc.input)\n", - " return dict(type=\"tool_result\", tool_use_id=fc.id, content=str(res)) " + " return dict(type=\"tool_result\", tool_use_id=fc.id, content=str(res))" ] }, { diff --git a/claudette/_modidx.py b/claudette/_modidx.py index 0797160..3b2fefb 100644 --- a/claudette/_modidx.py +++ b/claudette/_modidx.py @@ -31,30 +31,4 @@ 'claudette.core.mk_toolres': ('core.html#mk_toolres', 'claudette/core.py'), 'claudette.core.text_msg': ('core.html#text_msg', 'claudette/core.py'), 'claudette.core.usage': ('core.html#usage', 'claudette/core.py')}, - 'claudette.core2': { 'claudette.core2.Chat': ('core.html#chat', 'claudette/core2.py'), - 'claudette.core2.Chat.__call__': ('core.html#chat.__call__', 'claudette/core2.py'), - 'claudette.core2.Chat.__init__': ('core.html#chat.__init__', 'claudette/core2.py'), - 'claudette.core2.Chat._stream': ('core.html#chat._stream', 'claudette/core2.py'), - 'claudette.core2.Chat.use': ('core.html#chat.use', 'claudette/core2.py'), - 'claudette.core2.Client': ('core.html#client', 'claudette/core2.py'), - 'claudette.core2.Client.__call__': ('core.html#client.__call__', 'claudette/core2.py'), - 'claudette.core2.Client.__init__': ('core.html#client.__init__', 'claudette/core2.py'), - 'claudette.core2.Client._r': ('core.html#client._r', 'claudette/core2.py'), - 'claudette.core2.Client._stream': ('core.html#client._stream', 'claudette/core2.py'), - 'claudette.core2.Message._repr_markdown_': ('core.html#message._repr_markdown_', 'claudette/core2.py'), - 'claudette.core2.Usage.__add__': ('core.html#usage.__add__', 'claudette/core2.py'), - 'claudette.core2.Usage.__repr__': ('core.html#usage.__repr__', 'claudette/core2.py'), - 'claudette.core2.Usage.total': ('core.html#usage.total', 'claudette/core2.py'), - 'claudette.core2._mk_content': ('core.html#_mk_content', 'claudette/core2.py'), - 'claudette.core2._mk_ns': ('core.html#_mk_ns', 'claudette/core2.py'), - 'claudette.core2.call_func': ('core.html#call_func', 'claudette/core2.py'), - 'claudette.core2.contents': ('core.html#contents', 'claudette/core2.py'), - 'claudette.core2.find_block': ('core.html#find_block', 'claudette/core2.py'), - 'claudette.core2.img_msg': ('core.html#img_msg', 'claudette/core2.py'), - 'claudette.core2.mk_msg': ('core.html#mk_msg', 'claudette/core2.py'), - 'claudette.core2.mk_msgs': ('core.html#mk_msgs', 'claudette/core2.py'), - 'claudette.core2.mk_tool_choice': ('core.html#mk_tool_choice', 'claudette/core2.py'), - 'claudette.core2.mk_toolres': ('core.html#mk_toolres', 'claudette/core2.py'), - 'claudette.core2.text_msg': ('core.html#text_msg', 'claudette/core2.py'), - 'claudette.core2.usage': ('core.html#usage', 'claudette/core2.py')}, 'claudette.toolloop': {'claudette.toolloop.Chat.toolloop': ('toolloop.html#chat.toolloop', 'claudette/toolloop.py')}}} diff --git a/claudette/core.py b/claudette/core.py index d3c2e91..bdd6dc2 100644 --- a/claudette/core.py +++ b/claudette/core.py @@ -96,6 +96,8 @@ def _r(self:Client, r:Message, prefill=''): blk.text = prefill + (blk.text or '') self.result = r self.use += r.usage + self.stop_reason = r.stop_reason + self.stop_sequence = r.stop_sequence return r # %% ../00_core.ipynb 66 @@ -116,10 +118,14 @@ def __call__(self:Client, maxtok=4096, # Maximum tokens prefill='', # Optional prefill to pass to Claude as start of its response stream:bool=False, # Stream response? + stop=None, # Stop sequence **kwargs): "Make a call to Claude." pref = [prefill.strip()] if prefill else [] if not isinstance(msgs,list): msgs = [msgs] + if stop is not None: + if not isinstance(stop, (list)): stop = [stop] + kwargs["stop_sequences"] = stop msgs = mk_msgs(msgs+pref) if stream: return self._stream(msgs, prefill=prefill, max_tokens=maxtok, system=sp, temperature=temp, **kwargs) res = self.c.messages.create( @@ -127,17 +133,17 @@ def __call__(self:Client, self._r(res, prefill) return self.result -# %% ../00_core.ipynb 84 +# %% ../00_core.ipynb 88 def mk_tool_choice(choose:Union[str,bool,None])->dict: "Create a `tool_choice` dict that's 'auto' if `choose` is `None`, 'any' if it is True, or 'tool' otherwise" return {"type": "tool", "name": choose} if isinstance(choose,str) else {'type':'any'} if choose else {'type':'auto'} -# %% ../00_core.ipynb 94 +# %% ../00_core.ipynb 98 def _mk_ns(*funcs:list[callable]) -> dict[str,callable]: "Create a `dict` of name to function in `funcs`, to use as a namespace" return {f.__name__:f for f in funcs} -# %% ../00_core.ipynb 96 +# %% ../00_core.ipynb 100 def call_func(fc:ToolUseBlock, # Tool use block from Claude's message ns:Optional[abc.Mapping]=None, # Namespace to search for tools, defaults to `globals()` obj:Optional=None # Object to search for tools @@ -148,9 +154,9 @@ def call_func(fc:ToolUseBlock, # Tool use block from Claude's message func = getattr(obj, fc.name, None) if not func: func = ns[fc.name] res = func(**fc.input) - return dict(type="tool_result", tool_use_id=fc.id, content=str(res)) + return dict(type="tool_result", tool_use_id=fc.id, content=str(res)) -# %% ../00_core.ipynb 99 +# %% ../00_core.ipynb 103 def mk_toolres( r:abc.Mapping, # Tool use request response from Claude ns:Optional[abc.Mapping]=None, # Namespace to search for tools @@ -163,7 +169,7 @@ def mk_toolres( if tcs: res.append(mk_msg(tcs)) return res -# %% ../00_core.ipynb 109 +# %% ../00_core.ipynb 113 class Chat: def __init__(self, model:Optional[str]=None, # Model to use (leave empty if passing `cli`) @@ -179,13 +185,13 @@ def __init__(self, @property def use(self): return self.c.use -# %% ../00_core.ipynb 112 +# %% ../00_core.ipynb 116 @patch def _stream(self:Chat, res): yield from res self.h += mk_toolres(self.c.result, ns=self.tools, obj=self) -# %% ../00_core.ipynb 113 +# %% ../00_core.ipynb 117 @patch def __call__(self:Chat, pr=None, # Prompt / message @@ -204,7 +210,7 @@ def __call__(self:Chat, self.h += mk_toolres(self.c.result, ns=self.tools, obj=self) return res -# %% ../00_core.ipynb 132 +# %% ../00_core.ipynb 136 def img_msg(data:bytes)->dict: "Convert image `data` into an encoded `dict`" img = base64.b64encode(data).decode("utf-8") @@ -212,19 +218,19 @@ def img_msg(data:bytes)->dict: r = dict(type="base64", media_type=mtype, data=img) return {"type": "image", "source": r} -# %% ../00_core.ipynb 134 +# %% ../00_core.ipynb 138 def text_msg(s:str)->dict: "Convert `s` to a text message" return {"type": "text", "text": s} -# %% ../00_core.ipynb 138 +# %% ../00_core.ipynb 142 def _mk_content(src): "Create appropriate content data structure based on type of content" if isinstance(src,str): return text_msg(src) if isinstance(src,bytes): return img_msg(src) return src -# %% ../00_core.ipynb 141 +# %% ../00_core.ipynb 145 def mk_msg(content, # A string, list, or dict containing the contents of the message role='user', # Must be 'user' or 'assistant' **kw): @@ -235,9 +241,9 @@ def mk_msg(content, # A string, list, or dict containing the contents of the mes content = [_mk_content(o) for o in content] if content else '.' return dict(role=role, content=content, **kw) -# %% ../00_core.ipynb 148 +# %% ../00_core.ipynb 152 models_aws = ('anthropic.claude-3-haiku-20240307-v1:0', 'anthropic.claude-3-sonnet-20240229-v1:0', 'anthropic.claude-3-opus-20240229-v1:0', 'anthropic.claude-3-5-sonnet-20240620-v1:0') -# %% ../00_core.ipynb 154 +# %% ../00_core.ipynb 158 models_goog = 'claude-3-haiku@20240307', 'claude-3-sonnet@20240229', 'claude-3-opus@20240229', 'claude-3-5-sonnet@20240620' diff --git a/settings.ini b/settings.ini index e35deed..02bb27e 100644 --- a/settings.ini +++ b/settings.ini @@ -6,7 +6,7 @@ min_python = 3.8 license = apache2 black_formatting = False requirements = fastcore>=1.5.33 anthropic toolslm -dev_requirements = google-auth +dev_requirements = google-auth boto3 doc_path = _docs lib_path = claudette nbs_path = . @@ -29,8 +29,8 @@ language = English status = 3 user = AnswerDotAI readme_nb = index.ipynb -allowed_metadata_keys = -allowed_cell_metadata_keys = +allowed_metadata_keys = +allowed_cell_metadata_keys = jupyter_hooks = True clean_ids = True clear_all = False From 959fcbc1cd988702973af3cf23cb28aad43ee4e8 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Tue, 30 Jul 2024 15:38:59 +1000 Subject: [PATCH 2/2] Update settings.ini --- settings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.ini b/settings.ini index 02bb27e..833ca31 100644 --- a/settings.ini +++ b/settings.ini @@ -6,7 +6,7 @@ min_python = 3.8 license = apache2 black_formatting = False requirements = fastcore>=1.5.33 anthropic toolslm -dev_requirements = google-auth boto3 +dev_requirements = google-auth doc_path = _docs lib_path = claudette nbs_path = .