From c34255882efc79eb11cfbc5e1df3ed459c3d2581 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 20 Jun 2024 14:07:47 +0200 Subject: [PATCH] Consider color palette when generating SVG variations #78 --- notebooks/svg_variations_ui_widget.ipynb | 456 ++++++++++++++++++++++- src/penai/models.py | 3 + src/penai/variations/svg_variations.py | 28 +- 3 files changed, 475 insertions(+), 12 deletions(-) diff --git a/notebooks/svg_variations_ui_widget.ipynb b/notebooks/svg_variations_ui_widget.ipynb index 91ddd48..45e294e 100644 --- a/notebooks/svg_variations_ui_widget.ipynb +++ b/notebooks/svg_variations_ui_widget.ipynb @@ -18,12 +18,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 18, "id": "793ea6ad30db8951", "metadata": { "ExecuteTime": { - "end_time": "2024-06-18T08:40:23.486556400Z", - "start_time": "2024-06-18T08:40:20.457429900Z" + "end_time": "2024-06-20T12:03:18.733454100Z", + "start_time": "2024-06-20T12:03:18.652921600Z" } }, "outputs": [], @@ -32,7 +32,7 @@ "\n", "from penai.registries.projects import SavedPenpotProject\n", "from penai.llm.llm_model import RegisteredLLM\n", - "from penai.variations.svg_variations import SVGVariationsGenerator, VariationsPromptBuilder" + "from penai.variations.svg_variations import SVGVariationsGenerator, VariationInstructionSnippet, VariationDescriptionSequence" ] }, { @@ -47,19 +47,56 @@ }, { "cell_type": "code", - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO 2024-06-20 14:02:40,448 penai.registries.projects:get_path:48 - Pulling data for project Generative variations to C:\\Users\\DominikJain\\Dev\\penpot\\penai\\data\\raw\\designs\\Generative variations\n", + "INFO 2024-06-20 14:02:40,449 accsr.remote_storage:bucket:461 - Establishing connection to bucket penpot\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Scanning remote paths in penpot/data/raw/designs/Generative variations: 100%|██████████| 4/4 [00:00<00:00, 800.02it/s]\n", + "force pulling (bytes): 0it [00:00, ?it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO 2024-06-20 14:02:41,691 sensai.util.cache:cached:670 - Loading cached result of function 'load_page_svg_text' from C:\\Users\\DominikJain\\Dev\\penpot\\penai\\temp\\cache\\SavedPenpotProject.load_page_svg_with_viewboxes.load_page_svg_text-7e0e33747955629084b605e3ae91dd0fe18a4888.cache.pickle\n" + ] + } + ], "source": [ - "page_svg = SavedPenpotProject.GENERATIVE_VARIATIONS.load_page_svg_with_viewboxes(\"examples\")" + "saved_project = SavedPenpotProject.GENERATIVE_VARIATIONS\n", + "project = saved_project.load(pull=True)\n", + "main_file = project.get_main_file()\n", + "page_svg = saved_project.load_page_svg_with_viewboxes(\"examples\")" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-06-18T08:40:23.799398600Z", - "start_time": "2024-06-18T08:40:23.487551100Z" + "end_time": "2024-06-20T12:02:41.962395800Z", + "start_time": "2024-06-20T12:02:40.365022800Z" } }, "id": "7f41585019d4f27d", - "execution_count": 3 + "execution_count": 16 + }, + { + "cell_type": "markdown", + "source": [ + "## Generating Variations Depending on UI States" + ], + "metadata": { + "collapsed": false + }, + "id": "ea8bb1f54f860125" }, { "cell_type": "code", @@ -409,10 +446,407 @@ "shape = page_svg.get_shape_by_name(\"Dark / Input / Rest\", require_unique=False)\n", "var_gen = SVGVariationsGenerator(shape=shape, semantics=\"input field\", model=RegisteredLLM.GPT4O)\n", "\n", - "variations = var_gen.create_variations_sequentially()\n", + "variations = var_gen.create_variations_sequentially(\n", + " variation_scope=VariationInstructionSnippet.SPECIFIC_COLORS_SHAPES,\n", + " variation_description_sequence=VariationDescriptionSequence.UI_ELEMENT_STATES)\n", "HTML(variations.to_html())" ] }, + { + "cell_type": "markdown", + "source": [ + "## Providing Context on Color Palette" + ], + "metadata": { + "collapsed": false + }, + "id": "e067c526e558256" + }, + { + "cell_type": "code", + "outputs": [ + { + "data": { + "text/plain": "[PenpotColor(id='542a8be7-f427-80a2-8004-872ecc9580ab', name='foreground', color='#8f9da3', opacity=1.0, path=''),\n PenpotColor(id='542a8be7-f427-80a2-8004-872fca9949f0', name='background-primary', color='#18181a', opacity=1.0, path=''),\n PenpotColor(id='542a8be7-f427-80a2-8004-872f492b92b4', name='accent-secondary', color='#00d1b8', opacity=1.0, path=''),\n PenpotColor(id='542a8be7-f427-80a2-8004-87287dcf48de', name='secondary-blue', color='#40a9ff', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-8729e89f2565', name='gray', color='#dbdbdb', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-872f7ce0f6f4', name='error', color='#ff3277', opacity=1.0, path=''),\n PenpotColor(id='542a8be7-f427-80a2-8004-8729d5faddb4', name='black', color='#000000', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-872c0120f650', name='primary-red', color='#f5222d', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-872f317a8d30', name='background-tertiary', color='#2e3434', opacity=1.0, path=''),\n PenpotColor(id='542a8be7-f427-80a2-8004-8729ceedaef8', name='white', color='#ffffff', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-872c0b629972', name='secondary-red', color='#ff4d4f', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-872f15157857', name='accent-primary', color='#7efff5', opacity=1.0, path=''),\n PenpotColor(id='542a8be7-f427-80a2-8004-872869a74f12', name='primary-blue', color='#1890ff', opacity=1.0, path='Basics'),\n PenpotColor(id='542a8be7-f427-80a2-8004-872fdaadf94f', name='background-secondary', color='#212426', opacity=1.0, path='')]" + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "main_file.colors.get_colors()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-06-20T12:06:18.281604300Z", + "start_time": "2024-06-20T12:06:18.209290100Z" + } + }, + "id": "2cce49b6fda27790", + "execution_count": 21 + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO 2024-06-20 14:04:46,931 httpx:_send_single_request:1026 - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", + "To refactor the SVG and make the shapes explicit using the respective shape tags (`rect`, `circle`, `ellipse`, etc.), we need to replace the `path` elements with the appropriate shape elements. Additionally, we need to ensure that any cutouts or masks are preserved. Here is the refactored SVG:\n", + "\n", + "```xml\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " Label\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "```\n", + "\n", + "In this refactored SVG:\n", + "- The main rectangle for the input field is represented by a `rect` element with rounded corners.\n", + "- The smaller rectangle inside the input field is also represented by a `rect` element.\n", + "- The complex path inside the smaller rectangle is left as a `path` element because it does not correspond to a simple shape like a `rect`, `circle`, or `ellipse`.\n", + "- The text element is preserved as it is, with the necessary attributes and styles.\n", + "\n", + "This refactoring makes the SVG more readable and semantically clear by using explicit shape tags where applicable.\n", + "INFO 2024-06-20 14:05:17,187 httpx:_send_single_request:1026 - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", + "## Adapted for UI State 'Active'\n", + "\n", + "```xml\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " Label\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "```\n", + "\n", + "In this variation:\n", + "- The border color of the main rectangle is changed to `#00d1b8` to indicate the active state.\n", + "- The stroke width is increased to `2` to make the active state more prominent.\n", + "INFO 2024-06-20 14:05:47,361 httpx:_send_single_request:1026 - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", + "## Adapted for UI State 'Disabled'\n", + "\n", + "```xml\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " Label\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "```\n", + "\n", + "In this variation:\n", + "- The background color of the main rectangle is changed to `#2e3434` to indicate the disabled state.\n", + "- The border color of the main rectangle is changed to `#dbdbdb`.\n", + "- The text color is changed to `#dbdbdb` to indicate that the text is disabled.\n", + "- The stroke width is kept at `1` to maintain a subtle appearance for the disabled state.\n", + "INFO 2024-06-20 14:06:18,194 httpx:_send_single_request:1026 - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", + "## Adapted for UI State 'Error'\n", + "\n", + "```xml\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " Label\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "```\n", + "\n", + "In this variation:\n", + "- The border color of the main rectangle is changed to `#ff3277` to indicate the error state.\n", + "- The stroke width is increased to `2` to make the error state more prominent.\n", + "- The background color remains the same to maintain consistency with the original design.\n", + "INFO 2024-06-20 14:06:18,198 sensai.util.io.ResultWriter:write_text_file:14 - Saving full conversation to C:\\Users\\DominikJain\\Dev\\penpot\\penai\\results\\svg_variations\\Dark Input Rest\\20240618-172150\\full_conversation.txt\n", + "INFO 2024-06-20 14:06:18,199 sensai.util.io.ResultWriter:write_text_file:14 - Saving variations response as HTML to C:\\Users\\DominikJain\\Dev\\penpot\\penai\\results\\svg_variations\\Dark Input Rest\\20240618-172150\\variations.html\n", + "INFO 2024-06-20 14:06:18,200 sensai.util.io.ResultWriter:write_text_file:14 - Saving variation 'Adapted for UI State 'Active'' as SVG to C:\\Users\\DominikJain\\Dev\\penpot\\penai\\results\\svg_variations\\Dark Input Rest\\20240618-172150\\variation_1.svg\n", + "INFO 2024-06-20 14:06:18,202 sensai.util.io.ResultWriter:write_text_file:14 - Saving variation 'Adapted for UI State 'Disabled'' as SVG to C:\\Users\\DominikJain\\Dev\\penpot\\penai\\results\\svg_variations\\Dark Input Rest\\20240618-172150\\variation_2.svg\n", + "INFO 2024-06-20 14:06:18,203 sensai.util.io.ResultWriter:write_text_file:14 - Saving variation 'Adapted for UI State 'Error'' as SVG to C:\\Users\\DominikJain\\Dev\\penpot\\penai\\results\\svg_variations\\Dark Input Rest\\20240618-172150\\variation_3.svg\n" + ] + }, + { + "data": { + "text/plain": "", + "text/html": "

Original

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Label\n \n \n \n \n \n \n \n \n \n \n

Variations

Adapted for UI State 'Active'

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Label\n \n \n \n \n \n \n \n \n \n

Adapted for UI State 'Disabled'

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Label\n \n \n \n \n \n \n \n \n \n

Adapted for UI State 'Error'

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Label\n \n \n \n \n \n \n \n \n \n
" + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "variations_col = var_gen.create_variations_sequentially(\n", + " variation_scope=VariationInstructionSnippet.SPECIFIC_COLORS_SHAPES,\n", + " variation_description_sequence=VariationDescriptionSequence.UI_ELEMENT_STATES,\n", + " colors=main_file.colors)\n", + "HTML(variations_col.to_html())" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-06-20T12:06:18.209290100Z", + "start_time": "2024-06-20T12:04:08.127061700Z" + } + }, + "id": "23e0598fbebf65cc", + "execution_count": 20 + }, { "cell_type": "code", "outputs": [], @@ -420,7 +854,7 @@ "metadata": { "collapsed": false }, - "id": "23e0598fbebf65cc" + "id": "ac8b96ae520d9a22" } ], "metadata": { diff --git a/src/penai/models.py b/src/penai/models.py index 1ba9a25..3affe72 100644 --- a/src/penai/models.py +++ b/src/penai/models.py @@ -158,6 +158,9 @@ class PenpotColor(BaseModel): id: str | None = Field(default_factory=lambda: None) name: str color: str + """ + The color in hex format, e.g. '#ff0000' for red. + """ opacity: float path: str diff --git a/src/penai/variations/svg_variations.py b/src/penai/variations/svg_variations.py index 898d2a3..016a6f6 100644 --- a/src/penai/variations/svg_variations.py +++ b/src/penai/variations/svg_variations.py @@ -11,6 +11,7 @@ from penai.config import get_config from penai.llm.conversation import Conversation, Response from penai.llm.llm_model import RegisteredLLM +from penai.models import PenpotColors from penai.svg import SVG, PenpotShapeElement from penai.types import PathLike from penai.utils.io import ResultWriter, fn_compatible @@ -245,24 +246,49 @@ def create_variations( ) return self.create_variations_for_prompt(prompt) + def _create_colors_prompt(self, penpot_colors: PenpotColors) -> str: + colors = penpot_colors.get_colors() + prompt = "" + if len(colors) > 0: + prompt += "The design uses the following colors:\n" + for color in colors: + prompt += f"{color.name}: {color.color}\n" + prompt += ( + "In the SVGs you create, use these colors where applicable " + "and make sure that any additional colors fit well with the existing color scheme." + ) + return prompt + + def _create_variation_scope_prompt( + self, variation_scope: VariationInstructionSnippet | str, colors: PenpotColors | None = None + ) -> str: + prompt = str(variation_scope) + if colors is not None: + colors_prompt = self._create_colors_prompt(colors) + if colors_prompt != "": + prompt += "\n" + colors_prompt + return prompt + def create_variations_sequentially( self, variation_scope: VariationInstructionSnippet | str = VariationInstructionSnippet.SPECIFIC_COLORS_SHAPES, variation_description_sequence: VariationDescriptionSequence | Sequence[str] = VariationDescriptionSequence.UI_ELEMENT_STATES, + colors: PenpotColors | None = None, ) -> SVGVariations: """Generates variations sequentially, one at a time, accounting for limitations in response token count (~4K for GPT-4o, which is not enough for multiple variations at once). :param variation_scope: describes the scope of variations to apply in generation :param variation_description_sequence: a sequence of instructions describing what to do for each variation + :param colors: the colors used in the design, which shall be considered in the generation process :return: the variations """ conversation = self._create_conversation() conversation.query(self.get_svg_refactoring_prompt()) - variation_scope_prompt = str(variation_scope) + variation_scope_prompt = self._create_variation_scope_prompt(variation_scope, colors) initial_variation_query = ( "In the following, your task is to create variations of the SVG, one variation at a time. "