diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e192759..a9c70f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ on: # cancel workflow if there is already one running concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-{github.event_name}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -63,11 +63,7 @@ jobs: with: context: . file: ./Dockerfile - push: true # ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }} - platforms: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + push: true + platforms: 'linux/amd64,linux/arm64' tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - - diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d8bb725 --- /dev/null +++ b/.gitignore @@ -0,0 +1,373 @@ +### Java template +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +replay_pid* + +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Linux template +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Gradle template +.gradle +**/build/ +!src/**/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Avoid ignore Gradle wrappper properties +!gradle-wrapper.properties + +# Cache of project +.gradletasknamecache + +# Eclipse Gradle plugin generated files +# Eclipse Core +.project +# JDT-specific (Eclipse Java Development Tools) +.classpath + +### Windows template +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +### JupyterNotebooks template +# gitignore template for Jupyter Notebooks +# website: http://jupyter.org/ + +.ipynb_checkpoints +*/.ipynb_checkpoints/* + +# IPython +profile_default/ +ipython_config.py + +# Remove previous ipynb_checkpoints +# git rm -r .ipynb_checkpoints/ + +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + diff --git a/Dockerfile b/Dockerfile index 1855ef5..70a38ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,13 @@ FROM ubuntu:20.04 # Install Java RUN apt-get update && apt-get install -y \ openjdk-17-jdk \ + && rm -rf /var/lib/apt/lists/* \ && java -version # Install Git -RUN apt-get install -y \ +RUN apt-get update && apt-get install -y \ git \ + && rm -rf /var/lib/apt/lists/* \ && git --version # Install Python3 and pip3 @@ -16,14 +18,25 @@ RUN apt-get update && apt-get install -y \ python3-pip \ python3-setuptools \ python3-wheel \ + && rm -rf /var/lib/apt/lists/* \ && python3 --version +# Install rsync and tmux +RUN apt-get update && apt-get install -y \ + rsync \ + tmux \ + && rm -rf /var/lib/apt/lists/* \ + && rsync --version \ + && tmux -V + # Disable gradle daemon ENV GRADLE_OPTS "-Dorg.gradle.daemon=false" # Copy the plugin COPY ide-former-plugin /ide-former-plugin +EXPOSE 8080 + # prebuild the plugin with gradle RUN cd /ide-former-plugin && ./gradlew buildPlugin diff --git a/ide-former-plugin/runPluginStarter.sh b/ide-former-plugin/runPluginStarter.sh index 53e18c4..9f99def 100755 --- a/ide-former-plugin/runPluginStarter.sh +++ b/ide-former-plugin/runPluginStarter.sh @@ -12,7 +12,7 @@ if [ $# -gt 3 ]; then echo "Illegal number of parameters" echo "Usage: runPluginStarter.sh " echo "\t - path to the project to be opened in IDE, default is the plugin itself" - echo "\t - host of the server to connect to, default is localhost" + echo "\t - host of the server to connect to, default is 0.0.0.0" echo "\t - port of the server to connect to, default is 8080" echo "" echo "Examples:" @@ -30,8 +30,8 @@ else fi if [ -z "$2" ]; then - echo "No server host specified. Using 127.0.0.1" - SERVER_HOST="127.0.0.1" + echo "No server host specified. Using 0.0.0.0" + SERVER_HOST="0.0.0.0" else SERVER_HOST=$2 fi diff --git a/ide_former_simulation/__init__.py b/ide_former_simulation/__init__.py new file mode 100644 index 0000000..4bb4dfe --- /dev/null +++ b/ide_former_simulation/__init__.py @@ -0,0 +1,4 @@ +from .prompt_helpers import tee, ColorCodes +from .docker_session import docker_session + +__all__ = ['tee', 'ColorCodes', 'docker_session'] \ No newline at end of file diff --git a/ide_former_simulation/docker_session.py b/ide_former_simulation/docker_session.py new file mode 100644 index 0000000..4e51d75 --- /dev/null +++ b/ide_former_simulation/docker_session.py @@ -0,0 +1,387 @@ +import random +import re +from contextlib import contextmanager +from dataclasses import dataclass +from os import PathLike +from pathlib import Path +from time import sleep +from typing import List, Dict, Union + +import docker +from docker import DockerClient +from docker.models.containers import Container + + +@dataclass +class ShelOutput: + """Represents the output of a shell command execution. + + :param output: The output of the shell command. + :type output: str + :param exit_code: The exit code of the shell command. + :type exit_code: int + """ + output: str + exit_code: int + + # make class behave like a string + def __getattr__(self, attr): + return getattr(self.output, attr) + + def __str__(self): + return self.output + + +class DockerInterface: + """ Docker interface to execute commands in a container""" + _node_regex: re.Pattern = re.compile(r'@.*:') + + container: Container = None + container_create_configuration = None + + _interactive_session_name: str = None + + def __init__(self, docker_client: DockerClient=None): + """ Docker interface to execute commands in a container + + :param docker_client: An optional Docker client instance to use. If not provided, a new client will be + created using the default configuration. + """ + self.container = None + self.container_create_configuration = None + self._interactive_session_name = None + + self.docker_client = docker_client or docker.from_env() + + def initialize(self, image: str = None, command: Union[str, List[str]] = None, + working_dir: PathLike = None, ports: Dict[int, int] = None, + volumes: Union[Dict[Union[str, Path], Dict], List[str]] = None, + environment: Dict[str, str] = None, user: str = None, + interactive: bool = False, interactive_interpreter: str = None, + destroy_if_exists: bool = False): + """ + Initializes the container for the DockerManager. + + :param image: The name or ID of the Docker image to use. Defaults to 'ubuntu' if not provided. + :param command: The command to execute inside the container. Defaults to '/bin/sh' if not provided. + :param working_dir: The working directory inside the container. Defaults to '/' if not provided. + :param ports: A dictionary of ports to expose inside the container. The key is the port inside the container + and the value is the port on the host machine. + :param volumes: Either a dictionary of volume configurations or a list of volume names. Defaults to None if + not provided. + :param environment: A dictionary of environment variables to set in the container. Defaults to None if not + provided. + :param user: The user to run the container as. Defaults to 'root' if not provided. + :param interactive: Whether to start an interactive session inside the container. Defaults to False if not + provided. + :param interactive_interpreter: The interpreter to use in the interactive session. Defaults to None if not + provided. + :param destroy_if_exists: Whether to destroy the existing container if one is already initialized. Defaults + to False if not provided. + :param docker_client: An optional Docker client instance to use. If not provided, a new client will be + created using the default configuration. + :return: None + """ + image = image or 'ubuntu' + working_dir = working_dir or '/' + command = command or '/bin/sh' + user = user or 'root' + + self.container_create_configuration = {'image': image, 'working_dir': working_dir, 'command': command, + 'ports': ports, 'volumes': volumes, 'environment': environment, + 'user': user} + + if isinstance(volumes, dict): + # filter out volumes with mode 'cp' + # (additional mode that copies the contents of the volume to the container) + copy_volumes = {k: v for k, v in volumes.items() if v.get('mode', 'rw') == 'cp'} + volumes = {k: v for k, v in volumes.items() if v.get('mode', 'rw') != 'cp'} + + # mount every copy volume to a temporary directory + for source_path, volume_config in copy_volumes.items(): + temp_dir = Path('/tmp') / str(volume_config['bind']).replace('/', '_') + new_volume_config = {'bind': str(temp_dir), 'mode': 'ro'} + volumes[source_path] = new_volume_config + + create_configuration = self.container_create_configuration.copy() + create_configuration['volumes'] = volumes + + if self.container is not None and destroy_if_exists: + self.destroy() + elif self.container is not None: + raise Exception('Container is already initialized') + + try: + # Start a new container with open stdin and tty + self.container = self.docker_client.containers.run(stdin_open=True, tty=True, detach=True, + **create_configuration) + + # copy contents of copy volumes to the actual target directories + if isinstance(volumes, dict) and len(copy_volumes) > 0: + self._ensure_installed('rsync') + for source_path, volume_config in copy_volumes.items(): + temp_dir = Path('/tmp') / str(volume_config['bind']).replace('/', '_') + self.container.exec_run(['ls', '-la', str(temp_dir)]) + self.container.exec_run(['mkdir', '-p', str(volume_config['bind'])]) + self.container.exec_run(['rsync', '-a', str(temp_dir) + '/', str(volume_config['bind'])]) + + if interactive or self._interactive_session_name is not None: + self._interactive_session_name = self._ensure_tmux_session( + name=f'interactive_session_{random.randint(10000, 99999)}', interpreter=interactive_interpreter) + except Exception as e: + # rethrow exception + raise + + def recreate(self): + """ + Reinitializes and recreates the container. + + :return: None + :raises Exception: If the container is not initialized or if there are any errors during reinitialization. + """ + if self.container_create_configuration is None: + raise Exception('Container is not initialized') + + try: + self.destroy() + except Exception: + pass + + try: + self.initialize(**self.container_create_configuration) + except Exception: + # rethrow exception + raise + + def destroy(self, silent: bool = False): + """ + Destroy the container. + + :param silent: Flag indicating whether to suppress exception if container is not initialized. Defaults to False. + :return: None + """ + if self.container is None and not silent: + raise Exception('Container is not initialized') + elif self.container is None: + return + + try: + self.container.remove(force=True) + self.container = None + except Exception: + if not silent: + raise + + def execute_command(self, command: str, force_noninteractive: bool = False): + """ + Execute a command inside the container. + + :param command: The command to execute inside the container. + :param force_noninteractive: Flag indicating whether to force non-interactive mode for command execution. + :return: The output of the executed command. + + :raises Exception: If the container is not initialized or not running. + + """ + # check if container is initialized + if self.container is None: + raise Exception('Container is not initialized') + if self.container.status != 'running' and self.container.status != 'created': + raise Exception('Container is not running') + + return (self._execute_interactive_command( + command) if self._interactive_session_name is not None and not force_noninteractive else + self._execute_noninteractive_command( + command)) + + def execute_batch(self, commands: List[str]): + """ + Executes a batch of commands. + + :param commands: A list of strings representing the commands to be executed. + :type commands: list[str] + :return: An iterator that yields the results of executing each command. + :rtype: generator + """ + for command in commands: + yield self.execute_command(command) + + def _execute_noninteractive_command(self, command: str): + """ + Execute a non-interactive command inside a container. + + :param command: The command to be executed. + :type command: str + :return: The stdout output and the exit code of the command. + :rtype: ShelOutput + :raises Exception: If the command execution fails. + """ + try: + exit_code, output = self.container.exec_run(command) + return ShelOutput(output.decode('utf-8').strip(), exit_code) + except Exception as e: + raise Exception(f'Failed to execute command "{command}". {e}') + + def _execute_interactive_command(self, command): + """ + Execute an interactive command inside the container's interactive session. + + :param command: The command to execute. + :return: The command's output and exit code. + """ + # TODO: wonder if there is a better way + if isinstance(command, list): + command = ' '.join(command) + + try: + self.container.exec_run( + ['tmux', 'send-keys', '-t', f'{self._interactive_session_name}', f"{command} ; echo '##DONE##'", 'C-m']) + + # get output + for _ in range(60): + command_output = self.container.exec_run( + ['tmux', 'capture-pane', '-p', '-t', f'{self._interactive_session_name}']) + + output = command_output.output.decode('utf-8').strip() + if '\n##DONE##\n' in output: + break + else: + sleep(1.0) + + # scrab node name and the last welcome message + # output = self._node_regex.sub(':', output) + # output = '\n'.join(output.split('\n')[:-1]) + + # clear screen + self.container.exec_run(['tmux', 'send-keys', '-t', f'{self._interactive_session_name}', 'clear', 'C-m']) + + return ShelOutput(output, command_output.exit_code) + except Exception as e: + raise Exception(f'Failed to execute command "{command}". {e}') + + def _ensure_tmux_session(self, name=None, interpreter=None): + """ + This method ensures that a tmux session with the specified name exists. If the session does not exist, + it starts a new session using the specified interpreter. If the session still does + * not exist after starting, an exception is raised. + + :param name: The name of the tmux session to ensure. Default is 'agent_session'. + :param interpreter: The interpreter to use when starting a new tmux session. Default is 'sh'. + :return: The name of the tmux session that was ensured. + """ + self._ensure_installed("tmux") + + name = name or 'agent_session' + interpreter = interpreter or 'sh' + + command_result = self.container.exec_run(['tmux', 'has-session', '-t', name]) + + # check if tmux session exists + if command_result.exit_code != 0: + # start new tmux session + command_result = self.container.exec_run(['tmux', 'new-session', '-d', '-s', name, interpreter]) + if command_result.exit_code != 0: + raise Exception(f'Failed to start tmux session {name}. {command_result.output.decode("utf-8")}') + + command_result = self.container.exec_run(['tmux', 'has-session', '-t', name]) + if command_result.exit_code != 0: + raise Exception(f'Failed to start tmux session {name}. {command_result.output.decode("utf-8")}') + + return name + + def _ensure_installed(self, app_name: str, package_name: str = None): + """ + Ensures that an app is installed by checking if it is already installed and installing it if necessary. + + :param app_name: The name of the app to be installed. + :param package_name: The name of the package to be installed. If not provided, the app_name will be used as + the package_name. + :return: None + + Raises: + Exception: If no known package manager is found. + Exception: If the installation fails. + """ + # check if app is installed + if not self._has_app(app_name): + # install package + package_name = package_name or app_name + try: + if self._has_app('apt'): + installation_result = self.container.exec_run( + ['sh', '-c', f'apt update -y && apt install -y {package_name}']) + elif self._has_app('yum'): + installation_result = self.container.exec_run( + ['sh', '-c', f'yum update -y && yum install -y {package_name}']) + elif self._has_app('dnf'): + installation_result = self.container.exec_run( + ['sh', '-c', f'dnf update -y && dnf install -y {package_name}']) + else: + raise Exception('No known package manager found') + + if installation_result.exit_code != 0: + raise Exception(f'Installation failed with: {installation_result.output.decode("utf-8")}') + except Exception as e: + raise Exception(f'Failed to install {package_name}. {e}') + + def _has_app(self, app_name: str): + """ + :param app_name: The name of the application to check if it exists. + :return: Returns True if the application exists, False otherwise. + + """ + try: + command_result = self.container.exec_run(['which', f'{app_name}']) + return command_result.exit_code == 0 + except Exception as e: + return False + + +@contextmanager +def docker_session(image: str = None, command: Union[str, List[str]] = None, + working_dir: PathLike = None, ports: Dict[int, int] = None, + volumes: Union[Dict[Union[str, Path], Dict], List[str]] = None, + environment: Dict[str, str] = None, user: str = None, + interactive: bool = False, interactive_interpreter: str = None, + destroy_if_exists: bool = False, docker_client=None) -> DockerInterface: + """ + Context manager that provides a docker session. + + :param image: The name or ID of the Docker image to use. + :param command: The command to execute inside the container. Defaults to '/bin/sh' if not provided. + :param working_dir: The working directory inside the Docker container. + :param ports: A dictionary of ports to expose inside the container. The key is the port inside the container + and the value is the port on the host machine. + :param volumes: A dictionary or list specifying the volumes to mount inside the container. + :param environment: A dictionary of environment variables to set inside the container. + :param user: The user to run commands as inside the container. + :param interactive: Whether to run the container in interactive mode. + :param interactive_interpreter: The interpreter to use for interactive mode. + :param destroy_if_exists: Whether to destroy the container if it already exists. + :param docker_client: The Docker client object to use. + :return: A DockerInterface object that represents the Docker session. + :rtype: DockerInterface + + Example usage: + + ```python + with docker_session( + image='my_image', + work_dir='/app', + environment={'FOO': 'bar'}, + volumes={'/host_path': '/app'}, + ) as docker: + # Do something with the Docker interface + docker.run('python app.py') + ``` + """ + docker_interface = DockerInterface(docker_client) + + try: + docker_interface.initialize(image=image, working_dir=working_dir, command=command, ports=ports, + volumes=volumes, environment=environment, user=user, + interactive=interactive, interactive_interpreter=interactive_interpreter, + destroy_if_exists=destroy_if_exists) + yield docker_interface + finally: + docker_interface.destroy(silent=True) diff --git a/ide_former_simulation/prompt_helpers.py b/ide_former_simulation/prompt_helpers.py new file mode 100644 index 0000000..4f1caab --- /dev/null +++ b/ide_former_simulation/prompt_helpers.py @@ -0,0 +1,40 @@ +import io +import sys +from typing import Union + + +class ColorCodes: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +def tee(*values, end: str = "", color: Union[ColorCodes, str] = None, file=sys.stdout, flush=False) -> str: + """Prints the values to a stream, or to sys.stdout by default + and returns the string that was printed. + + Optional arguments except `end` affect only the output stream and not the return value. + `end` is always appended to both the stream and the return value. + + :param values: The values to print. Follows the same syntax as the print function. + :param end: The string to append to the end of the string. + :param color: The color to print the string in. + :param file: The file to print the string to. + :param flush: Whether to flush the file after printing. + + :return: The string that was printed. + """ + output_stream = io.StringIO() + print(*values, end=end, file=output_stream, flush=True) + output = output_stream.getvalue() + + output_to_print = output if color is None else f"{color}{output}{ColorCodes.ENDC}" + print(output_to_print, end=end, file=file, flush=flush) + + return output diff --git a/notebooks/01-ReAct-Bash.ipynb b/notebooks/01-ReAct-Bash.ipynb new file mode 100644 index 0000000..8f54703 --- /dev/null +++ b/notebooks/01-ReAct-Bash.ipynb @@ -0,0 +1,613 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2023-11-10T13:46:35.560731Z", + "start_time": "2023-11-10T13:46:34.084225Z" + } + }, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "\n", + "import outlines.models as models\n", + "import outlines.text as text\n", + "\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Load model" + ], + "metadata": { + "collapsed": false + }, + "id": "df97d985115e3596" + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "# move outside of file\n", + "import os\n", + "import openai\n", + "\n", + "openai.api_key = \"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\"\n", + "os.environ[\"OPENAI_API_KEY\"] = openai.api_key" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T13:46:40.749720Z", + "start_time": "2023-11-10T13:46:40.623044Z" + } + }, + "id": "2102b97f37d38465" + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [ + { + "data": { + "text/plain": "\"Hello John! I am an AI language model developed by OpenAI, and I don't have a personal name. You can simply call me GPT-3 or Assistant. How can I assist you today?\"" + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import backoff\n", + "\n", + "complete_request = models.text_completion.openai(\n", + " \"gpt-3.5-turbo-16k\", max_tokens=128, temperature=1.0\n", + ")\n", + "\n", + "@backoff.on_exception(backoff.expo, openai.error.RateLimitError, max_time=60)\n", + "def complete(prompt: str, **kwargs) -> str:\n", + " return complete_request(prompt, **kwargs)\n", + "\n", + "complete(\"Hello, my name is John. What is your name?\")" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T13:46:42.187748Z", + "start_time": "2023-11-10T13:46:42.155255Z" + } + }, + "id": "31f113f789e1711" + }, + { + "cell_type": "markdown", + "source": [ + "## External function" + ], + "metadata": { + "collapsed": false + }, + "id": "ccb470347e66267" + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created file /tmp/test.txt\n", + "List of files in /tmp:\n", + "total 8\n", + "drwxrwxrwt 1 root root 4096 Nov 10 13:46 .\n", + "drwxr-xr-x 1 root root 4096 Nov 10 13:46 ..\n", + "-rw-r--r-- 1 root root 0 Nov 10 13:46 test.txt\n", + "Recreated container; list of files in /tmp:\n", + "total 8\n", + "drwxrwxrwt 2 root root 4096 Sep 16 02:33 .\n", + "drwxr-xr-x 1 root root 4096 Nov 10 13:46 ..\n" + ] + } + ], + "source": [ + "from ide_former_simulation import docker_session\n", + "\n", + "with docker_session(\n", + " image=\"ubuntu\",\n", + ") as session_interface:\n", + " out = session_interface.execute_command(\"touch /tmp/test.txt\")\n", + " print(f\"Created file /tmp/test.txt\")\n", + " out = session_interface.execute_command('ls -la /tmp')\n", + " print(f\"List of files in /tmp:\\n{out}\")\n", + " session_interface.recreate()\n", + " out = session_interface.execute_command('ls -la /tmp')\n", + " print(f\"Recreated container; list of files in /tmp:\\n{out}\")" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T13:46:44.395845Z", + "start_time": "2023-11-10T13:46:43.799429Z" + } + }, + "id": "830476b4c35a6afd" + }, + { + "cell_type": "markdown", + "source": [ + " ## Define prompts" + ], + "metadata": { + "collapsed": false + }, + "id": "e6ac8d875e2ec485" + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "You are an agent operation in the environment of the PyCharm IDE. On request, you need to find the answer to the question and write it down using the FINISH action. You have the whole power of the non-interactive linux command execution via the COMMAND action to explore the code base and operate it to find the answer, you can communicate with the user via COMMUNICATE action, and restart the state to to the specified step use RESET action. You start at the project root and allowed to roam only around the project folder and user $HOME directory.\n", + " \n", + " You are expected to execute according to the ReAct pattern and structure your logic like that (use example bellow for additional guidance):\n", + " * Tho : #### -- one line textual description of the current state and next actions. Its main intention is to help you to keep the context, history and future actions in mind. Thought block is required after every step.\n", + " * Act : -- one line textual action to be executed:\n", + " * COMMAND: you are given the whole power of the non-interactive linux command execution to explore the codebase. Mind that your actions may have stateful effects (like change of the current directory) and you need to keep that in mind when executing the next action or reverting back to the previous state. Use the format `Act : COMMAND ####` to execute the command in the environment. If the command execution fails, and it is not expected, check the syntax and parameters and try again. The implementation is error-prone and may fail for various reasons, in that case try other paths to the solution, behaviour is deterministic and if it fails, it fails. Due to the limites context window try sticking to commands producing limited output. For example, avoid using `cat` command to print the whole file, instead use `grep` to search for the specific keyword.\n", + " * COMMUNICATE: you can request additional information from the user and ask clarifying question. Use the format `Act : COMMUNICATE ####` to ask the question. If the user does not provide the answer, and it is not expected, check the question and try again. Although depends on the question, you should avoid asking that can be answered with command. For example, avoid asking \"What is the current directory?\" and instead use `pwd` command, or instead of asking \"What is the content of the file?\" use `cat` instead. Additionally, don't expect user answer to be precise and use it more as a hint to explore the codebase.\n", + " * RESET #### -- reset the state up to by the step K. Use the format `Act : RESET ` to reset the state. The command internally with reinitialize the environment and execute all the steps up to the step (including). Use K=0 to reset to the initial state.\n", + " * FINISH: when you are done with the task, use this command to submit the answer. Use the format `Act : FINISH ####` to write down the answer. If the answer is communicated to be wrong, and it is not expected, check the answer and try again.\n", + " * Obs : #### -- the result of the action execution provided by the environment. It can be the result of the command execution, the answer to the question, or the result of the action execution. Observation block will automatically be added after every action. If observation is not as expected you can always return to the previous step and reevaluate the options.\n", + " \n", + " The use of ## (double has is required): this symbol is used as a separator and is required to properly parse the output and assist in guided generation.\n", + " \n", + " # EXAMPLE\n", + " Obs 0: ##I am lost and can't find a file containing a function I need, the search seems to be broken...##\n", + " Tho 0: ##The user seems to be lost and can't find a file containing a function. I will try to help him to find it. But first, I need to figure out what is the function he is looking for. I will ask him.##\n", + " Act 0: COMMUNICATE ##Sorry to hear that. What is the name of the function you are looking for?##\n", + " Obs 0: ##It was called transformers but seems nowhere to be found...##\n", + " Tho 1: ##I am at the project root and need to find the file containing `transformers` function. I can either search for the file or roam the folders to find it manually. I will start with searching for the file.##\n", + " Act 1: COMMAND ##find . -name \"transformers.py\"##\n", + " Obs 1: ##/project# find . -name \"transformers.py\"##\n", + " Tho 2: ##Seems there is no such file in the project. I will try to find it manually. Let's list the files in the current directory.##\n", + " Act 2: COMMAND ##ls -la##\n", + " Obs 2: ##/project# ls -la\n", + " total 4\n", + " drwxr-xr-x 6 root root 192 Oct 9 12:55 .\n", + " drwxr-xr-x 1 root root 4096 Oct 9 13:34 ..\n", + " drwxr-xr-x 5 root root 160 Oct 9 12:55 docs\n", + " drwxr-xr-x 11 root root 352 Oct 9 12:55 examples\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 outlines\n", + " drwxr-xr-x 8 root root 256 Oct 9 12:55 tests##\n", + " Tho 3: ##There is no file with transformers in the name. I will try to find it in the subfolders. Taken I operate in the PyCharm IDE, and the project is most likely written in Python, 'outlines' sounds like a good candidate. Let's list the files in the outlines folder.##\n", + " Act 3: COMMAND ##ls -la outlines##\n", + " Obs 3: ##/project# ls -la outlins\n", + " ls: cannot access 'outlins': No such file or directory##\n", + " Tho 4: ##Wait, there is a spelling mistake in the folder name. Let's try again.##\n", + " Act 4: COMMAND ##ls -la outlines##\n", + " Obs 4: ##/project# ls -la outlines\n", + " total 24\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 .\n", + " drwxr-xr-x 6 root root 192 Oct 9 12:55 ..\n", + " -rw-r--r-- 1 root root 299 Sep 19 11:44 __init__.py\n", + " -rw-r--r-- 1 root root 444 Oct 6 14:14 _version.py\n", + " -rw-r--r-- 1 root root 9746 Sep 19 11:44 base.py\n", + " -rw-r--r-- 1 root root 1665 Sep 19 11:44 caching.py\n", + " drwxr-xr-x 12 root root 384 Oct 9 12:55 models\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 text\n", + " drwxr-xr-x 4 root root 128 Oct 9 12:55 vectors##\n", + " Tho 5: ##Transformers if that is related to the HuggingFace library should be in the `models` folder. Let's list the files in the models folder.##\n", + " Act 5: COMMAND ##ls -la outlines/models##\n", + " Obs 5: ##/project# ls -la outlines/models\n", + " total 68\n", + " drwxr-xr-x 12 root root 384 Oct 9 12:55 .\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 ..\n", + " -rw-r--r-- 1 root root 512 Sep 20 20:38 __init__.py\n", + " -rw-r--r-- 1 root root 99 Sep 19 11:44 embeddings.py\n", + " -rw-r--r-- 1 root root 1586 Sep 19 11:44 hf_diffusers.py\n", + " -rw-r--r-- 1 root root 15681 Sep 19 16:43 hf_transformers.py\n", + " -rw-r--r-- 1 root root 187 Sep 19 11:44 image_generation.py\n", + " -rw-r--r-- 1 root root 11969 Sep 19 11:44 openai.py\n", + " -rw-r--r-- 1 root root 1225 Sep 19 11:44 routers.py\n", + " -rw-r--r-- 1 root root 183 Sep 19 11:44 text_completion.py\n", + " -rw-r--r-- 1 root root 1040 Sep 19 11:44 tokenizer.py\n", + " -rw-r--r-- 1 root root 11925 Sep 29 15:37 transformers.py##\n", + " Tho 6: ##There is a transformers.py file. Let's check if it is the one we are looking for. The file can be quite large, while output is limited to 23 rows, so let's use `grep` to search for the function.##\n", + " Act 6: COMMAND ##grep -n \"transformers\" outlines/models/transformers.py##\n", + " Obs 6: ##/project# grep -n \"transformers\" outlines/models/transformers.py\n", + " 113: from transformers import AutoTokenizer, PreTrainedTokenizerBase\n", + " 115: raise ImportError(\"The `tokenizers` and `transformers` libraries\n", + " need to be installed in order \"\n", + " 116: \"to use `transformers` models.\")\n", + " 211: from transformers.models.code_llama import CodeLlamaTokenizerFas\n", + "t\n", + " 231:def transformers(model_or_name: Union[str, \"PreTrainedModel\"], *,\n", + " 235: Instantiate a model from the `transformers` library and its tokenizer.\n", + " 260: from transformers import AutoModelForCausalLM, AutoTokenizer, PreTra\n", + " inedModel\n", + " 262: raise ImportError(\"The `transformers` library needs to be installed\n", + " in order to use `transformers` models.\")##\n", + " Tho 7: ##There is a function called transformers. Let's check if it is the one we are looking for. The file can be quite large, so let's user `grep` to search for the function. We found the function in the file `outlines/models/transformers.py`. Let's figure out its full path.##\n", + " Act 7: COMMAND ##pwd##\n", + " Obs 7: ##/project# pwd\n", + " /project##\n", + " Tho 8: ##The full path of the file containing transformers function is `/project/outlines/models/transformers.py`.##\n", + " Act 8: FINISH ##/project/outlines/models/transformers.py, right?##\n", + " \n", + " ###\n", + " Obs 0: ##Install the project in the current folder in development mode##\n", + " \n", + " \n" + ] + } + ], + "source": [ + "@text.prompt\n", + "def build_reAct_prompt(question, partial_completion=None):\n", + " \"\"\"You are an agent operation in the environment of the PyCharm IDE. On request, you need to find the answer to the question and write it down using the FINISH action. You have the whole power of the non-interactive linux command execution via the COMMAND action to explore the code base and operate it to find the answer, you can communicate with the user via COMMUNICATE action, and restart the state to to the specified step use RESET action. You start at the project root and allowed to roam only around the project folder and user $HOME directory.\n", + " \n", + " You are expected to execute according to the ReAct pattern and structure your logic like that (use example bellow for additional guidance):\n", + " * Tho : #### -- one line textual description of the current state and next actions. Its main intention is to help you to keep the context, history and future actions in mind. Thought block is required after every step.\n", + " * Act : -- one line textual action to be executed:\n", + " * COMMAND: you are given the whole power of the non-interactive linux command execution to explore the codebase. Mind that your actions may have stateful effects (like change of the current directory) and you need to keep that in mind when executing the next action or reverting back to the previous state. Use the format `Act : COMMAND ####` to execute the command in the environment. If the command execution fails, and it is not expected, check the syntax and parameters and try again. The implementation is error-prone and may fail for various reasons, in that case try other paths to the solution, behaviour is deterministic and if it fails, it fails. Due to the limites context window try sticking to commands producing limited output. For example, avoid using `cat` command to print the whole file, instead use `grep` to search for the specific keyword.\n", + " * COMMUNICATE: you can request additional information from the user and ask clarifying question. Use the format `Act : COMMUNICATE ####` to ask the question. If the user does not provide the answer, and it is not expected, check the question and try again. Although depends on the question, you should avoid asking that can be answered with command. For example, avoid asking \"What is the current directory?\" and instead use `pwd` command, or instead of asking \"What is the content of the file?\" use `cat` instead. Additionally, don't expect user answer to be precise and use it more as a hint to explore the codebase.\n", + " * RESET #### -- reset the state up to by the step K. Use the format `Act : RESET ` to reset the state. The command internally with reinitialize the environment and execute all the steps up to the step (including). Use K=0 to reset to the initial state.\n", + " * FINISH: when you are done with the task, use this command to submit the answer. Use the format `Act : FINISH ####` to write down the answer. If the answer is communicated to be wrong, and it is not expected, check the answer and try again.\n", + " * Obs : #### -- the result of the action execution provided by the environment. It can be the result of the command execution, the answer to the question, or the result of the action execution. Observation block will automatically be added after every action. If observation is not as expected you can always return to the previous step and reevaluate the options.\n", + " \n", + " The use of ## (double has is required): this symbol is used as a separator and is required to properly parse the output and assist in guided generation.\n", + " \n", + " # EXAMPLE\n", + " Obs 0: ##I am lost and can't find a file containing a function I need, the search seems to be broken...##\n", + " Tho 0: ##The user seems to be lost and can't find a file containing a function. I will try to help him to find it. But first, I need to figure out what is the function he is looking for. I will ask him.##\n", + " Act 0: COMMUNICATE ##Sorry to hear that. What is the name of the function you are looking for?##\n", + " Obs 0: ##It was called transformers but seems nowhere to be found...##\n", + " Tho 1: ##I am at the project root and need to find the file containing `transformers` function. I can either search for the file or roam the folders to find it manually. I will start with searching for the file.##\n", + " Act 1: COMMAND ##find . -name \"transformers.py\"##\n", + " Obs 1: ##/project# find . -name \"transformers.py\"##\n", + " Tho 2: ##Seems there is no such file in the project. I will try to find it manually. Let's list the files in the current directory.##\n", + " Act 2: COMMAND ##ls -la##\n", + " Obs 2: ##/project# ls -la\n", + " total 4\n", + " drwxr-xr-x 6 root root 192 Oct 9 12:55 .\n", + " drwxr-xr-x 1 root root 4096 Oct 9 13:34 ..\n", + " drwxr-xr-x 5 root root 160 Oct 9 12:55 docs\n", + " drwxr-xr-x 11 root root 352 Oct 9 12:55 examples\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 outlines\n", + " drwxr-xr-x 8 root root 256 Oct 9 12:55 tests##\n", + " Tho 3: ##There is no file with transformers in the name. I will try to find it in the subfolders. Taken I operate in the PyCharm IDE, and the project is most likely written in Python, 'outlines' sounds like a good candidate. Let's list the files in the outlines folder.##\n", + " Act 3: COMMAND ##ls -la outlines##\n", + " Obs 3: ##/project# ls -la outlins\n", + " ls: cannot access 'outlins': No such file or directory##\n", + " Tho 4: ##Wait, there is a spelling mistake in the folder name. Let's try again.##\n", + " Act 4: COMMAND ##ls -la outlines##\n", + " Obs 4: ##/project# ls -la outlines\n", + " total 24\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 .\n", + " drwxr-xr-x 6 root root 192 Oct 9 12:55 ..\n", + " -rw-r--r-- 1 root root 299 Sep 19 11:44 __init__.py\n", + " -rw-r--r-- 1 root root 444 Oct 6 14:14 _version.py\n", + " -rw-r--r-- 1 root root 9746 Sep 19 11:44 base.py\n", + " -rw-r--r-- 1 root root 1665 Sep 19 11:44 caching.py\n", + " drwxr-xr-x 12 root root 384 Oct 9 12:55 models\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 text\n", + " drwxr-xr-x 4 root root 128 Oct 9 12:55 vectors##\n", + " Tho 5: ##Transformers if that is related to the HuggingFace library should be in the `models` folder. Let's list the files in the models folder.##\n", + " Act 5: COMMAND ##ls -la outlines/models##\n", + " Obs 5: ##/project# ls -la outlines/models\n", + " total 68\n", + " drwxr-xr-x 12 root root 384 Oct 9 12:55 .\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 ..\n", + " -rw-r--r-- 1 root root 512 Sep 20 20:38 __init__.py\n", + " -rw-r--r-- 1 root root 99 Sep 19 11:44 embeddings.py\n", + " -rw-r--r-- 1 root root 1586 Sep 19 11:44 hf_diffusers.py\n", + " -rw-r--r-- 1 root root 15681 Sep 19 16:43 hf_transformers.py\n", + " -rw-r--r-- 1 root root 187 Sep 19 11:44 image_generation.py\n", + " -rw-r--r-- 1 root root 11969 Sep 19 11:44 openai.py\n", + " -rw-r--r-- 1 root root 1225 Sep 19 11:44 routers.py\n", + " -rw-r--r-- 1 root root 183 Sep 19 11:44 text_completion.py\n", + " -rw-r--r-- 1 root root 1040 Sep 19 11:44 tokenizer.py\n", + " -rw-r--r-- 1 root root 11925 Sep 29 15:37 transformers.py##\n", + " Tho 6: ##There is a transformers.py file. Let's check if it is the one we are looking for. The file can be quite large, while output is limited to 23 rows, so let's use `grep` to search for the function.##\n", + " Act 6: COMMAND ##grep -n \"transformers\" outlines/models/transformers.py##\n", + " Obs 6: ##/project# grep -n \"transformers\" outlines/models/transformers.py\n", + " 113: from transformers import AutoTokenizer, PreTrainedTokenizerBase\n", + " 115: raise ImportError(\"The `tokenizers` and `transformers` libraries\n", + " need to be installed in order \"\n", + " 116: \"to use `transformers` models.\")\n", + " 211: from transformers.models.code_llama import CodeLlamaTokenizerFas\n", + "t\n", + " 231:def transformers(model_or_name: Union[str, \"PreTrainedModel\"], *,\n", + " 235: Instantiate a model from the `transformers` library and its tokenizer.\n", + " 260: from transformers import AutoModelForCausalLM, AutoTokenizer, PreTra\n", + " inedModel\n", + " 262: raise ImportError(\"The `transformers` library needs to be installed\n", + " in order to use `transformers` models.\")##\n", + " Tho 7: ##There is a function called transformers. Let's check if it is the one we are looking for. The file can be quite large, so let's user `grep` to search for the function. We found the function in the file `outlines/models/transformers.py`. Let's figure out its full path.##\n", + " Act 7: COMMAND ##pwd##\n", + " Obs 7: ##/project# pwd\n", + " /project##\n", + " Tho 8: ##The full path of the file containing transformers function is `/project/outlines/models/transformers.py`.##\n", + " Act 8: FINISH ##/project/outlines/models/transformers.py, right?##\n", + " \n", + " ###\n", + " Obs 0: ##{{ question }}##\n", + " {{ partial_completion if partial_completion else '' }}\n", + " \"\"\"\n", + "\n", + "\n", + "question = \"Install the project in the current folder in development mode\"\n", + "prompt = build_reAct_prompt(question)\n", + "\n", + "print(prompt)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T13:46:47.020640Z", + "start_time": "2023-11-10T13:46:47.008912Z" + } + }, + "id": "21982b096608b018" + }, + { + "cell_type": "code", + "execution_count": 10, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Project path: /Users/Evgeny.Grigorenko/Workspaces/projects/ideformer-plugin\n", + "\u001B[92mYou are an agent operation in the environment of the PyCharm IDE. On request, you need to find the answer to the question and write it down using the FINISH action. You have the whole power of the non-interactive linux command execution via the COMMAND action to explore the code base and operate it to find the answer, you can communicate with the user via COMMUNICATE action, and restart the state to to the specified step use RESET action. You start at the project root and allowed to roam only around the project folder and user $HOME directory.\n", + " \n", + " You are expected to execute according to the ReAct pattern and structure your logic like that (use example bellow for additional guidance):\n", + " * Tho : #### -- one line textual description of the current state and next actions. Its main intention is to help you to keep the context, history and future actions in mind. Thought block is required after every step.\n", + " * Act : -- one line textual action to be executed:\n", + " * COMMAND: you are given the whole power of the non-interactive linux command execution to explore the codebase. Mind that your actions may have stateful effects (like change of the current directory) and you need to keep that in mind when executing the next action or reverting back to the previous state. Use the format `Act : COMMAND ####` to execute the command in the environment. If the command execution fails, and it is not expected, check the syntax and parameters and try again. The implementation is error-prone and may fail for various reasons, in that case try other paths to the solution, behaviour is deterministic and if it fails, it fails. Due to the limites context window try sticking to commands producing limited output. For example, avoid using `cat` command to print the whole file, instead use `grep` to search for the specific keyword.\n", + " * COMMUNICATE: you can request additional information from the user and ask clarifying question. Use the format `Act : COMMUNICATE ####` to ask the question. If the user does not provide the answer, and it is not expected, check the question and try again. Although depends on the question, you should avoid asking that can be answered with command. For example, avoid asking \"What is the current directory?\" and instead use `pwd` command, or instead of asking \"What is the content of the file?\" use `cat` instead. Additionally, don't expect user answer to be precise and use it more as a hint to explore the codebase.\n", + " * RESET #### -- reset the state up to by the step K. Use the format `Act : RESET ` to reset the state. The command internally with reinitialize the environment and execute all the steps up to the step (including). Use K=0 to reset to the initial state.\n", + " * FINISH: when you are done with the task, use this command to submit the answer. Use the format `Act : FINISH ####` to write down the answer. If the answer is communicated to be wrong, and it is not expected, check the answer and try again.\n", + " * Obs : #### -- the result of the action execution provided by the environment. It can be the result of the command execution, the answer to the question, or the result of the action execution. Observation block will automatically be added after every action. If observation is not as expected you can always return to the previous step and reevaluate the options.\n", + " \n", + " The use of ## (double has is required): this symbol is used as a separator and is required to properly parse the output and assist in guided generation.\n", + " \n", + " # EXAMPLE\n", + " Obs 0: ##I am lost and can't find a file containing a function I need, the search seems to be broken...##\n", + " Tho 0: ##The user seems to be lost and can't find a file containing a function. I will try to help him to find it. But first, I need to figure out what is the function he is looking for. I will ask him.##\n", + " Act 0: COMMUNICATE ##Sorry to hear that. What is the name of the function you are looking for?##\n", + " Obs 0: ##It was called transformers but seems nowhere to be found...##\n", + " Tho 1: ##I am at the project root and need to find the file containing `transformers` function. I can either search for the file or roam the folders to find it manually. I will start with searching for the file.##\n", + " Act 1: COMMAND ##find . -name \"transformers.py\"##\n", + " Obs 1: ##/project# find . -name \"transformers.py\"##\n", + " Tho 2: ##Seems there is no such file in the project. I will try to find it manually. Let's list the files in the current directory.##\n", + " Act 2: COMMAND ##ls -la##\n", + " Obs 2: ##/project# ls -la\n", + " total 4\n", + " drwxr-xr-x 6 root root 192 Oct 9 12:55 .\n", + " drwxr-xr-x 1 root root 4096 Oct 9 13:34 ..\n", + " drwxr-xr-x 5 root root 160 Oct 9 12:55 docs\n", + " drwxr-xr-x 11 root root 352 Oct 9 12:55 examples\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 outlines\n", + " drwxr-xr-x 8 root root 256 Oct 9 12:55 tests##\n", + " Tho 3: ##There is no file with transformers in the name. I will try to find it in the subfolders. Taken I operate in the PyCharm IDE, and the project is most likely written in Python, 'outlines' sounds like a good candidate. Let's list the files in the outlines folder.##\n", + " Act 3: COMMAND ##ls -la outlines##\n", + " Obs 3: ##/project# ls -la outlins\n", + " ls: cannot access 'outlins': No such file or directory##\n", + " Tho 4: ##Wait, there is a spelling mistake in the folder name. Let's try again.##\n", + " Act 4: COMMAND ##ls -la outlines##\n", + " Obs 4: ##/project# ls -la outlines\n", + " total 24\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 .\n", + " drwxr-xr-x 6 root root 192 Oct 9 12:55 ..\n", + " -rw-r--r-- 1 root root 299 Sep 19 11:44 __init__.py\n", + " -rw-r--r-- 1 root root 444 Oct 6 14:14 _version.py\n", + " -rw-r--r-- 1 root root 9746 Sep 19 11:44 base.py\n", + " -rw-r--r-- 1 root root 1665 Sep 19 11:44 caching.py\n", + " drwxr-xr-x 12 root root 384 Oct 9 12:55 models\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 text\n", + " drwxr-xr-x 4 root root 128 Oct 9 12:55 vectors##\n", + " Tho 5: ##Transformers if that is related to the HuggingFace library should be in the `models` folder. Let's list the files in the models folder.##\n", + " Act 5: COMMAND ##ls -la outlines/models##\n", + " Obs 5: ##/project# ls -la outlines/models\n", + " total 68\n", + " drwxr-xr-x 12 root root 384 Oct 9 12:55 .\n", + " drwxr-xr-x 9 root root 288 Oct 9 12:55 ..\n", + " -rw-r--r-- 1 root root 512 Sep 20 20:38 __init__.py\n", + " -rw-r--r-- 1 root root 99 Sep 19 11:44 embeddings.py\n", + " -rw-r--r-- 1 root root 1586 Sep 19 11:44 hf_diffusers.py\n", + " -rw-r--r-- 1 root root 15681 Sep 19 16:43 hf_transformers.py\n", + " -rw-r--r-- 1 root root 187 Sep 19 11:44 image_generation.py\n", + " -rw-r--r-- 1 root root 11969 Sep 19 11:44 openai.py\n", + " -rw-r--r-- 1 root root 1225 Sep 19 11:44 routers.py\n", + " -rw-r--r-- 1 root root 183 Sep 19 11:44 text_completion.py\n", + " -rw-r--r-- 1 root root 1040 Sep 19 11:44 tokenizer.py\n", + " -rw-r--r-- 1 root root 11925 Sep 29 15:37 transformers.py##\n", + " Tho 6: ##There is a transformers.py file. Let's check if it is the one we are looking for. The file can be quite large, while output is limited to 23 rows, so let's use `grep` to search for the function.##\n", + " Act 6: COMMAND ##grep -n \"transformers\" outlines/models/transformers.py##\n", + " Obs 6: ##/project# grep -n \"transformers\" outlines/models/transformers.py\n", + " 113: from transformers import AutoTokenizer, PreTrainedTokenizerBase\n", + " 115: raise ImportError(\"The `tokenizers` and `transformers` libraries\n", + " need to be installed in order \"\n", + " 116: \"to use `transformers` models.\")\n", + " 211: from transformers.models.code_llama import CodeLlamaTokenizerFas\n", + "t\n", + " 231:def transformers(model_or_name: Union[str, \"PreTrainedModel\"], *,\n", + " 235: Instantiate a model from the `transformers` library and its tokenizer.\n", + " 260: from transformers import AutoModelForCausalLM, AutoTokenizer, PreTra\n", + " inedModel\n", + " 262: raise ImportError(\"The `transformers` library needs to be installed\n", + " in order to use `transformers` models.\")##\n", + " Tho 7: ##There is a function called transformers. Let's check if it is the one we are looking for. The file can be quite large, so let's user `grep` to search for the function. We found the function in the file `outlines/models/transformers.py`. Let's figure out its full path.##\n", + " Act 7: COMMAND ##pwd##\n", + " Obs 7: ##/project# pwd\n", + " /project##\n", + " Tho 8: ##The full path of the file containing transformers function is `/project/outlines/models/transformers.py`.##\n", + " Act 8: FINISH ##/project/outlines/models/transformers.py, right?##\n", + " \n", + " ###\n", + " Obs 0: ##List all available git branches and their last commit message##\n", + " \n", + " \u001B[0m\u001B[92m\n", + "Tho 0: ##\u001B[0m\u001B[94mThe task is to list all available git branches and their last commit message.\n", + "\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Act 0: \u001B[0m\u001B[94mCOMMAND\u001B[0m\u001B[92m ##\u001B[0m\u001B[94mgit branch -v\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Obs 0: ##\u001B[0m\u001B[96mroot@771d1aaad16b:/project# git branch -v ; echo '##DONE##'\n", + " evgri243/add-docker-container f1fdb52 Fix Actions definition\n", + "* evgri243/build-fix-repository-name bb8f22c Fix docker repository requiring nam\n", + "e to be lowercase.\n", + " main 2c155d1 Merge pull request #3 from evgri243\n", + "/main\n", + "##DONE##\n", + "root@771d1aaad16b:/project#\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Tho 1: ##\u001B[0m\u001B[94mI am at the project root and need to list all available git branches and their last commit message. I will start by executing the git command.\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Act 1: \u001B[0m\u001B[94mCOMMAND\u001B[0m\u001B[92m ##\u001B[0m\u001B[94mgit branch -v\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Obs 1: ##\u001B[0m\u001B[96mroot@771d1aaad16b:/project# git branch -v ; echo '##DONE##'\n", + " evgri243/add-docker-container f1fdb52 Fix Actions definition\n", + "* evgri243/build-fix-repository-name bb8f22c Fix docker repository requiring nam\n", + "e to be lowercase.\n", + " main 2c155d1 Merge pull request #3 from evgri243\n", + "/main\n", + "##DONE##\n", + "root@771d1aaad16b:/project#\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Tho 2: ##\u001B[0m\u001B[94mI executed the `git branch -v` command to list all available git branches and their last commit message. Here is the result:\n", + "\n", + "- evgri243/add-docker-container f1fdb52 Fix Actions definition\n", + "- evgri243/build-fix-repository-name bb8f22c Fix docker repository requiring name to be lowercase.\n", + "- main 2c155d1 Merge pull request #3 from evgri243/main\u001B[0m\u001B[92m##\u001B[0m\u001B[92m\n", + "Act 2: \u001B[0m\u001B[94mFINISH\u001B[0m\u001B[92m ##\u001B[0m\u001B[94m- evgri243/add-docker-container f1fdb52 Fix Actions definition\n", + "- evgri243/build-fix-repository-name bb8f22c Fix docker repository requiring name to be lowercase.\n", + "- main 2c155d1 Merge pull request #3 from evgri243/main\u001B[0m\u001B[92m##\u001B[0m" + ] + } + ], + "source": [ + "import time\n", + "from ide_former_simulation import tee, ColorCodes\n", + "from pathlib import Path\n", + "\n", + "test_project_path = (Path().resolve().parent).as_posix()\n", + "print(f\"Project path: {test_project_path}\")\n", + "\n", + "question = \"List all available git branches and their last commit message\"\n", + "prompt = tee(build_reAct_prompt(question), color=ColorCodes.OKGREEN)\n", + "\n", + "is_interactive = True\n", + "max_steps = 20\n", + "\n", + "executed_commands = [None] * max_steps\n", + "\n", + "with docker_session(\n", + " image=\"ghcr.io/jetbrains-research/ideformer-plugin/simulator:latest\",\n", + " command=[\"/project\"],\n", + " ports={8080: 8080},\n", + " working_dir=\"/project\",\n", + " volumes={\n", + " test_project_path: {\"bind\": \"/project\", \"mode\": \"cp\"}\n", + " },\n", + " interactive=is_interactive,\n", + " interactive_interpreter=\"bash\",\n", + ") as session_interface:\n", + " for i in range(0, max_steps):\n", + " # sleep to avoid throttling\n", + " time.sleep(1)\n", + " # Tho brock should be first for every step\n", + " prompt += tee(f\"\\nTho {i}: ##\", color=ColorCodes.OKGREEN)\n", + " \n", + " thought = complete(str(prompt), stop_at=[\"##\", \"\\nAct\", \"\\nTho\", \"\\nObs\"])\n", + " prompt += tee(thought, color=ColorCodes.OKBLUE) + tee(\"##\", color=ColorCodes.OKGREEN)\n", + " \n", + " # Act block is the next\n", + " prompt += tee(f\"\\nAct {i}: \", color=ColorCodes.OKGREEN)\n", + " action = complete(str(prompt), is_in=[\"COMMAND\", \"COMMUNICATE\", \"RESET\", \"FINISH\"])\n", + " prompt += tee(action, color=ColorCodes.OKBLUE) + tee(\" ##\", color=ColorCodes.OKGREEN)\n", + "\n", + " if action == \"RESET\":\n", + " subject = complete(str(prompt), is_in=[str(j) for j in range(0, i)])\n", + " else:\n", + " subject = complete(str(prompt), stop_at=[\"##\", \"\\nAct\", \"\\nTho\", \"\\nObs\"])\n", + " prompt += tee(subject, color=ColorCodes.OKBLUE) + tee(\"##\", color=ColorCodes.OKGREEN)\n", + " \n", + " if action == \"COMMAND\":\n", + " result = session_interface.execute_command(subject).output\n", + " executed_commands[i] = subject\n", + " # add an indicator if result is empty\n", + " if len(result.split(\"\\n\")) == (1 if is_interactive else 0):\n", + " result += \"\\n\"\n", + "\n", + " elif action == \"COMMUNICATE\":\n", + " result = input(subject)\n", + " if len(result) == 0:\n", + " result = \"\"\n", + "\n", + " elif action == \"RESET\":\n", + " session_interface.recreate()\n", + " # reapply all the commands up to the target step\n", + " target = int(subject)\n", + " j = 0\n", + " try:\n", + " for j in range(0, target):\n", + " if executed_commands[j] is not None:\n", + " session_interface.execute_command(executed_commands[j])\n", + "\n", + " result = f\"Successfully rerun commands up to step {subject}.\"\n", + " except Exception as e:\n", + " result = f\"Failed to reset to step {subject}: {e}. Last executed step: {j}.\"\n", + " break\n", + " \n", + " # remove all the executed commands after the target step\n", + " executed_commands[target:] = [None] * (max_steps - target)\n", + "\n", + " elif action == \"FINISH\":\n", + " correct = input(f\"Model response '{subject}'. \"\n", + " \"Print 'stop' to finish or reply to the model\")\n", + " if correct.lower() == \"stop\":\n", + " break\n", + " else:\n", + " result = correct\n", + "\n", + " prompt += tee(f\"\\nObs {i}: ##\", color=ColorCodes.OKGREEN) + tee(result, color=ColorCodes.OKCYAN) + tee(\"##\", color=ColorCodes.OKGREEN)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T13:51:19.915847Z", + "start_time": "2023-11-10T13:50:40.592762Z" + } + }, + "id": "39dc05dbe780c6b4" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2b005e3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[project] +name = "ide_former_simulation" +description = "Helper library IDE Former Simulation" +license = { file = "LICENSE" } +version = "0.1.0" +authors = [ + { name = "Evgeny Grigorenko", email = "evgeny.grigorenko@jetbrains.com" }, + { name = "Evgenija Fedotova", email = "evgeniia.fedotova@jetbrains.com" }, + { name = "Yaroslav Zharov", email = "yaroslav.zharov@jetbrains.com" } +] +classifiers = [ + "License :: OSI Approved :: MIT License", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] + +requires-python = ">=3.9,<3.12" + +dependencies = [ + "docker~=6.1.3", + "outlines @ git+https://github.com/evgri243/outlines.git@evgri243/relax-model-loading", + "jinja2~=3.1.2", + "openai~=0.28.1", + "backoff~=2.2.1", + "jupyter~=1.0.0", + "tiktoken~=0.5.1", + "nest-asyncio~=1.5.8", + "transformers~=4.33.3", + "tokenizers~=0.13.0", + "torch~=2.1.0", +] + +[build-system] +requires = ["setuptools ~= 68.2.2", "cython ~= 3.0.5"]