Merge branch 'self_configure'

.github/workflows/build.yml

@@ -117,8 +117,8 @@ jobs:
       - name: Build and test artifact
         if: matrix.cross == null && !startsWith(matrix.os, 'windows')
         run: |
-          cargo test --release --target ${{ matrix.target }} || exit 1
-          cargo build --release --target ${{ matrix.target }} || exit 1
+          OPENSSL_STATIC=1 cargo test --release --target ${{ matrix.target }} || exit 1
+          OPENSSL_STATIC=1 cargo build --release --target ${{ matrix.target }} || exit 1
           mkdir python_binding_and_cmdline/refact/bin
           cp ./target/${{ matrix.target }}/release/refact-lsp${{ matrix.ext }} python_binding_and_cmdline/refact/bin
           cp ./target/${{ matrix.target }}/release/refact-lsp${{ matrix.ext }} .

.github/workflows/release.yml

@@ -6,7 +6,6 @@ on:
     tags:
       - '*'
 
-
 env:
   CARGO_INCREMENTAL: 0
   CARGO_NET_RETRY: 10
@@ -110,8 +109,8 @@ jobs:
       - name: Build and test artifact
         if: matrix.cross == null && !startsWith(matrix.os, 'windows')
         run: |
-          cargo test --release --target ${{ matrix.target }} || exit 1
-          cargo build --release --target ${{ matrix.target }} || exit 1
+          OPENSSL_STATIC=1 cargo test --release --target ${{ matrix.target }} || exit 1
+          OPENSSL_STATIC=1 cargo build --release --target ${{ matrix.target }} || exit 1
           mkdir python_binding_and_cmdline/refact/bin
           cp ./target/${{ matrix.target }}/release/refact-lsp${{ matrix.ext }} python_binding_and_cmdline/refact/bin
           cp ./target/${{ matrix.target }}/release/refact-lsp${{ matrix.ext }} .

Cargo.toml

@@ -6,7 +6,7 @@ lto = true
 
 [package]
 name = "refact-lsp"
-version = "0.10.5"
+version = "0.10.6"
 edition = "2021"
 build = "build.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -16,12 +16,12 @@ default = ["vecdb"]
 vecdb = ["arrow", "arrow-array", "arrow-schema", "lance", "vectordb"]
 
 [build-dependencies]
-shadow-rs = "0.25.0"
+shadow-rs = "0.36.0"
 
 [dependencies]
 sled = { version = "0.34", default-features = false, features = [] }
 # all features = ["compression", "docs", "event_log", "failpoints", "io_uring", "lock_free_delays", "measure_allocs", "miri_optimizations", "mutex", "no_inline", "no_logs", "pretty_backtrace", "testing"]
-shadow-rs = { version = "0.25.0", features = [], default-features = false }
+shadow-rs = { version = "0.36.0", features = [], default-features = false }
 hyper = { version = "0.14", features = ["server", "stream"] }
 reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls-webpki-roots", "charset", "http2"] }
 tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "rt-multi-thread", "signal", "process"] }
@@ -102,10 +102,12 @@ hashbrown = "0.14"
 shell-words = "1.1.0"
 sha2 = "0.10.8"
 glob = "0.3.1"
-base64 = "0.21.7"
+base64 = "0.22.1"
 image = "0.25.2"
 headless_chrome = "1.0.15"
 nix = { version = "0.29.0", features = ["signal"] }
 resvg = "0.44.0"
 async-tar = "0.5.0"
+git2 = "0.19.0"
 process-wrap = { version = "8.0.2", features = ["tokio1"] }
+rust-embed = "8.5.0"

build.rs

@@ -1,3 +1,4 @@
+
 fn main() -> shadow_rs::SdResult<()> {
     shadow_rs::new()
-}
+}

docker/lsp-debug.Dockerfile

@@ -8,6 +8,7 @@ RUN apk add --no-cache \
     curl \
     git \
     openssl-dev \
+    openssl-libs-static \
     pkgconfig \
     protobuf-dev \
     zlib-static

examples/links.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+
+echo
+echo Follow up example
+curl -X POST http://localhost:8001/v1/links \
+    -H "Content-Type: application/json" \
+    -d '{
+        "meta": {
+            "chat_id": "chat-example",
+            "chat_remote": false,
+            "chat_mode": "EXPLORE"
+        },
+        "model_name": "gpt-4o-mini",
+        "messages": [
+            {
+                "role": "system",
+                "content": "[mode2] You are a refact exploration assistant",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "user",
+                "content": "List all containers",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "assistant",
+                "content": "Docker tool not found, you can use cli and run docker ps",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            }
+        ]
+    }'
+
+echo
+echo Configuration goto example
+curl -X POST http://localhost:8001/v1/links \
+    -H "Content-Type: application/json" \
+    -d '{
+        "meta": {
+            "chat_id": "chat-example",
+            "chat_remote": false,
+            "chat_mode": "AGENT"
+        },
+        "model_name": "gpt-4o-mini",
+        "messages": [
+            {
+                "role": "system",
+                "content": "[mode3] You are a refact agentic assistant",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "user",
+                "content": "use psql tool to list all tables",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                    {
+                        "id": "call_BGLjpFFQJbD00euir9rQ8V7w",
+                        "function": {
+                            "arguments": "{\"query\":\"\\\\dt;\"}",
+                            "name": "postgres"
+                        },
+                        "type": "function"
+                    }
+                ],
+                "finish_reason": "tool_calls",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "tool",
+                "content": "🧩 for configuration go to SETTINGS:postgres, psql failed:\nNo such file or directory (os error 2)",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "call_BGLjpFFQJbD00euir9rQ8V7w",
+                "usage": null,
+                "subchats": null
+            }
+        ]
+    }'
+
+echo
+echo Summarize project example
+curl -X POST http://localhost:8001/v1/links \
+    -H "Content-Type: application/json" \
+    -d '{
+        "meta": {
+            "chat_id": "chat-example",
+            "chat_remote": false,
+            "chat_mode": "NOTOOLS"
+        },
+        "model_name": "gpt-4o-mini",
+        "messages": []
+    }'
+
+echo
+echo Example of Save and return in configuration chat
+curl -X POST http://localhost:8001/v1/links \
+    -H "Content-Type: application/json" \
+    -d '{
+        "meta": {
+            "chat_id": "chat-example",
+            "chat_remote": false,
+            "chat_mode": "CONFIGURE"
+        },
+        "model_name": "gpt-4o-mini",
+        "messages": [
+            {
+                "role": "system",
+                "content": "[mode3config] You are a refact configuration assistant",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "system",
+                "content": "[mode3config] You are a refact coding agent working on improving configuration files",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "user",
+                "content": "create a new file called \"hello_world.py\"",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            },
+            {
+                "role": "assistant",
+                "content": "📍REWRITE_WHOLE_FILE 000 \"/app/hello_world.py\"\n```python\nprint(\"Hello, World!\")\n```",
+                "tool_calls": null,
+                "finish_reason": "",
+                "tool_call_id": "",
+                "usage": null,
+                "subchats": null
+            }
+        ]
+    }'

python_binding_and_cmdline/refact/chat_client.py

@@ -222,7 +222,8 @@ async def ask_using_http(
     only_deterministic_messages: bool = False,
     postprocess_parameters: Optional[Dict[str, Any]] = None,
     callback: Optional[Callable] = None,
-    chat_id: Optional[str] = None
+    chat_id: Optional[str] = None,
+    chat_remote: bool = False,
 ) -> List[List[Message]]:
     deterministic: List[Message] = []
     subchats: DefaultDict[str, List[Message]] = collections.defaultdict(list)
@@ -241,8 +242,13 @@ async def ask_using_http(
     }
     if postprocess_parameters is not None:
         post_me["postprocess_parameters"] = postprocess_parameters
+    meta = {}
     if chat_id is not None:
-        post_me["chat_id"] = chat_id
+        meta["chat_id"] = chat_id
+        meta["chat_mode"] = "AGENT"
+        meta["chat_remote"] = chat_remote
+        # meta["current_config_file"] = "/Users/user/.config/refact/integrations.d/postgres.yaml"
+    post_me["meta"] = meta
     choices: List[Optional[Message]] = [None] * n_answers
     async with aiohttp.ClientSession() as session:
         async with session.post(base_url + "/chat", json=post_me) as response:
@@ -275,12 +281,13 @@ async def ask_using_http(
                     if not end_of_http_chunk:
                         continue
                     line_str = buffer.decode('utf-8').strip()
-                    buffer = b""
                     if not line_str:
+                        buffer = b""
                         continue
                     if not line_str.startswith("data: "):
                         print("unrecognized streaming data (1):", line_str)
                         continue
+                    buffer = b""
                     line_str = line_str[6:]
                     if line_str == "[DONE]":
                         break
@@ -390,28 +397,29 @@ async def diff_apply(
         async with session.post(base_url + "/diff-apply", json=post_me) as response:
             if response.status != 200:
                 raise Exception(f"unexpected response status {response.status}, response: {await response.text()}")
-            return await response.json(content_type=None)
+            return await _better_response_json(response)
 
 
-async def mem_add(base_url: str, mem_type: str, goal: str, project: str, payload: str) -> Dict[str, Any]:
+async def mem_add(base_url: str, mem_type: str, goal: str, project: str, payload: str, origin: str = "local-committed") -> Dict[str, Any]:
     url = f"{base_url}/mem-add"
     data = {
         "mem_type": mem_type,
         "goal": goal,
         "project": project,
-        "payload": payload
+        "payload": payload,
+        "origin": origin,
     }
     async with aiohttp.ClientSession() as session:
         async with session.post(url, json=data) as response:
-            return await response.json()
+            return await _better_response_json(response)
 
 
 async def mem_block_until_vectorized(base_url: str) -> Tuple[Dict[str, Any], float]:
     url = f"{base_url}/mem-block-until-vectorized"
     t0 = time.time()
     async with aiohttp.ClientSession() as session:
         async with session.get(url) as response:
-            return (await response.json(), time.time() - t0)
+            return (await _better_response_json(response), time.time() - t0)
 
 
 async def mem_update_used(base_url: str, memid: str, correct: float, relevant: float) -> Dict[str, Any]:
@@ -423,7 +431,7 @@ async def mem_update_used(base_url: str, memid: str, correct: float, relevant: f
     }
     async with aiohttp.ClientSession() as session:
         async with session.post(url, json=data) as response:
-            return await response.json()
+            return await _better_response_json(response)
 
 
 async def mem_erase(base_url: str, memid: str) -> Dict[str, Any]:
@@ -433,7 +441,7 @@ async def mem_erase(base_url: str, memid: str) -> Dict[str, Any]:
     }
     async with aiohttp.ClientSession() as session:
         async with session.post(url, json=data) as response:
-            return await response.json()
+            return await _better_response_json(response)
 
 
 async def mem_query(base_url: str, goal: str, project: str, top_n: Optional[int] = 5) -> Tuple[int, Dict[str, Any]]:
@@ -445,20 +453,19 @@ async def mem_query(base_url: str, goal: str, project: str, top_n: Optional[int]
     }
     async with aiohttp.ClientSession() as session:
         async with session.post(url, json=data) as response:
-            return response.status, await response.json()
-
-
-async def ongoing_update(base_url: str, goal: str, progress: Dict[str, Any], actseq: Dict[str, Any], output: Dict[str, Any]):
-    url = f"{base_url}/ongoing-update"
-    data = {
-        "goal": goal,
-        "ongoing_progress": progress,
-        "ongoing_action_new_sequence": actseq,
-        "ongoing_output": output,
-    }
-    async with aiohttp.ClientSession() as session:
-        async with session.post(url, json=data) as response:
-            return await response.json()
+            return response.status, await _better_response_json(response)
+
+
+async def _better_response_json(response):
+    if response.status == 200:
+        return await response.json()
+    txt = await response.text()
+    if txt.startswith("{"):
+        j = json.loads(txt)
+        if "detail" in j:
+            raise ValueError(j['detail'])
+        return j
+    raise ValueError("Unexpected response: %r" % txt)
 
 
 def gen_function_call_id():