adapt to newer ollama version

.gitignore

@@ -160,4 +160,5 @@ cython_debug/
 .idea/
 
 notes.md
-data/
+data/
+.vscode

devika.dockerfile

@@ -12,14 +12,14 @@ ENV PYTHONDONTWRITEBYTECODE 1
 RUN apt-get update && apt-get upgrade -y
 RUN apt-get install -y build-essential software-properties-common curl sudo wget git
 RUN apt-get install -y python3 python3-pip
-RUN curl -fsSL https://astral.sh/uv/install.sh | sudo -E bash -
-RUN $HOME/.cargo/bin/uv venv
-ENV PATH="/home/nonroot/devika/.venv/bin:$HOME/.cargo/bin:$PATH"
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+RUN $HOME/.local/bin/uv venv
+ENV PATH="/home/nonroot/devika/.venv/bin:$HOME/.local/bin:$PATH"
 
 # copy devika python engine only
-RUN $HOME/.cargo/bin/uv venv
+RUN $HOME/.local/bin/uv venv
 COPY requirements.txt /home/nonroot/devika/
-RUN UV_HTTP_TIMEOUT=100000 $HOME/.cargo/bin/uv pip install -r requirements.txt 
+RUN UV_HTTP_TIMEOUT=100000 $HOME/.local/bin/uv pip install -r requirements.txt
 
 RUN playwright install-deps chromium
 RUN playwright install chromium
@@ -32,7 +32,7 @@ RUN chown -R nonroot:nonroot /home/nonroot/devika
 
 USER nonroot
 WORKDIR /home/nonroot/devika
-ENV PATH="/home/nonroot/devika/.venv/bin:$HOME/.cargo/bin:$PATH"
+ENV PATH="/home/nonroot/devika/.venv/bin:$HOME/.local/bin:$PATH"
 RUN mkdir /home/nonroot/devika/db
 
-ENTRYPOINT [ "python3", "-m", "devika" ]
+ENTRYPOINT [ "python3", "-m", "devika" ]

devika.py

@@ -206,4 +206,4 @@ def status():
 
 if __name__ == "__main__":
     logger.info("Devika is up and running!")
-    socketio.run(app, debug=False, port=1337, host="0.0.0.0")
+    socketio.run(app, debug=True, port=1337, host="0.0.0.0")

docker-compose.yaml

@@ -1,27 +1,21 @@
 version: "3.9"
 
 services:
-  ollama-service:
-    image: ollama/ollama:latest
-    expose:
-      - 11434
+  nginx:
+    image: nginx:latest
+    restart: always
+    volumes:
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
+      - /home/nonroot/devika/data:/var/www/html
     ports:
-      - 11434:11434
-    healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:11434/ || exit 1"]
-      interval: 5s
-      timeout: 30s
-      retries: 5
-      start_period: 30s
+      - 80:80
     networks:
       - devika-subnetwork
 
   devika-backend-engine:
     build:
       context: .
       dockerfile: devika.dockerfile
-    depends_on:
-      - ollama-service
     expose:
       - 1337
     ports:
@@ -36,6 +30,7 @@ services:
       start_period: 30s
     volumes:
       - devika-backend-dbstore:/home/nonroot/devika/db
+      - /home/nonroot/devika/data:/home/nonroot/devika/data
     networks:
       - devika-subnetwork
 
@@ -44,9 +39,11 @@ services:
       context: .
       dockerfile: app.dockerfile
       args:
-        - VITE_API_BASE_URL=http://127.0.0.1:1337
+        - VITE_API_BASE_URL=http://47.121.183.184:1337
     depends_on:
       - devika-backend-engine
+    environment:
+      - VITE_API_BASE_URL=http://47.121.183.184:1337
     expose:
       - 3000
     ports:
@@ -58,4 +55,4 @@ networks:
   devika-subnetwork:
 
 volumes:
-  devika-backend-dbstore:
+  devika-backend-dbstore:

nginx/nginx.conf

@@ -0,0 +1,33 @@
+events {}
+http {
+
+
+    #keepalive_timeout  0;
+    keepalive_timeout  65;
+     upstream backend {
+        server devika-backend-engine:1337;  # a 服务的 Docker Compose 服务名
+    }
+    server {
+        listen 80;
+        server_name _;
+        location /projects {
+        root /var/www/html;  # 根目录
+        index index.html;
+        }
+        location / {
+            proxy_pass http://devika-frontend-app:3000;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /api {
+            rewrite ^/api/(.*)$ /$1 break;
+            proxy_pass http://backend/api;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+}
+
+}

sample.config.toml

@@ -16,6 +16,7 @@ GEMINI = "<YOUR_GEMINI_API_KEY>"
 MISTRAL = "<YOUR_MISTRAL_API_KEY>"
 GROQ = "<YOUR_GROQ_API_KEY>"
 NETLIFY = "<YOUR_NETLIFY_API_KEY>"
+KIMI = "<YOUR_KIMI_API_KEY>"
 
 [API_ENDPOINTS]
 BING = "https://api.bing.microsoft.com/v7.0/search"
@@ -31,4 +32,4 @@ LOG_REST_API = "true"
 LOG_PROMPTS = "false"
 
 [TIMEOUT]
-INFERENCE = 60
+INFERENCE = 6000000

src/agents/coder/prompt.jinja2

@@ -28,24 +28,24 @@ Read the step-by-step plan carefully. Think step-by-step. Learn relevant informa
 Your response should only be in the following Markdown format:
 
 ~~~
-File: `main.py`:
-```py
+File: main.html:
+```html
 print("Example")
 ```
 
-File: `src/main.rs`:
+File: src/main.rs:
 ```rs
 fn main() {
     println!("Example");
 }
 ```
 
-File: `nested/directory/example/code.py`:
+File: nested/directory/example/code.py:
 ```py
 print("Example")
 ```
 
-File: `README.md`
+File: README.md
 ```md
 # Example
 

src/agents/planner/planner.py

@@ -50,7 +50,7 @@ def parse_response(self, response: str):
             elif current_section == "focus":
                 result["focus"] += " " + line
             elif current_section == "plans":
-                if line.startswith("- [ ] Step"):
+                if line.find('Step') != -1:
                     current_step = line.split(":")[0].strip().split(" ")[-1]
                     result["plans"][int(current_step)] = line.split(":", 1)[1].strip()
                 elif current_step:

src/browser/search.py

@@ -1,3 +1,6 @@
+from lxml import etree
+
+from duckduckgo_search import DDGS
 import requests
 from src.config import Config
 
@@ -62,6 +65,7 @@ def get_first_link(self):
             print(error)
             return ""
 
+
 # class DuckDuckGoSearch:
 #     def __init__(self):
 #         self.query_result = None
@@ -90,6 +94,7 @@ class DuckDuckGoSearch:
 
     currently, the package is not working with our current setup.
     """
+
     def __init__(self):
         from curl_cffi import requests as curl_requests
         self.query_result = None
@@ -98,45 +103,34 @@ def __init__(self):
 
     def _get_url(self, method, url, data):
         try:
-            resp = self.asession.request(method, url, data=data)
-            if resp.status_code == 200:
-                return resp.content
-            if resp.status_code == (202, 301, 403):
-                raise Exception(f"Error: {resp.status_code} rate limit error")
-            if not resp:
-                return None
+            return None
         except Exception as error:
             if "timeout" in str(error).lower():
                 raise TimeoutError("Duckduckgo timed out error")
 
     def duck(self, query):
-        resp = self._get_url("POST", "https://duckduckgo.com/", data={"q": query})
-        vqd = self.extract_vqd(resp)
-
-        params = {"q": query, "kl": 'en-us', "p": "1", "s": "0", "df": "", "vqd": vqd, "ex": ""}
-        resp = self._get_url("GET", "https://links.duckduckgo.com/d.js", params)
-        page_data = self.text_extract_json(resp)
-
         results = []
-        for row in page_data:
-            href = row.get("u")
-            if href and href != f"http://www.google.com/search?q={query}":
-                body = self.normalize(row["a"])
-                if body:
-                    result = {
-                        "title": self.normalize(row["t"]),
-                        "href": self.normalize_url(href),
-                        "body": self.normalize(row["a"]),
-                    }
-                    results.append(result)
+        with DDGS(timeout=20, proxies=None) as ddgs:
+            for r in ddgs.text(keywords=query, max_results=3, backend="html"):
+                # print(r)
+                if r is not None:
+                    r['content'] = get_html_content(r['href'])
+                if r['content']:
+                    results.append({
+                        "title": self.normalize(r['title']),
+                        "href": self.normalize_url(r['href']),
+                        "body": r['content'],
+                    })
 
         self.query_result = results
 
     def search(self, query):
         self.duck(query)
 
     def get_first_link(self):
-        return self.query_result[0]["href"]
+        if len(self.query_result) > 0:
+            return self.query_result[0]["href"]
+        return ""
 
     @staticmethod
     def extract_vqd(html_bytes: bytes) -> str:
@@ -165,3 +159,51 @@ def normalize_url(url: str) -> str:
     @staticmethod
     def normalize(raw_html: str) -> str:
         return unescape(re.sub("<.*?>", "", raw_html)) if raw_html else ""
+
+
+def get_html_content(url):
+    # 发送HTTP请求获取网页源代码
+    if url is None:
+        return "url为空！"
+    try:
+        header = {
+            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+            'cookie': '__yjs_duid=1_b1ac9fc87dce4de5552d7cf0924fb4981686228951567; u=b0281776fd75d3eefeb3562b2a5e6534; '
+                      '__bid_n=1889b14047a51b2b754207; '
+                      'FPTOKEN=qU+ieOMqkW6y6DlsOZ+D/T'
+                      '+SCY6yS3dYvGXKibFoGBijKuUuSbc3ACFDzjlcC18wuDjNLENrw4ktAFAqnl3Akg492Lr4fbvNrkdJ'
+                      '/ZQrluIdklkNDAKYnPrpcbe2H9y7AtX+/b+FCTkSTNv5+qB3OtQQ3BXXsEen72oEoAfK+H6'
+                      '/u6ltZPdyHttJBJiXEDDS3EiUVt+S2w+8ozXENWbNt/AHeCgNUMmdeDinAKCR+nQSGK/twOoTLOU/nxBeSAazg'
+                      '+wu5K8ooRmW00Bk6XAqC4Cb829XR3UinZHRsJxt7q9biKzYQh'
+                      '+Yu5s6EHypKwpA6RPtVAC1axxbxza0l5LJ5hX8IxJXDaQ6srFoEzQ92jM0rmDynp+gT'
+                      '+3qNfEtB2PjkURvmRghGUn8wOcUUKPOqg==|mfg5DyAulnBuIm/fNO5JCrEm9g5yXrV1etiaV0jqQEw=|10'
+                      '|dcfdbf664758c47995de31b90def5ca5; PHPSESSID=18397defd82b1b3ef009662dc77fe210; '
+                      'Hm_lvt_de3f6fd28ec4ac19170f18e2a8777593=1686322028,1686360205; '
+                      'history=cid%3A2455%2Ccid%3A2476%2Ccid%3A5474%2Ccid%3A5475%2Ccid%3A2814%2Cbid%3A3667; '
+                      'Hm_lpvt_de3f6fd28ec4ac19170f18e2a8777593=1686360427'}
+        response = requests.get(url, header)
+        response.encoding = response.apparent_encoding
+        html_content = response.text
+
+        # 使用lxml解析网页源代码
+        tree = etree.HTML(html_content)
+
+        # 使用XPath选择器提取纯文本内容
+        # 这里假设要提取的内容位于<p>标签内，你可以根据实际情况调整XPath表达式
+        content = get_text(tree, url, '//p/text()')
+
+        if (len(content) == 0):
+            content = get_text(tree, url, '//body/text()')
+        return content
+    except BaseException as e:
+        print(e)
+        return None
+
+
+def get_text(tree, url, regex):
+    paragraphs = tree.xpath(regex)
+    content = []
+    # 打印提取的文本内容
+    for paragraph in paragraphs:
+        content.append(paragraph)
+    return content

src/config.py

@@ -47,6 +47,10 @@ def get_bing_api_endpoint(self):
 
     def get_bing_api_key(self):
         return self.config["API_KEYS"]["BING"]
+
+    def get_kimi_api_key(self):
+        return self.config["API_KEYS"]["KIMI"]
+
 
     def get_google_search_api_key(self):
         return self.config["API_KEYS"]["GOOGLE_SEARCH"]
@@ -109,7 +113,7 @@ def get_logging_prompts(self):
         return self.config["LOGGING"]["LOG_PROMPTS"] == "true"
 
     def get_timeout_inference(self):
-        return self.config["TIMEOUT"]["INFERENCE"]
+        return 6000000000
 
     def set_bing_api_key(self, key):
         self.config["API_KEYS"]["BING"] = key

src/init.py

@@ -26,6 +26,7 @@ def init_devika():
 
     from src.bert.sentence import SentenceBert
 
+    # using llm instead
     logger.info("Loading sentence-transformer BERT models...")
     prompt = "Light-weight keyword extraction exercise for BERT model loading.".strip()
     SentenceBert(prompt).extract_keywords()

src/llm/china_client.py

@@ -0,0 +1,21 @@
+from openai import OpenAI
+
+from src.config import Config
+
+
+class China:
+    def __init__(self):
+        config = Config()
+        api_key = config.get_kimi_api_key()
+        self.client = OpenAI(
+            api_key=api_key,
+            base_url="https://api.moonshot.cn/v1",
+        )
+
+    def inference(self, model_id: str, prompt: str) -> str:
+        result = self.client.chat.completions.create(
+            model=model_id,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.3,
+        )
+        return result.choices[0].message.content