add mistral_ocr, init can expand @filename in paths

Author: iovdin

README.md

@@ -17,6 +17,7 @@ Basic toolset for [Tune](https://github.com/iovdin/tune).
   - [search_tools](#search_tools) find appropriate tools for a task
   - [osa](#osa) manage reminders/notes/calendar (AppleScript/macOS)
   - [jina_r](#jina_r) fetch webpage content
+  - [mistral_ocr](#mistral_ocr) extract text from documents and images via OCR
   - [websearch](#websearch) search the web with web-enabled llms
   - [list](#list) keep list of tasks todo (loops for LLM)
   - [sqlite](#sqlite) execute sqlite queries 
@@ -291,6 +292,40 @@ Tune is a versatile toolkit designed for developers and users to effectively int
 <cut for brevity>
 ```
 
+### `mistral_ocr`
+Extract text from documents and images using the [Mistral OCR API](https://mistral.ai/). Requires a `MISTRAL_KEY` set in `.env`.
+
+Supports documents: `.pdf`, `.docx`, `.pptx`, `.txt`, `.epub`, `.xml`, `.rtf`, `.odt`, `.bib`, `.fb2`, `.ipynb`, `.tex`, `.opml`, `.1`, `.man`
+
+Supports images: `.jpg`, `.jpeg`, `.png`, `.avif`, `.tiff`, `.gif`, `.heic`, `.heif`, `.bmp`, `.webp`
+
+```chat
+user: @mistral_ocr
+extract text from invoice.pdf
+
+tool_call: mistral_ocr {"filename":"invoice.pdf"}
+tool_result:
+# Invoice #1042
+
+**Date:** 2024-04-01
+**Bill To:** Acme Corp
+
+| Description        | Amount  |
+|--------------------|---------|
+| Consulting (10h)   | $1500   |
+| Hosting (1 month)  | $50     |
+| **Total**          | $1550   |
+
+user: 
+what does this screenshot say?
+
+tool_call: mistral_ocr {"filename":"screenshot.png"}
+tool_result:
+## System Alert
+
+Your disk usage has exceeded 90%. Please free up space to avoid performance issues.
+```
+
 ### `websearch`
 Search the web with web enabled llms
 Supports search with  `perplexity/sonar`, `perplexity/sonar-pro`, `gpt-4o-search-preview`, `gpt-4o-mini-search-preview` models via the `model` parameter (defaults to `perplexity/sonar`).

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tune-basic-toolset",
-  "version": "0.1.21",
+  "version": "0.1.22",
   "description": "Basic toolset for tune",
   "main": "src/index.js",
   "files": [

src/init.proc.js

@@ -6,10 +6,7 @@ module.exports = async function init(node, args, ctx) {
 
   // include file
   if (content.indexOf("@") === 0) {
-    return {
-      type: "text", 
-      read: async () => this.read(content.replace(/^@{1,2}/, "")) 
-    }
+    return  ctx.resolve(content.replace(/^@{1,2}/, "")) 
   }
 
   return {

src/jina_r.schema.json

@@ -18,5 +18,5 @@
     },
     "required": ["url"]
   },
-  "$escape_output": false
+  "$escape_output": true
 }

src/mistral_ocr.schema.json

@@ -0,0 +1,14 @@
+{
+  "description": "Perform OCR on a local files using Mistral OCR API, returns extracted text as markdown",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "filename": {
+        "type": "string",
+        "description": "Path to the file to perform OCR on"
+      }
+    },
+    "required": ["filename"]
+  },
+  "$escape_output": true
+}

src/mistral_ocr.tool.js

@@ -0,0 +1,74 @@
+const path = require('path');
+
+const MIME_TYPES = {
+  // Documents → document_url
+  '.pdf':   { kind: 'document', mime: 'application/pdf' },
+  '.docx':  { kind: 'document', mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' },
+  '.pptx':  { kind: 'document', mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation' },
+  '.txt':   { kind: 'document', mime: 'text/plain' },
+  '.epub':  { kind: 'document', mime: 'application/epub+zip' },
+  '.xml':   { kind: 'document', mime: 'application/xml' },
+  '.rtf':   { kind: 'document', mime: 'application/rtf' },
+  '.odt':   { kind: 'document', mime: 'application/vnd.oasis.opendocument.text' },
+  '.bib':   { kind: 'document', mime: 'application/x-bibtex' },
+  '.fb2':   { kind: 'document', mime: 'application/x-fictionbook+xml' },
+  '.ipynb': { kind: 'document', mime: 'application/x-ipynb+json' },
+  '.tex':   { kind: 'document', mime: 'application/x-tex' },
+  '.opml':  { kind: 'document', mime: 'text/x-opml' },
+  '.1':     { kind: 'document', mime: 'application/x-troff-man' },
+  '.man':   { kind: 'document', mime: 'application/x-troff-man' },
+  // Images → image_url
+  '.jpg':   { kind: 'image', mime: 'image/jpeg' },
+  '.jpeg':  { kind: 'image', mime: 'image/jpeg' },
+  '.png':   { kind: 'image', mime: 'image/png' },
+  '.avif':  { kind: 'image', mime: 'image/avif' },
+  '.tiff':  { kind: 'image', mime: 'image/tiff' },
+  '.gif':   { kind: 'image', mime: 'image/gif' },
+  '.heic':  { kind: 'image', mime: 'image/heic' },
+  '.heif':  { kind: 'image', mime: 'image/heif' },
+  '.bmp':   { kind: 'image', mime: 'image/bmp' },
+  '.webp':  { kind: 'image', mime: 'image/webp' },
+};
+
+module.exports = async function mistralOcr({ filename }, ctx) {
+  const apiKey = await ctx.read("MISTRAL_KEY");
+  if (!apiKey) {
+    throw new Error("MISTRAL_KEY is not set");
+  }
+
+  const ext = path.extname(filename).toLowerCase();
+  const typeInfo = MIME_TYPES[ext];
+  if (!typeInfo) {
+    throw new Error(`Unsupported file extension: ${ext}`);
+  }
+
+  const buf = await ctx.read(filename);
+  const base64 = buf.toString('base64');
+  const dataUrl = `data:${typeInfo.mime};base64,${base64}`;
+
+  const document = typeInfo.kind === 'image'
+    ? { type: 'image_url',    image_url:    dataUrl }
+    : { type: 'document_url', document_url: dataUrl };
+
+  const response = await fetch("https://api.mistral.ai/v1/ocr", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model: "mistral-ocr-latest",
+      document,
+      include_image_base64: true,
+    }),
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Mistral OCR error: ${response.status} ${response.statusText} - ${errorText}`);
+  }
+
+  const result = await response.json();
+
+  return result.pages.map(page => page.markdown).join("\n\n");
+};

src/sh.schema.json

@@ -5,11 +5,11 @@
     "properties": {
       "text": {
         "type": "string",
-        "description": "The shell command to execute"
+        "description": "The shell command to execute, always prepend input with comment explaining the command so user that is not familiar with shell can understand and estimate how dangerous the command is"
       },
       "host": {
         "type": "string",
-        "description": "remote host like user@host.com to execute the shell comand on (uses ssh)"
+        "description": "remote host like user@host.com to execute the shell comand on (uses ssh), default is empty (local shell) "
       }
     },
     "required": ["text"]