Merge branch 'implement-read-image-tool'

Author: qorzj

package-lock.json

@@ -41,6 +41,7 @@
     "ink-gradient": "^4.0.1",
     "react": "^19.2.5",
     "read-package-up": "^12.0.0",
+    "sharp": "^0.34.5",
     "yargs": "^18.0.0"
   },
   "devDependencies": {

packages/cli/package.json

@@ -5,7 +5,7 @@ import * as os from "os";
 import * as path from "path";
 import { render, type Instance } from "ink";
 import { UpdatePrompt, type UpdatePromptChoice } from "../ui";
-import { killProcessTree } from "@vegamo/deepcode-core";
+import { killProcessTree, TENCENT_MIRROR_REGISTRY } from "@vegamo/deepcode-core";
 import type { PackageJson } from "../utils/package";
 
 type UpdateState = {
@@ -21,7 +21,6 @@ type UpdateState = {
 const UPDATE_STATE_FILE = "update-check.json";
 const NPM_VIEW_TIMEOUT_MS = 5000;
 const MAX_NPM_VIEW_OUTPUT_CHARS = 64 * 1024;
-const TENCENT_MIRROR_REGISTRY = "https://mirrors.cloud.tencent.com/npm/";
 export const UPDATE_SUCCESS_MESSAGE = "🎉 Update ran successfully! Please restart Deep Code.";
 
 export async function promptForPendingUpdate(packageInfo: PackageJson): Promise<{ installed: boolean }> {

packages/cli/src/common/update-check.ts

@@ -33,6 +33,7 @@
     "gray-matter": "^4.0.3",
     "ignore": "^7.0.5",
     "openai": "^6.35.0",
+    "sharp": "^0.34.5",
     "undici": "^7.29.0",
     "zod": "^4.4.3"
   }

packages/core/package.json

@@ -0,0 +1 @@
+export const TENCENT_MIRROR_REGISTRY = "https://mirrors.cloud.tencent.com/npm/";

packages/core/src/common/npm-registry.ts

@@ -219,12 +219,12 @@ export function describeToolPermissionRequest(options: {
   const name = options.toolCall.function.name;
   const args = parseToolArgumentsForPermissions(options.toolCall.function.arguments);
 
-  if (name === "read" || name === "Read") {
+  if (name === "read" || name === "Read" || name === "ReadImage") {
     const filePath = typeof args.file_path === "string" ? args.file_path : "";
     return {
       toolCallId: options.toolCall.id,
       name,
-      command: formatToolPathCommand("read", filePath),
+      command: formatToolPathCommand(name === "ReadImage" ? "read-image" : "read", filePath),
       scopes:
         filePath && !isPathInAnyDirectory(options.projectRoot, filePath, options.readPermissionExemptPaths)
           ? [isPathInProject(options.projectRoot, filePath) ? "read-in-cwd" : "read-out-cwd"]

packages/core/src/common/permissions.ts

@@ -1,4 +1,5 @@
 import type OpenAI from "openai";
+import type sharp from "sharp";
 import type { ReasoningEffort } from "../settings";
 
 export type CreateOpenAIClient = () => {
@@ -28,11 +29,14 @@ export type ToolCall = {
 
 export type PluginRateLimitedTool = "UnderstandImage" | "WebSearch";
 
+export type SharpLoader = () => Promise<typeof sharp>;
+
 export type ToolExecutionContext = {
   sessionId: string;
   projectRoot: string;
   toolCall: ToolCall;
   createOpenAIClient?: CreateOpenAIClient;
+  loadSharp?: SharpLoader;
   onProcessStart?: (processId: string | number, command: string) => void;
   onProcessExit?: (processId: string | number) => void;
   onProcessStdout?: (processId: string | number, chunk: string) => void;

packages/core/src/common/tool-types.ts

@@ -80,11 +80,13 @@ export type {
   BackgroundProcessCompletion,
   ToolExecutionFollowUpMessage,
   PluginRateLimitedTool,
+  SharpLoader,
 } from "./common/tool-types";
 
 // Tool handlers
 export { handleBashTool, clearSessionWorkingDir } from "./tools/bash-handler";
 export { handleReadTool } from "./tools/read-handler";
+export { handleReadImageTool } from "./tools/read-image-handler";
 export { handleWriteTool } from "./tools/write-handler";
 export { handleEditTool } from "./tools/edit-handler";
 export { handleUpdatePlanTool } from "./tools/update-plan-handler";
@@ -104,6 +106,7 @@ export { readTextFileWithMetadata, writeTextFile, buildDiffPreview, ensureParent
 export { normalizeFilePath, getSnippet, clearSessionState, recordFileState, getFileState } from "./common/state";
 export { GitFileHistory } from "./common/file-history";
 export { killProcessTree } from "./common/process-tree";
+export { TENCENT_MIRROR_REGISTRY } from "./common/npm-registry";
 export { launchNotifyScript } from "./common/notify";
 export { reportNewPrompt } from "./common/telemetry";
 export { DEEPSEEK_V4_MODELS, supportsMultimodal, defaultsToThinkingMode } from "./common/model-capabilities";

packages/core/src/index.ts

@@ -584,7 +584,7 @@ export function getTools(_options: PromptToolOptions = {}, externalTools: ToolDe
       type: "function",
       function: {
         name: "read",
-        description: "Read files from the filesystem (text, images, notebooks).",
+        description: "Read text files and notebooks from the filesystem. Image files require a dedicated image tool.",
         parameters: {
           type: "object",
           properties: {
@@ -696,7 +696,27 @@ export function getTools(_options: PromptToolOptions = {}, externalTools: ToolDe
     },
   });
 
-  if (!supportsMultimodal(_options.model ?? "", _options.multimodal)) {
+  if (supportsMultimodal(_options.model ?? "", _options.multimodal)) {
+    tools.push({
+      type: "function",
+      function: {
+        name: "ReadImage",
+        description:
+          "Read a PNG, JPEG, WebP, or GIF file and return the image itself. Large images are validated and downscaled before the next model request.",
+        parameters: {
+          type: "object",
+          properties: {
+            file_path: {
+              type: "string",
+              description: "The absolute path of the PNG, JPEG, WebP, or GIF image to read.",
+            },
+          },
+          required: ["file_path"],
+          additionalProperties: false,
+        },
+      },
+    });
+  } else {
     tools.push({
       type: "function",
       function: {

packages/core/src/prompt.ts

@@ -25,6 +25,7 @@ import {
   type ProcessTimeoutControl,
   type ProcessTimeoutInfo,
   type PluginRateLimitedTool,
+  type SharpLoader,
   type ToolCallExecution,
   type ToolExecutionHooks,
   type ToolExecutionFollowUpMessage,
@@ -353,6 +354,7 @@ export type SessionManagerOptions = {
   onLlmStreamProgress?: (progress: LlmStreamProgress) => void;
   onMcpStatusChanged?: () => void;
   onProcessStdout?: (pid: number, chunk: string) => void;
+  loadSharp?: SharpLoader;
   nonInteractive?: boolean;
 };
 
@@ -404,7 +406,7 @@ export class SessionManager {
     this.onMcpStatusChanged = options.onMcpStatusChanged;
     this.onProcessStdout = options.onProcessStdout;
     this.nonInteractive = options.nonInteractive === true;
-    this.toolExecutor = new ToolExecutor(this.projectRoot, this.createOpenAIClient, this.mcpManager);
+    this.toolExecutor = new ToolExecutor(this.projectRoot, this.createOpenAIClient, this.mcpManager, options.loadSharp);
     this.mcpManager.prepare(this.getResolvedSettings().mcpServers);
     this.messageConverter = new OpenAIMessageConverter({
       renderInitPrompt: () => this.renderInitCommandPrompt(),
@@ -2869,7 +2871,7 @@ ${agentInstructions}
 
     const value = args[firstKey];
     const text = typeof value === "string" ? value : JSON.stringify(value);
-    if (toolName === "read" && text.startsWith(this.projectRoot)) {
+    if ((toolName === "read" || toolName === "ReadImage") && text.startsWith(this.projectRoot)) {
       return text.slice(this.projectRoot.length).replace(/^[\\/]/, "");
     }
     return text;