ai: gemini: files

.config/quickshell/ii/modules/common/functions/StringUtils.qml

@@ -40,7 +40,6 @@ Singleton {
       * @returns { string }
       */
     function shellSingleQuoteEscape(str) {
-        //  escape single quotes
         return String(str)
         // .replace(/\\/g, '\\\\')
         .replace(/'/g, "'\\''");

.config/quickshell/ii/modules/sidebarLeft/AiChat.qml

@@ -38,6 +38,13 @@ Item {
     }
 
     property var allCommands: [
+        {
+            name: "attach",
+            description: Translation.tr("Attach a file. Only works with Gemini."),
+            execute: (args) => {
+                Ai.attachFile(args.join(" ").trim());
+            }
+        },
         {
             name: "model",
             description: Translation.tr("Choose model"),
@@ -421,13 +428,13 @@ Inline w/ backslash and round brackets \\(e^{i\\pi} + 1 = 0\\)
 
         Rectangle { // Input area
             id: inputWrapper
-            property real columnSpacing: 5
+            property real spacing: 5
             Layout.fillWidth: true
             radius: Appearance.rounding.small
             color: Appearance.colors.colLayer1
-            implicitWidth: messageInputField.implicitWidth
+            implicitHeight: Math.max(inputFieldRowLayout.implicitHeight + inputFieldRowLayout.anchors.topMargin
-            implicitHeight: Math.max(inputFieldRowLayout.implicitHeight + inputFieldRowLayout.anchors.topMargin 
+                + commandButtonsRow.implicitHeight + commandButtonsRow.anchors.bottomMargin + spacing, 45)
-                + commandButtonsRow.implicitHeight + commandButtonsRow.anchors.bottomMargin + columnSpacing, 45)
+                + (attachedFileIndicator.implicitHeight + spacing + attachedFileIndicator.anchors.topMargin)
             clip: true
             border.color: Appearance.colors.colOutlineVariant
             border.width: 1
@@ -436,12 +443,26 @@ Inline w/ backslash and round brackets \\(e^{i\\pi} + 1 = 0\\)
                 animation: Appearance.animation.elementMove.numberAnimation.createObject(this)
             }
 
+            AttachedFileIndicator {
+                id: attachedFileIndicator
+                anchors {
+                    top: parent.top
+                    left: parent.left
+                    right: parent.right
+                    margins: visible ? 5 : 0
+                }
+                filePath: Ai.pendingFilePath
+                onRemove: Ai.attachFile("")
+            }
+
             RowLayout { // Input field and send button
                 id: inputFieldRowLayout
-                anchors.top: parent.top
+                anchors {
-                anchors.left: parent.left
+                    top: attachedFileIndicator.bottom
-                anchors.right: parent.right
+                    left: parent.left
-                anchors.topMargin: 5
+                    right: parent.right
+                    topMargin: 5
+                }
                 spacing: 0
 
                 StyledTextArea { // The actual TextArea

.config/quickshell/ii/modules/sidebarLeft/aiChat/AiMessage.qml

@@ -233,6 +233,15 @@ Rectangle {
             }
         }
 
+        Loader {
+            Layout.fillWidth: true
+            active: root.messageData?.localFilePath && root.messageData?.localFilePath.length > 0
+            sourceComponent: AttachedFileIndicator {
+                filePath: root.messageData?.localFilePath
+                canRemove: false
+            }
+        }
+
         ColumnLayout { // Message content
             id: messageContentColumnLayout
 

.config/quickshell/ii/modules/sidebarLeft/aiChat/AnnotationSourceButton.qml

@@ -1,7 +1,7 @@
 import qs.modules.common
 import qs.modules.common.widgets
-import qs.services
 import qs.modules.common.functions
+import qs.services
 import QtQuick
 import QtQuick.Layouts
 import Quickshell.Hyprland

.config/quickshell/ii/modules/sidebarLeft/aiChat/AttachedFileIndicator.qml

@@ -0,0 +1,152 @@
+pragma ComponentBehavior: Bound
+
+import QtQuick
+import QtQuick.Layouts
+import Qt5Compat.GraphicalEffects
+import Quickshell.Io
+import qs
+import qs.modules.common
+import qs.modules.common.widgets
+import qs.services
+
+Rectangle {
+    id: root
+
+    signal remove()
+    property bool canRemove: true
+    property string filePath: ""
+    property string mimeType: ""
+    property real maxHeight: 200
+    property real imageWidth: -1
+    property real imageHeight: -1
+    property real scale: Math.min(root.maxHeight / imageHeight, root.width / imageWidth)
+    onFilePathChanged: refresh()
+    visible: filePath !== ""
+
+    function refresh() {
+        root.mimeType = "";
+        root.imageWidth = -1;
+        root.imageHeight = -1;
+        fileTypeProc.exec(["file", "-b", "--mime-type", filePath]);
+    }
+
+    Process {
+        id: fileTypeProc
+        command: ["file", "-b", "--mime-type", filePath]
+        stdout: StdioCollector {
+            onStreamFinished: {
+                root.mimeType = this.text;
+                if (root.mimeType.startsWith("image/"))
+                    imageSizeProc.exec(["identify", "-format", "%wx%h", filePath]);
+            }
+        }
+    }
+
+    Process {
+        id: imageSizeProc
+        command: ["identify", "-format", "%wx%h", filePath]
+        stdout: StdioCollector {
+            onStreamFinished: {
+                const dimensions = this.text.split("x");
+                root.imageWidth = parseInt(dimensions[0]);
+                root.imageHeight = parseInt(dimensions[1]);
+            }
+        }
+    }
+
+    // Styles/widgets
+    property real horizontalPadding: 10
+    property real verticalPadding: 10
+    radius: Appearance.rounding.small - anchors.margins
+    color: Appearance.colors.colLayer2
+    implicitHeight: visible ? (contentItem.implicitHeight + verticalPadding * 2) : 0
+
+    ColumnLayout {
+        id: contentItem
+        anchors {
+            fill: parent
+            leftMargin: root.horizontalPadding
+            rightMargin: root.horizontalPadding
+            topMargin: root.verticalPadding
+            bottomMargin: root.verticalPadding
+        }
+
+        RowLayout {
+            MaterialSymbol {
+                Layout.alignment: Qt.AlignTop
+                text: {
+                    if (root.mimeType.startsWith("image/"))
+                        return "image";
+                    if (root.mimeType.startsWith("audio/"))
+                        return "music_note";
+                    if (root.mimeType.startsWith("video/"))
+                        return "movie";
+                    if (root.mimeType === "application/pdf")
+                        return "picture_as_pdf";
+                    if (root.mimeType.startsWith("text/"))
+                        return "description";
+                    return "file_present";
+                }
+                iconSize: Appearance.font.pixelSize.hugeass
+            }
+
+            StyledText {
+                Layout.fillWidth: true
+                Layout.topMargin: 4
+                text: root.filePath
+                font.pixelSize: Appearance.font.pixelSize.smaller
+                font.family: Appearance.font.family.monospace
+                wrapMode: Text.Wrap
+            }
+
+            RippleButton {
+                visible: root.canRemove
+                Layout.alignment: Qt.AlignTop
+                buttonRadius: Appearance.rounding.full
+                colBackground: Appearance.colors.colLayer2
+                implicitHeight: 28
+                implicitWidth: 28
+                contentItem: MaterialSymbol {
+                    anchors.centerIn: parent
+                    text: "close"
+                    horizontalAlignment: Text.AlignHCenter
+                    iconSize: Appearance.font.pixelSize.larger
+                    color: Appearance.colors.colOnSurfaceVariant
+                }
+
+                onClicked: root.remove()
+            }
+        }
+
+        Loader {
+            id: imagePreviewLoader
+            visible: (root.imageWidth != -1) && (root.imageHeight != -1)
+            Layout.alignment: Qt.AlignHCenter
+            sourceComponent: Item {
+                implicitHeight: root.imageHeight * root.scale
+                implicitWidth: imagePreview.implicitWidth
+                Image {
+                    id: imagePreview
+                    anchors.fill: parent
+                    source: Qt.resolvedUrl(root.filePath)
+                    fillMode: Image.PreserveAspectFit
+                    antialiasing: true
+                    asynchronous: true
+                    width: root.imageWidth * root.scale
+                    height: root.imageHeight * root.scale
+                    sourceSize.width: root.imageWidth * root.scale
+                    sourceSize.height: root.imageHeight * root.scale
+
+                    layer.enabled: true
+                    layer.effect: OpacityMask {
+                        maskSource: Rectangle {
+                            width: imagePreview.width
+                            height: imagePreview.height
+                            radius: Appearance.rounding.normal
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

.config/quickshell/ii/services/Ai.qml

@@ -368,6 +368,9 @@ Singleton {
         }
     }
 
+    property string requestScriptFilePath: "/tmp/quickshell/ai/request.sh"
+    property string pendingFilePath: ""
+
     Component.onCompleted: {
         setModel(currentModelId, false, false); // Do necessary setup for model
     }
@@ -617,9 +620,13 @@ Singleton {
         root.tokenCount.total = -1;
     }
 
+    FileView {
+        id: requesterScriptFile
+    }
+
     Process {
         id: requester
-        property list<string> baseCommand: ["bash", "-c"]
+        property list<string> baseCommand: ["bash"]
         property AiMessageData message
         property ApiStrategy currentStrategy
 
@@ -645,7 +652,7 @@ Singleton {
             const endpoint = root.currentApiStrategy.buildEndpoint(model);
             const messageArray = root.messageIDs.map(id => root.messageByID[id]);
             const filteredMessageArray = messageArray.filter(message => message.role !== Ai.interfaceRole);
-            const data = root.currentApiStrategy.buildRequestData(model, filteredMessageArray, root.systemPrompt, root.temperature, root.tools[model.api_format][root.currentTool]);
+            const data = root.currentApiStrategy.buildRequestData(model, filteredMessageArray, root.systemPrompt, root.temperature, root.tools[model.api_format][root.currentTool], root.pendingFilePath);
             // console.log("[Ai] Request data: ", JSON.stringify(data, null, 2));
 
             let requestHeaders = {
@@ -677,14 +684,31 @@ Singleton {
             /* Get authorization header from strategy */
             const authHeader = requester.currentStrategy.buildAuthorizationHeader(root.apiKeyEnvVarName);
             
+            /* Script shebang */
+            const scriptShebang = "#!/usr/bin/env bash\n";
+
+            /* Create extra setup when there's an attached file */
+            let scriptFileSetupContent = ""
+            if (root.pendingFilePath && root.pendingFilePath.length > 0) {
+                requester.message.localFilePath = root.pendingFilePath;
+                scriptFileSetupContent = requester.currentStrategy.buildScriptFileSetup(root.pendingFilePath);
+                root.pendingFilePath = ""
+            }
+
             /* Create command string */
-            const requestCommandString = `curl --no-buffer "${endpoint}"`
+            let scriptRequestContent = ""
+            scriptRequestContent += `curl --no-buffer "${endpoint}"`
                 + ` ${headerString}`
                 + (authHeader ? ` ${authHeader}` : "")
-                + ` -d '${CF.StringUtils.shellSingleQuoteEscape(JSON.stringify(data))}'`
+                + ` --data '${CF.StringUtils.shellSingleQuoteEscape(JSON.stringify(data))}'`
+                + "\n"
             
             /* Send the request */
-            requester.command = baseCommand.concat([requestCommandString]);
+            const scriptContent = requester.currentStrategy.finalizeScriptContent(scriptShebang + scriptFileSetupContent + scriptRequestContent)
+            const shellScriptPath = CF.FileUtils.trimFileProtocol(root.requestScriptFilePath)
+            requesterScriptFile.path = Qt.resolvedUrl(shellScriptPath)
+            requesterScriptFile.setText(scriptContent)
+            requester.command = baseCommand.concat([shellScriptPath]);
             requester.running = true
         }
 
@@ -698,7 +722,7 @@ Singleton {
                 try {
                     const result = requester.currentStrategy.parseResponseLine(data, requester.message);
                     // console.log("[Ai] Parsed response result: ", JSON.stringify(result, null, 2));
-                    
+
                     if (result.functionCall) {
                         requester.message.functionCall = result.functionCall;
                         root.handleFunctionCall(result.functionCall.name, result.functionCall.args, requester.message);
@@ -742,6 +766,10 @@ Singleton {
         requester.makeRequest();
     }
 
+    function attachFile(filePath: string) {
+        root.pendingFilePath = CF.FileUtils.trimFileProtocol(filePath);
+    }
+
     function createFunctionOutputMessage(name, output, includeOutputInChat = true) {
         return aiMessageComponent.createObject(root, {
             "role": "user",
@@ -841,6 +869,9 @@ Singleton {
             return ({
                 "role": message.role,
                 "rawContent": message.rawContent,
+                "fileMimeType": message.fileMimeType,
+                "fileUri": message.fileUri,
+                "localFilePath": message.localFilePath,
                 "model": message.model,
                 "thinking": false,
                 "done": true,
@@ -858,7 +889,7 @@ Singleton {
         id: chatSaveFile
         property string chatName: "chat"
         path: `${Directories.aiChats}/${chatName}.json`
-        blockLoading: true
+        blockLoading: true // Prevent race conditions
     }
 
     /**
@@ -894,6 +925,9 @@ Singleton {
                     "role": message.role,
                     "rawContent": message.rawContent,
                     "content": message.rawContent,
+                    "fileMimeType": message.fileMimeType,
+                    "fileUri": message.fileUri,
+                    "localFilePath": message.localFilePath,
                     "model": message.model,
                     "thinking": message.thinking,
                     "done": message.done,

.config/quickshell/ii/services/ai/AiMessageData.qml

@@ -7,6 +7,9 @@ QtObject {
     property string role
     property string content
     property string rawContent
+    property string fileMimeType
+    property string fileUri
+    property string localFilePath
     property string model
     property bool thinking: true
     property bool done: false

.config/quickshell/ii/services/ai/ApiStrategy.qml

@@ -2,9 +2,11 @@ import QtQuick
 
 QtObject {
     function buildEndpoint(model: AiModel): string { throw new Error("Not implemented") }
-    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) { throw new Error("Not implemented") }
+    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) { throw new Error("Not implemented") }
     function buildAuthorizationHeader(apiKeyEnvVarName: string): string { throw new Error("Not implemented") }
     function parseResponseLine(line: string, message: AiMessageData) { throw new Error("Not implemented") }
     function onRequestFinished(message: AiMessageData): var { return {} } // Default: no special handling
     function reset() { } // Reset any internal state if needed
+    function buildScriptFileSetup(filePath) { return "" } // Default: no setup
+    function finalizeScriptContent(scriptContent: string): string { return scriptContent } // Optionally modify/finalize script
 }

.config/quickshell/ii/services/ai/GeminiApiStrategy.qml

@@ -1,6 +1,12 @@
 import QtQuick
+import qs.modules.common.functions as CF
 
 ApiStrategy {
+    readonly property string apiKeyEnvVarName: "API_KEY"
+    readonly property string fileUriVarName: "file_uri"
+    readonly property string fileMimeTypeVarName: "MIME_TYPE"
+    readonly property string fileUriSubstitutionString: "{{ fileUriVarName }}"
+    readonly property string fileMimeTypeSubstitutionString: "{{ fileMimeTypeVarName }}"
     property string buffer: ""
     
     function buildEndpoint(model: AiModel): string {
@@ -9,39 +15,57 @@ ApiStrategy {
         return result;
     }
 
-    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) {
+    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) {
-        let baseData = {
+        let contents = messages.map(message => {
-            "contents": messages.map(message => {
+            // console.log("[AI] Building request data for message:", JSON.stringify(message, null, 2));
-                const geminiApiRoleName = (message.role === "assistant") ? "model" : message.role;
+            const geminiApiRoleName = (message.role === "assistant") ? "model" : message.role;
-                const usingSearch = tools[0]?.google_search !== undefined
+            const usingSearch = tools[0]?.google_search !== undefined
-                if (!usingSearch && message.functionCall != undefined && message.functionName.length > 0) {
+            if (!usingSearch && message.functionCall != undefined && message.functionName.length > 0) {
-                    return {
+                return {
-                        "role": geminiApiRoleName,
+                    "role": geminiApiRoleName,
-                        "parts": [{
+                    "parts": [{
-                            functionCall: {
+                        functionCall: {
-                                "name": message.functionName,
+                            "name": message.functionName,
-                            }
+                        }
-                        }]
+                    }]
-                    }
-                }
-                if (!usingSearch && message.functionResponse != undefined && message.functionName.length > 0) {
-                    return {
-                        "role": geminiApiRoleName,
-                        "parts": [{ 
-                            functionResponse: {
-                                "name": message.functionName,
-                                "response": { "content": message.functionResponse }
-                            }
-                        }]
-                    }
                 }
+            }
+            if (!usingSearch && message.functionResponse != undefined && message.functionName.length > 0) {
                 return {
                     "role": geminiApiRoleName,
                     "parts": [{ 
-                        text: message.rawContent,
+                        functionResponse: {
+                            "name": message.functionName,
+                            "response": { "content": message.functionResponse }
+                        }
                     }]
                 }
-            }),
+            }
+            return {
+                "role": geminiApiRoleName,
+                "parts": [
+                    { text: message.rawContent },
+                    ...(message.fileUri && message.fileUri.length > 0 ? [{ 
+                        "file_data": {
+                            "mime_type": message.fileMimeType,
+                            "file_uri": message.fileUri
+                        }
+                    }] : [])
+                ]
+            }
+        })
+        if (filePath && filePath.length > 0) {
+            const trimmedFilePath = CF.FileUtils.trimFileProtocol(filePath);
+            // Add file_data part to the last message's parts array
+            contents[contents.length - 1].parts.unshift({
+                file_data: {
+                    mime_type: fileMimeTypeSubstitutionString,
+                    file_uri: fileUriSubstitutionString
+                }
+            });
+        }
+        let baseData = {
+            "contents": contents,
             "tools": tools,
             "system_instruction": {
                 "parts": [{ text: systemPrompt }]
@@ -50,6 +74,7 @@ ApiStrategy {
                 "temperature": temperature,
             },
         };
+        // print("Gemini API call payload:", JSON.stringify(baseData, null, 2));
         return model.extraParams ? Object.assign({}, baseData, model.extraParams) : baseData;
     }
 
@@ -78,8 +103,18 @@ ApiStrategy {
         try {
             if (buffer.length === 0) return {};
             const dataJson = JSON.parse(buffer);
+
+            // Uploaded file
+            if (dataJson.uploadedFile) {
+                message.fileUri = dataJson.uploadedFile.uri;
+                message.fileMimeType = dataJson.uploadedFile.mimeType;
+                return ({})
+            }
+
+            // No candidates?
             if (!dataJson.candidates) return {};
             
+            // Finished?
             if (dataJson.candidates[0]?.finishReason) {
                 finished = true;
             }
@@ -152,4 +187,55 @@ ApiStrategy {
     function reset() {
         buffer = "";
     }
+
+    function buildScriptFileSetup(filePath) {
+        const trimmedFilePath = CF.FileUtils.trimFileProtocol(filePath);
+        let content = ""
+
+        // print("file path:", filePath)
+        // print("trimmed file path:", trimmedFilePath)
+        // print("escaped file path:", CF.StringUtils.shellSingleQuoteEscape(trimmedFilePath))
+
+        content += `IMAGE_PATH='${CF.StringUtils.shellSingleQuoteEscape(trimmedFilePath)}'\n`;
+        content += `${fileMimeTypeVarName}=$(file -b --mime-type "$IMAGE_PATH")\n`;
+        content += 'NUM_BYTES=$(wc -c < "${IMAGE_PATH}")\n';
+        content += 'tmp_header_file="/tmp/quickshell/ai/upload-header.tmp"\n';
+        content += 'tmp_file_info_file="/tmp/quickshell/ai/file-info.json.tmp"\n';
+
+        // Initial resumable request defining metadata.
+        // The upload url is in the response headers dump them to a file.
+        content += 'curl "https://generativelanguage.googleapis.com/upload/v1beta/files"'
+            + ` -H "x-goog-api-key: \$${apiKeyEnvVarName}"`
+            + ' -D $tmp_header_file'
+            + ' -H "X-Goog-Upload-Protocol: resumable"'
+            + ' -H "X-Goog-Upload-Command: start"'
+            + ' -H "X-Goog-Upload-Header-Content-Length: ${NUM_BYTES}"'
+            + ` -H "X-Goog-Upload-Header-Content-Type: \${${fileMimeTypeVarName}}"`
+            + ' -H "Content-Type: application/json"'
+            + ` -d "{'file': {'display_name': 'Image'}}" 2> /dev/null`
+            + '\n';
+
+        // Get file upload header
+        content += 'upload_url=$(grep -i "x-goog-upload-url: " "${tmp_header_file}" | cut -d" " -f2 | tr -d "\r")\n';
+        content += 'rm "${tmp_header_file}"\n';
+
+        // Upload the actual file
+        content += 'curl "${upload_url}"'
+            + ` -H "x-goog-api-key: \$${apiKeyEnvVarName}"`
+            + ' -H "Content-Length: ${NUM_BYTES}"'
+            + ' -H "X-Goog-Upload-Offset: 0"'
+            + ' -H "X-Goog-Upload-Command: upload, finalize"'
+            + ' --data-binary "@${IMAGE_PATH}" 2> /dev/null > "${tmp_file_info_file}"'
+            + '\n';
+
+        content += `${fileUriVarName}=$(jq -r ".file.uri" "$tmp_file_info_file")\n`
+        content += `printf "{\\"uploadedFile\\": {\\"uri\\": \\"$${fileUriVarName}\\", \\"mimeType\\": \\"$${fileMimeTypeVarName}\\"}}\\n,\\n"\n`
+
+        return content
+    }
+
+    function finalizeScriptContent(scriptContent: string): string {
+        return scriptContent.replace(fileMimeTypeSubstitutionString, `'"\$${fileMimeTypeVarName}"'`)
+                            .replace(fileUriSubstitutionString, `'"\$${fileUriVarName}"'`);
+    }
 }

.config/quickshell/ii/services/ai/MistralApiStrategy.qml

@@ -8,7 +8,7 @@ ApiStrategy {
         return model.endpoint;
     }
 
-    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) {
+    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) {
         let baseData = {
             "model": model.model,
             "messages": [

.config/quickshell/ii/services/ai/OpenAiApiStrategy.qml

@@ -8,7 +8,7 @@ ApiStrategy {
         return model.endpoint;
     }
 
-    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) {
+    function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) {
         let baseData = {
             "model": model.model,
             "messages": [