kenji/illogical-impulse
1
0
Fork 0
forked from Shinonome/dots-hyprland
Code Pull Requests Activity

ai: gemini: files

Browse Source
This commit is contained in:
end-4
2025-08-21 22:53:11 +07:00
parent be1974a89e
commit 690e934a46
11 changed files with 353 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/quickshell/ii/modules/common/functions/StringUtils.qml
-1
View File
@@ -40,7 +40,6 @@ Singleton {
* @returns { string }
*/
function shellSingleQuoteEscape(str) {
// escape single quotes
return String(str)
// .replace(/\\/g, '\\\\')
.replace(/'/g, "'\\''");
.config/quickshell/ii/modules/sidebarLeft/AiChat.qml
+29 -8
View File
@@ -38,6 +38,13 @@ Item {
}
property var allCommands: [
{
name: "attach",
description: Translation.tr("Attach a file. Only works with Gemini."),
execute: (args) => {
Ai.attachFile(args.join(" ").trim());
}
},
{
name: "model",
description: Translation.tr("Choose model"),
@@ -421,13 +428,13 @@ Inline w/ backslash and round brackets \\(e^{i\\pi} + 1 = 0\\)
Rectangle { // Input area
id: inputWrapper
property real columnSpacing: 5
property real spacing: 5
Layout.fillWidth: true
radius: Appearance.rounding.small
color: Appearance.colors.colLayer1
implicitWidth: messageInputField.implicitWidth
implicitHeight: Math.max(inputFieldRowLayout.implicitHeight + inputFieldRowLayout.anchors.topMargin
+ commandButtonsRow.implicitHeight + commandButtonsRow.anchors.bottomMargin + columnSpacing, 45)
implicitHeight: Math.max(inputFieldRowLayout.implicitHeight + inputFieldRowLayout.anchors.topMargin
+ commandButtonsRow.implicitHeight + commandButtonsRow.anchors.bottomMargin + spacing, 45)
+ (attachedFileIndicator.implicitHeight + spacing + attachedFileIndicator.anchors.topMargin)
clip: true
border.color: Appearance.colors.colOutlineVariant
border.width: 1
@@ -436,12 +443,26 @@ Inline w/ backslash and round brackets \\(e^{i\\pi} + 1 = 0\\)
animation: Appearance.animation.elementMove.numberAnimation.createObject(this)
}
AttachedFileIndicator {
id: attachedFileIndicator
anchors {
top: parent.top
left: parent.left
right: parent.right
margins: visible ? 5 : 0
}
filePath: Ai.pendingFilePath
onRemove: Ai.attachFile("")
}
RowLayout { // Input field and send button
id: inputFieldRowLayout
anchors.top: parent.top
anchors.left: parent.left
anchors.right: parent.right
anchors.topMargin: 5
anchors {
top: attachedFileIndicator.bottom
left: parent.left
right: parent.right
topMargin: 5
}
spacing: 0
StyledTextArea { // The actual TextArea
.config/quickshell/ii/modules/sidebarLeft/aiChat/AiMessage.qml
+9
View File
@@ -233,6 +233,15 @@ Rectangle {
}
}
Loader {
Layout.fillWidth: true
active: root.messageData?.localFilePath && root.messageData?.localFilePath.length > 0
sourceComponent: AttachedFileIndicator {
filePath: root.messageData?.localFilePath
canRemove: false
}
}
ColumnLayout { // Message content
id: messageContentColumnLayout
.config/quickshell/ii/modules/sidebarLeft/aiChat/AnnotationSourceButton.qml
+1 -1
View File
@@ -1,7 +1,7 @@
import qs.modules.common
import qs.modules.common.widgets
import qs.services
import qs.modules.common.functions
import qs.services
import QtQuick
import QtQuick.Layouts
import Quickshell.Hyprland
.config/quickshell/ii/modules/sidebarLeft/aiChat/AttachedFileIndicator.qml
+152
View File
@@ -0,0 +1,152 @@
pragma ComponentBehavior: Bound
import QtQuick
import QtQuick.Layouts
import Qt5Compat.GraphicalEffects
import Quickshell.Io
import qs
import qs.modules.common
import qs.modules.common.widgets
import qs.services
Rectangle {
id: root
signal remove()
property bool canRemove: true
property string filePath: ""
property string mimeType: ""
property real maxHeight: 200
property real imageWidth: -1
property real imageHeight: -1
property real scale: Math.min(root.maxHeight / imageHeight, root.width / imageWidth)
onFilePathChanged: refresh()
visible: filePath !== ""
function refresh() {
root.mimeType = "";
root.imageWidth = -1;
root.imageHeight = -1;
fileTypeProc.exec(["file", "-b", "--mime-type", filePath]);
}
Process {
id: fileTypeProc
command: ["file", "-b", "--mime-type", filePath]
stdout: StdioCollector {
onStreamFinished: {
root.mimeType = this.text;
if (root.mimeType.startsWith("image/"))
imageSizeProc.exec(["identify", "-format", "%wx%h", filePath]);
}
}
}
Process {
id: imageSizeProc
command: ["identify", "-format", "%wx%h", filePath]
stdout: StdioCollector {
onStreamFinished: {
const dimensions = this.text.split("x");
root.imageWidth = parseInt(dimensions[0]);
root.imageHeight = parseInt(dimensions[1]);
}
}
}
// Styles/widgets
property real horizontalPadding: 10
property real verticalPadding: 10
radius: Appearance.rounding.small - anchors.margins
color: Appearance.colors.colLayer2
implicitHeight: visible ? (contentItem.implicitHeight + verticalPadding * 2) : 0
ColumnLayout {
id: contentItem
anchors {
fill: parent
leftMargin: root.horizontalPadding
rightMargin: root.horizontalPadding
topMargin: root.verticalPadding
bottomMargin: root.verticalPadding
}
RowLayout {
MaterialSymbol {
Layout.alignment: Qt.AlignTop
text: {
if (root.mimeType.startsWith("image/"))
return "image";
if (root.mimeType.startsWith("audio/"))
return "music_note";
if (root.mimeType.startsWith("video/"))
return "movie";
if (root.mimeType === "application/pdf")
return "picture_as_pdf";
if (root.mimeType.startsWith("text/"))
return "description";
return "file_present";
}
iconSize: Appearance.font.pixelSize.hugeass
}
StyledText {
Layout.fillWidth: true
Layout.topMargin: 4
text: root.filePath
font.pixelSize: Appearance.font.pixelSize.smaller
font.family: Appearance.font.family.monospace
wrapMode: Text.Wrap
}
RippleButton {
visible: root.canRemove
Layout.alignment: Qt.AlignTop
buttonRadius: Appearance.rounding.full
colBackground: Appearance.colors.colLayer2
implicitHeight: 28
implicitWidth: 28
contentItem: MaterialSymbol {
anchors.centerIn: parent
text: "close"
horizontalAlignment: Text.AlignHCenter
iconSize: Appearance.font.pixelSize.larger
color: Appearance.colors.colOnSurfaceVariant
}
onClicked: root.remove()
}
}
Loader {
id: imagePreviewLoader
visible: (root.imageWidth != -1) && (root.imageHeight != -1)
Layout.alignment: Qt.AlignHCenter
sourceComponent: Item {
implicitHeight: root.imageHeight * root.scale
implicitWidth: imagePreview.implicitWidth
Image {
id: imagePreview
anchors.fill: parent
source: Qt.resolvedUrl(root.filePath)
fillMode: Image.PreserveAspectFit
antialiasing: true
asynchronous: true
width: root.imageWidth * root.scale
height: root.imageHeight * root.scale
sourceSize.width: root.imageWidth * root.scale
sourceSize.height: root.imageHeight * root.scale
layer.enabled: true
layer.effect: OpacityMask {
maskSource: Rectangle {
width: imagePreview.width
height: imagePreview.height
radius: Appearance.rounding.normal
}
}
}
}
}
}
}
.config/quickshell/ii/services/Ai.qml
+41 -7
View File
@@ -368,6 +368,9 @@ Singleton {
}
}
property string requestScriptFilePath: "/tmp/quickshell/ai/request.sh"
property string pendingFilePath: ""
Component.onCompleted: {
setModel(currentModelId, false, false); // Do necessary setup for model
}
@@ -617,9 +620,13 @@ Singleton {
root.tokenCount.total = -1;
}
FileView {
id: requesterScriptFile
}
Process {
id: requester
property list<string> baseCommand: ["bash", "-c"]
property list<string> baseCommand: ["bash"]
property AiMessageData message
property ApiStrategy currentStrategy
@@ -645,7 +652,7 @@ Singleton {
const endpoint = root.currentApiStrategy.buildEndpoint(model);
const messageArray = root.messageIDs.map(id => root.messageByID[id]);
const filteredMessageArray = messageArray.filter(message => message.role !== Ai.interfaceRole);
const data = root.currentApiStrategy.buildRequestData(model, filteredMessageArray, root.systemPrompt, root.temperature, root.tools[model.api_format][root.currentTool]);
const data = root.currentApiStrategy.buildRequestData(model, filteredMessageArray, root.systemPrompt, root.temperature, root.tools[model.api_format][root.currentTool], root.pendingFilePath);
// console.log("[Ai] Request data: ", JSON.stringify(data, null, 2));
let requestHeaders = {
@@ -677,14 +684,31 @@ Singleton {
/* Get authorization header from strategy */
const authHeader = requester.currentStrategy.buildAuthorizationHeader(root.apiKeyEnvVarName);
/* Script shebang */
const scriptShebang = "#!/usr/bin/env bash\n";
/* Create extra setup when there's an attached file */
let scriptFileSetupContent = ""
if (root.pendingFilePath && root.pendingFilePath.length > 0) {
requester.message.localFilePath = root.pendingFilePath;
scriptFileSetupContent = requester.currentStrategy.buildScriptFileSetup(root.pendingFilePath);
root.pendingFilePath = ""
}
/* Create command string */
const requestCommandString = `curl --no-buffer "${endpoint}"`
let scriptRequestContent = ""
scriptRequestContent += `curl --no-buffer "${endpoint}"`
+ ` ${headerString}`
+ (authHeader ? ` ${authHeader}` : "")
+ ` -d '${CF.StringUtils.shellSingleQuoteEscape(JSON.stringify(data))}'`
+ ` --data '${CF.StringUtils.shellSingleQuoteEscape(JSON.stringify(data))}'`
+ "\n"
/* Send the request */
requester.command = baseCommand.concat([requestCommandString]);
const scriptContent = requester.currentStrategy.finalizeScriptContent(scriptShebang + scriptFileSetupContent + scriptRequestContent)
const shellScriptPath = CF.FileUtils.trimFileProtocol(root.requestScriptFilePath)
requesterScriptFile.path = Qt.resolvedUrl(shellScriptPath)
requesterScriptFile.setText(scriptContent)
requester.command = baseCommand.concat([shellScriptPath]);
requester.running = true
}
@@ -698,7 +722,7 @@ Singleton {
try {
const result = requester.currentStrategy.parseResponseLine(data, requester.message);
// console.log("[Ai] Parsed response result: ", JSON.stringify(result, null, 2));
if (result.functionCall) {
requester.message.functionCall = result.functionCall;
root.handleFunctionCall(result.functionCall.name, result.functionCall.args, requester.message);
@@ -742,6 +766,10 @@ Singleton {
requester.makeRequest();
}
function attachFile(filePath: string) {
root.pendingFilePath = CF.FileUtils.trimFileProtocol(filePath);
}
function createFunctionOutputMessage(name, output, includeOutputInChat = true) {
return aiMessageComponent.createObject(root, {
"role": "user",
@@ -841,6 +869,9 @@ Singleton {
return ({
"role": message.role,
"rawContent": message.rawContent,
"fileMimeType": message.fileMimeType,
"fileUri": message.fileUri,
"localFilePath": message.localFilePath,
"model": message.model,
"thinking": false,
"done": true,
@@ -858,7 +889,7 @@ Singleton {
id: chatSaveFile
property string chatName: "chat"
path: `${Directories.aiChats}/${chatName}.json`
blockLoading: true
blockLoading: true // Prevent race conditions
}
/**
@@ -894,6 +925,9 @@ Singleton {
"role": message.role,
"rawContent": message.rawContent,
"content": message.rawContent,
"fileMimeType": message.fileMimeType,
"fileUri": message.fileUri,
"localFilePath": message.localFilePath,
"model": message.model,
"thinking": message.thinking,
"done": message.done,
.config/quickshell/ii/services/ai/AiMessageData.qml
+3
View File
@@ -7,6 +7,9 @@ QtObject {
property string role
property string content
property string rawContent
property string fileMimeType
property string fileUri
property string localFilePath
property string model
property bool thinking: true
property bool done: false
.config/quickshell/ii/services/ai/ApiStrategy.qml
+3 -1
View File
@@ -2,9 +2,11 @@ import QtQuick
QtObject {
function buildEndpoint(model: AiModel): string { throw new Error("Not implemented") }
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) { throw new Error("Not implemented") }
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) { throw new Error("Not implemented") }
function buildAuthorizationHeader(apiKeyEnvVarName: string): string { throw new Error("Not implemented") }
function parseResponseLine(line: string, message: AiMessageData) { throw new Error("Not implemented") }
function onRequestFinished(message: AiMessageData): var { return {} } // Default: no special handling
function reset() { } // Reset any internal state if needed
function buildScriptFileSetup(filePath) { return "" } // Default: no setup
function finalizeScriptContent(scriptContent: string): string { return scriptContent } // Optionally modify/finalize script
}
.config/quickshell/ii/services/ai/GeminiApiStrategy.qml
+113 -27
View File
@@ -1,6 +1,12 @@
import QtQuick
import qs.modules.common.functions as CF
ApiStrategy {
readonly property string apiKeyEnvVarName: "API_KEY"
readonly property string fileUriVarName: "file_uri"
readonly property string fileMimeTypeVarName: "MIME_TYPE"
readonly property string fileUriSubstitutionString: "{{ fileUriVarName }}"
readonly property string fileMimeTypeSubstitutionString: "{{ fileMimeTypeVarName }}"
property string buffer: ""
function buildEndpoint(model: AiModel): string {
@@ -9,39 +15,57 @@ ApiStrategy {
return result;
}
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) {
let baseData = {
"contents": messages.map(message => {
const geminiApiRoleName = (message.role === "assistant") ? "model" : message.role;
const usingSearch = tools[0]?.google_search !== undefined
if (!usingSearch && message.functionCall != undefined && message.functionName.length > 0) {
return {
"role": geminiApiRoleName,
"parts": [{
functionCall: {
"name": message.functionName,
}
}]
}
}
if (!usingSearch && message.functionResponse != undefined && message.functionName.length > 0) {
return {
"role": geminiApiRoleName,
"parts": [{
functionResponse: {
"name": message.functionName,
"response": { "content": message.functionResponse }
}
}]
}
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) {
let contents = messages.map(message => {
// console.log("[AI] Building request data for message:", JSON.stringify(message, null, 2));
const geminiApiRoleName = (message.role === "assistant") ? "model" : message.role;
const usingSearch = tools[0]?.google_search !== undefined
if (!usingSearch && message.functionCall != undefined && message.functionName.length > 0) {
return {
"role": geminiApiRoleName,
"parts": [{
functionCall: {
"name": message.functionName,
}
}]
}
}
if (!usingSearch && message.functionResponse != undefined && message.functionName.length > 0) {
return {
"role": geminiApiRoleName,
"parts": [{
text: message.rawContent,
functionResponse: {
"name": message.functionName,
"response": { "content": message.functionResponse }
}
}]
}
}),
}
return {
"role": geminiApiRoleName,
"parts": [
{ text: message.rawContent },
...(message.fileUri && message.fileUri.length > 0 ? [{
"file_data": {
"mime_type": message.fileMimeType,
"file_uri": message.fileUri
}
}] : [])
]
}
})
if (filePath && filePath.length > 0) {
const trimmedFilePath = CF.FileUtils.trimFileProtocol(filePath);
// Add file_data part to the last message's parts array
contents[contents.length - 1].parts.unshift({
file_data: {
mime_type: fileMimeTypeSubstitutionString,
file_uri: fileUriSubstitutionString
}
});
}
let baseData = {
"contents": contents,
"tools": tools,
"system_instruction": {
"parts": [{ text: systemPrompt }]
@@ -50,6 +74,7 @@ ApiStrategy {
"temperature": temperature,
},
};
// print("Gemini API call payload:", JSON.stringify(baseData, null, 2));
return model.extraParams ? Object.assign({}, baseData, model.extraParams) : baseData;
}
@@ -78,8 +103,18 @@ ApiStrategy {
try {
if (buffer.length === 0) return {};
const dataJson = JSON.parse(buffer);
// Uploaded file
if (dataJson.uploadedFile) {
message.fileUri = dataJson.uploadedFile.uri;
message.fileMimeType = dataJson.uploadedFile.mimeType;
return ({})
}
// No candidates?
if (!dataJson.candidates) return {};
// Finished?
if (dataJson.candidates[0]?.finishReason) {
finished = true;
}
@@ -152,4 +187,55 @@ ApiStrategy {
function reset() {
buffer = "";
}
function buildScriptFileSetup(filePath) {
const trimmedFilePath = CF.FileUtils.trimFileProtocol(filePath);
let content = ""
// print("file path:", filePath)
// print("trimmed file path:", trimmedFilePath)
// print("escaped file path:", CF.StringUtils.shellSingleQuoteEscape(trimmedFilePath))
content += `IMAGE_PATH='${CF.StringUtils.shellSingleQuoteEscape(trimmedFilePath)}'\n`;
content += `${fileMimeTypeVarName}=$(file -b --mime-type "$IMAGE_PATH")\n`;
content += 'NUM_BYTES=$(wc -c < "${IMAGE_PATH}")\n';
content += 'tmp_header_file="/tmp/quickshell/ai/upload-header.tmp"\n';
content += 'tmp_file_info_file="/tmp/quickshell/ai/file-info.json.tmp"\n';
// Initial resumable request defining metadata.
// The upload url is in the response headers dump them to a file.
content += 'curl "https://generativelanguage.googleapis.com/upload/v1beta/files"'
+ ` -H "x-goog-api-key: \$${apiKeyEnvVarName}"`
+ ' -D $tmp_header_file'
+ ' -H "X-Goog-Upload-Protocol: resumable"'
+ ' -H "X-Goog-Upload-Command: start"'
+ ' -H "X-Goog-Upload-Header-Content-Length: ${NUM_BYTES}"'
+ ` -H "X-Goog-Upload-Header-Content-Type: \${${fileMimeTypeVarName}}"`
+ ' -H "Content-Type: application/json"'
+ ` -d "{'file': {'display_name': 'Image'}}" 2> /dev/null`
+ '\n';
// Get file upload header
content += 'upload_url=$(grep -i "x-goog-upload-url: " "${tmp_header_file}" | cut -d" " -f2 | tr -d "\r")\n';
content += 'rm "${tmp_header_file}"\n';
// Upload the actual file
content += 'curl "${upload_url}"'
+ ` -H "x-goog-api-key: \$${apiKeyEnvVarName}"`
+ ' -H "Content-Length: ${NUM_BYTES}"'
+ ' -H "X-Goog-Upload-Offset: 0"'
+ ' -H "X-Goog-Upload-Command: upload, finalize"'
+ ' --data-binary "@${IMAGE_PATH}" 2> /dev/null > "${tmp_file_info_file}"'
+ '\n';
content += `${fileUriVarName}=$(jq -r ".file.uri" "$tmp_file_info_file")\n`
content += `printf "{\\"uploadedFile\\": {\\"uri\\": \\"$${fileUriVarName}\\", \\"mimeType\\": \\"$${fileMimeTypeVarName}\\"}}\\n,\\n"\n`
return content
}
function finalizeScriptContent(scriptContent: string): string {
return scriptContent.replace(fileMimeTypeSubstitutionString, `'"\$${fileMimeTypeVarName}"'`)
.replace(fileUriSubstitutionString, `'"\$${fileUriVarName}"'`);
}
}
.config/quickshell/ii/services/ai/MistralApiStrategy.qml
+1 -1
View File
@@ -8,7 +8,7 @@ ApiStrategy {
return model.endpoint;
}
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) {
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) {
let baseData = {
"model": model.model,
"messages": [
.config/quickshell/ii/services/ai/OpenAiApiStrategy.qml
+1 -1
View File
@@ -8,7 +8,7 @@ ApiStrategy {
return model.endpoint;
}
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>) {
function buildRequestData(model: AiModel, messages, systemPrompt: string, temperature: real, tools: list<var>, filePath: string) {
let baseData = {
"model": model.model,
"messages": [