add screen translation

2026-04-03 19:31:24 +02:00
parent ec3b92a938
commit 64b52f6a90
19 changed files with 1159 additions and 20 deletions
@@ -0,0 +1,69 @@
+pragma ComponentBehavior: Bound
+import QtQuick
+import qs.modules.common.functions
+import qs.modules.common.utils
+import qs.services
+import ".."
+
+NestableObject {
+    id: root
+
+    enum State {
+        Done, Preparing, Processing
+    }
+
+    signal finished()
+    property var outputData
+    property var state: GCloudTranslate.State.Done
+
+    property list<string> pendingStrings
+    property bool setupReady: false
+    readonly property bool preparationReady: GoogleCloud.tokenReady && setupReady
+
+    function translateStrings(strings: list<string>) {
+        GoogleCloud.load();
+        root.setupReady = false;
+        root.pendingStrings = strings;
+        root.state = GCloudTranslate.State.Preparing;
+        root.setupReady = true;
+    }
+
+    onPreparationReadyChanged: {
+        if (!preparationReady) return;
+        root.state = GCloudTranslate.State.Processing;
+
+        const targetLang = Translation.languageCode;
+        const payload = {
+            "targetLanguageCode": targetLang,
+            "contents": root.pendingStrings,
+            "mimeType": "text/plain"
+        };
+
+        // print("PENDING STRINGS:", root.pendingStrings)
+
+        var seq = [];
+        seq.push([ //
+            "bash", "-c", //
+            `curl -sL -X POST \
+-H "Authorization: Bearer ${GoogleCloud.token}" \
+-H "x-goog-user-project: ${GoogleCloud.projectId}" \
+-H "Content-Type: application/json" \
+-d '${StringUtils.shellSingleQuoteEscape(JSON.stringify(payload))}' \
+"https://translation.googleapis.com/v3/projects/${GoogleCloud.projectId}:translateText"`
+        ]);
+
+        seq.push(((out) => {
+            // print(out)
+            root.outputData = JSON.parse(out);
+            root.pendingStrings = [];
+            root.finished();
+            root.state = GCloudTranslate.State.Done;
+        }));
+
+        multiproc.runSequence(seq);
+    }
+
+    MultiTurnProcess {
+        id: multiproc
+    }
+}
@@ -0,0 +1,93 @@
+pragma ComponentBehavior: Bound
+import QtQuick
+import Quickshell
+import qs.modules.common.functions
+import qs.modules.common.utils
+import qs.services
+import qs.modules.common
+import ".."
+
+NestableObject {
+    id: root
+
+    enum State {
+        Done, Uploading, Processing, Error
+    }
+
+    signal finished()
+    signal error()
+    property var outputData
+    property var state: GCloudVision.State.Done
+
+    readonly property string imageBase64FilePath: `${Directories.screenshotTemp}/vision_base64.txt`
+    readonly property string payloadFilePath: `${Directories.screenshotTemp}/vision_payload.json`
+    property string uploadEndpoint: "https://uguu.se/upload"
+
+    property bool tokenReady: GoogleCloud.tokenReady
+    property bool onlineImageReady: false
+    readonly property bool preparationReady: tokenReady && onlineImageReady
+
+    function annotateImage(imageUri: string) {
+        root.state = GCloudVision.State.Uploading;
+        root.onlineImageReady = false
+        GoogleCloud.load();
+
+        var seq = []; // command sequence
+
+        const niceFilePath = StringUtils.shellSingleQuoteEscape(FileUtils.trimFileProtocol(imageUri))
+        seq = [ //
+            ["bash", "-c", `mkdir -p '${Directories.screenshotTemp}'; base64 '${niceFilePath}' -w 0 > '${imageBase64FilePath}'`], //
+            (out) => { //
+                root.onlineImageReady = true; //
+            }
+        ]
+
+        // Execute the base64 conversion & load the token
+        prepMultiproc.runSequence(seq);
+    }
+
+    onPreparationReadyChanged: {
+        if (!preparationReady) return;
+        if (GoogleCloud.tokenError || GoogleCloud.keyError) {
+            root.state = GCloudVision.State.Error;
+            root.error();
+            return;
+        }
+        root.state = GCloudVision.State.Processing;
+        var seq = []; // command sequence
+
+        // Construct the JSON payload using jq to read from the base64 file
+        seq.push([
+            "bash", "-c",
+            `jq -n --rawfile content '${imageBase64FilePath}' \
+'{"requests": [{"image": {"content": $content}, "features": [{"type": "DOCUMENT_TEXT_DETECTION"}]}]}' \
+> '${payloadFilePath}'`
+        ]);
+
+        seq.push([
+            "bash", "-c",
+            `curl -s -X POST \
+-H "Authorization: Bearer ${GoogleCloud.token}" \
+-H "x-goog-user-project: ${GoogleCloud.projectId}" \
+-H "Content-Type: application/json" \
+https://vision.googleapis.com/v1/images:annotate \
+-d @'${payloadFilePath}'`
+        ]);
+
+        seq.push((out) => {
+            root.outputData = JSON.parse(out);
+            root.finished();
+            root.state = GCloudVision.State.Done;
+        });
+
+        lookMultiproc.runSequence(seq);
+    }
+
+    MultiTurnProcess {
+        id: prepMultiproc
+    }
+
+    MultiTurnProcess {
+        id: lookMultiproc
+    }
+}
@@ -0,0 +1,90 @@
+pragma ComponentBehavior: Bound
+import QtQuick
+import ".."
+
+NestableObject {
+    id: root
+
+    property real confidenceThreshold: 0.5 // TODO tune this
+
+    property var rawData
+    property var rawBlocks
+    property var rawParagraphs
+    property var coherentParagraphs
+
+    function initializeWithData(apiOutputData: var): void {
+        // Null check
+        if (!apiOutputData) {
+            print("[GCloudVisionResult] Data is null/undefined")
+            return;
+        }
+
+        // Raw data
+        root.rawData = apiOutputData
+
+        // Raw blocks
+        var pages = apiOutputData.responses[0].fullTextAnnotation.pages
+        var blocks = [];
+        for (var i = 0; i < pages.length; i++) {
+            // print("this page", JSON.stringify(pages[i]))
+            var blocksThisPage = pages[i].blocks;
+            for (var j = 0; j < blocksThisPage.length; j++) {
+                const block = blocksThisPage[j];
+                // print("new block with confidence", block.confidence, ":", JSON.stringify(block, null, 2))
+                if (block.confidence > root.confidenceThreshold) {
+                    blocks.push(block);
+                }
+            }
+        }
+        
+        root.rawBlocks = blocks
+        // print("RAW BLOCKS:", blocks)
+
+        // Raw paragraphs
+        var paragraphs = []
+        for (var i = 0; i < blocks.length; i++) {
+            var blockParagraphs = blocks[i].paragraphs;
+            for (var j = 0; j < blockParagraphs.length; j++) {
+                const para = blockParagraphs[j];
+                // print("new paragraph", JSON.stringify(para))
+                paragraphs.push(para);
+            }
+        }
+        root.rawParagraphs = [...paragraphs];
+
+        // print("RAW PARAGRAPHS", paragraphs)
+
+        // Coherent paragraphs
+        // (raw data can be as granular as symbols)
+        // We're interested in paragraph level of granularity as it's good for translations
+        for (var i = 0; i < paragraphs.length; i++) {
+            const paragraph = paragraphs[i];
+            const words = paragraph.words;
+            var strList = []
+            for (var j = 0; j < words.length; j++) {
+                const symbols = words[j].symbols;
+                for (var k = 0; k < symbols.length; k++) {
+                    const sym = symbols[k];
+                    strList.push(sym.text);
+                    // print("CHAR:", JSON.stringify(sym, null, 2));
+                    // Breaks
+                    // Reference: https://docs.cloud.google.com/vision/docs/reference/rpc/google.cloud.vision.v1#breaktype
+                    if (sym.property?.detectedBreak.type == "SPACE" || sym.property?.detectedBreak.type == "UNKNOWN") {
+                        strList.push(" ");
+                    } else if (sym.property?.detectedBreak.type == "SURE_SPACE") {
+                        strList.push("　");
+                    } else if (sym.property?.detectedBreak.type == "EOL_SURE_SPACE" || sym.property?.detectedBreak.type == "LINE_BREAK") {
+                        strList.push("\n");
+                    } else if (sym.property?.detectedBreak.type == "HYPHEN") {
+                        strList.push("-\n");
+                    }
+                }
+            }
+            // print("STR LIST:", strList)
+            paragraphs[i].text = strList.join("").trim();
+            // print("PARA TEXT:", paragraphs[i].text)
+        }
+        root.coherentParagraphs = paragraphs
+        // print("COHERENT PARAGRAPHS", JSON.stringify(paragraphs))
+    }
+}