From 64b52f6a908138e91b1a800aed68993db69a7ff0 Mon Sep 17 00:00:00 2001 From: end-4 <97237370+end-4@users.noreply.github.com> Date: Fri, 3 Apr 2026 19:31:24 +0200 Subject: [PATCH] add screen translation --- dots/.config/quickshell/ii/GlobalStates.qml | 1 + .../common/models/gCloud/GCloudTranslate.qml | 69 ++++ .../common/models/gCloud/GCloudVision.qml | 93 ++++++ .../models/gCloud/GCloudVisionResult.qml | 90 +++++ .../common/widgets/MaskMultiEffect.qml | 9 + .../widgets/SqueezedAnnotationStyledText.qml | 74 +++++ .../ii/screenTranslator/ScreenTextOverlay.qml | 313 ++++++++++++++++++ .../ii/screenTranslator/ScreenTranslator.qml | 41 +++ .../ScreenTranslatorPanel.qml | 225 +++++++++++++ .../panelFamilies/IllogicalImpulseFamily.qml | 2 + .../ii/panelFamilies/WaffleFamily.qml | 2 + .../ii/scripts/images/text-color-venv.sh | 6 + .../ii/scripts/images/text_color.py | 60 ++++ .../quickshell/ii/services/GoogleCloud.qml | 93 ++++++ .../quickshell/ii/services/KeyringStorage.qml | 3 + .../ii/services/gCloud/token-from-key-venv.sh | 6 + .../ii/services/gCloud/token_from_key.py | 30 ++ sdata/uv/requirements.in | 2 + sdata/uv/requirements.txt | 60 ++-- 19 files changed, 1159 insertions(+), 20 deletions(-) create mode 100644 dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudTranslate.qml create mode 100644 dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVision.qml create mode 100644 dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVisionResult.qml create mode 100644 dots/.config/quickshell/ii/modules/common/widgets/MaskMultiEffect.qml create mode 100644 dots/.config/quickshell/ii/modules/common/widgets/SqueezedAnnotationStyledText.qml create mode 100644 dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTextOverlay.qml create mode 100644 dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslator.qml create mode 100644 dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslatorPanel.qml create mode 100755 dots/.config/quickshell/ii/scripts/images/text-color-venv.sh create mode 100755 dots/.config/quickshell/ii/scripts/images/text_color.py create mode 100644 dots/.config/quickshell/ii/services/GoogleCloud.qml create mode 100755 dots/.config/quickshell/ii/services/gCloud/token-from-key-venv.sh create mode 100755 dots/.config/quickshell/ii/services/gCloud/token_from_key.py diff --git a/dots/.config/quickshell/ii/GlobalStates.qml b/dots/.config/quickshell/ii/GlobalStates.qml index ba680220b..bfb531e6e 100644 --- a/dots/.config/quickshell/ii/GlobalStates.qml +++ b/dots/.config/quickshell/ii/GlobalStates.qml @@ -24,6 +24,7 @@ Singleton { property bool screenLocked: false property bool screenLockContainsCharacters: false property bool screenUnlockFailed: false + property bool screenTranslatorOpen: false property bool sessionOpen: false property bool superDown: false property bool superReleaseMightTrigger: true diff --git a/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudTranslate.qml b/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudTranslate.qml new file mode 100644 index 000000000..d117e8d3a --- /dev/null +++ b/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudTranslate.qml @@ -0,0 +1,69 @@ +pragma ComponentBehavior: Bound +import QtQuick +import qs.modules.common.functions +import qs.modules.common.utils +import qs.services +import ".." + +NestableObject { + id: root + + enum State { + Done, Preparing, Processing + } + + signal finished() + property var outputData + property var state: GCloudTranslate.State.Done + + property list pendingStrings + property bool setupReady: false + readonly property bool preparationReady: GoogleCloud.tokenReady && setupReady + + function translateStrings(strings: list) { + GoogleCloud.load(); + root.setupReady = false; + root.pendingStrings = strings; + root.state = GCloudTranslate.State.Preparing; + root.setupReady = true; + } + + onPreparationReadyChanged: { + if (!preparationReady) return; + root.state = GCloudTranslate.State.Processing; + + const targetLang = Translation.languageCode; + const payload = { + "targetLanguageCode": targetLang, + "contents": root.pendingStrings, + "mimeType": "text/plain" + }; + + // print("PENDING STRINGS:", root.pendingStrings) + + var seq = []; + seq.push([ // + "bash", "-c", // + `curl -sL -X POST \ +-H "Authorization: Bearer ${GoogleCloud.token}" \ +-H "x-goog-user-project: ${GoogleCloud.projectId}" \ +-H "Content-Type: application/json" \ +-d '${StringUtils.shellSingleQuoteEscape(JSON.stringify(payload))}' \ +"https://translation.googleapis.com/v3/projects/${GoogleCloud.projectId}:translateText"` + ]); + + seq.push(((out) => { + // print(out) + root.outputData = JSON.parse(out); + root.pendingStrings = []; + root.finished(); + root.state = GCloudTranslate.State.Done; + })); + + multiproc.runSequence(seq); + } + + MultiTurnProcess { + id: multiproc + } +} diff --git a/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVision.qml b/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVision.qml new file mode 100644 index 000000000..3e02fd561 --- /dev/null +++ b/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVision.qml @@ -0,0 +1,93 @@ +pragma ComponentBehavior: Bound +import QtQuick +import Quickshell +import qs.modules.common.functions +import qs.modules.common.utils +import qs.services +import qs.modules.common +import ".." + +NestableObject { + id: root + + enum State { + Done, Uploading, Processing, Error + } + + signal finished() + signal error() + property var outputData + property var state: GCloudVision.State.Done + + readonly property string imageBase64FilePath: `${Directories.screenshotTemp}/vision_base64.txt` + readonly property string payloadFilePath: `${Directories.screenshotTemp}/vision_payload.json` + property string uploadEndpoint: "https://uguu.se/upload" + + property bool tokenReady: GoogleCloud.tokenReady + property bool onlineImageReady: false + readonly property bool preparationReady: tokenReady && onlineImageReady + + function annotateImage(imageUri: string) { + root.state = GCloudVision.State.Uploading; + root.onlineImageReady = false + GoogleCloud.load(); + + var seq = []; // command sequence + + const niceFilePath = StringUtils.shellSingleQuoteEscape(FileUtils.trimFileProtocol(imageUri)) + seq = [ // + ["bash", "-c", `mkdir -p '${Directories.screenshotTemp}'; base64 '${niceFilePath}' -w 0 > '${imageBase64FilePath}'`], // + (out) => { // + root.onlineImageReady = true; // + } + ] + + // Execute the base64 conversion & load the token + prepMultiproc.runSequence(seq); + } + + onPreparationReadyChanged: { + if (!preparationReady) return; + if (GoogleCloud.tokenError || GoogleCloud.keyError) { + root.state = GCloudVision.State.Error; + root.error(); + return; + } + root.state = GCloudVision.State.Processing; + var seq = []; // command sequence + + // Construct the JSON payload using jq to read from the base64 file + seq.push([ + "bash", "-c", + `jq -n --rawfile content '${imageBase64FilePath}' \ +'{"requests": [{"image": {"content": $content}, "features": [{"type": "DOCUMENT_TEXT_DETECTION"}]}]}' \ +> '${payloadFilePath}'` + ]); + + seq.push([ + "bash", "-c", + `curl -s -X POST \ +-H "Authorization: Bearer ${GoogleCloud.token}" \ +-H "x-goog-user-project: ${GoogleCloud.projectId}" \ +-H "Content-Type: application/json" \ +https://vision.googleapis.com/v1/images:annotate \ +-d @'${payloadFilePath}'` + ]); + + seq.push((out) => { + root.outputData = JSON.parse(out); + root.finished(); + root.state = GCloudVision.State.Done; + }); + + lookMultiproc.runSequence(seq); + } + + MultiTurnProcess { + id: prepMultiproc + } + + MultiTurnProcess { + id: lookMultiproc + } +} diff --git a/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVisionResult.qml b/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVisionResult.qml new file mode 100644 index 000000000..d0794b37a --- /dev/null +++ b/dots/.config/quickshell/ii/modules/common/models/gCloud/GCloudVisionResult.qml @@ -0,0 +1,90 @@ +pragma ComponentBehavior: Bound +import QtQuick +import ".." + +NestableObject { + id: root + + property real confidenceThreshold: 0.5 // TODO tune this + + property var rawData + property var rawBlocks + property var rawParagraphs + property var coherentParagraphs + + function initializeWithData(apiOutputData: var): void { + // Null check + if (!apiOutputData) { + print("[GCloudVisionResult] Data is null/undefined") + return; + } + + // Raw data + root.rawData = apiOutputData + + // Raw blocks + var pages = apiOutputData.responses[0].fullTextAnnotation.pages + var blocks = []; + for (var i = 0; i < pages.length; i++) { + // print("this page", JSON.stringify(pages[i])) + var blocksThisPage = pages[i].blocks; + for (var j = 0; j < blocksThisPage.length; j++) { + const block = blocksThisPage[j]; + // print("new block with confidence", block.confidence, ":", JSON.stringify(block, null, 2)) + if (block.confidence > root.confidenceThreshold) { + blocks.push(block); + } + } + } + + root.rawBlocks = blocks + // print("RAW BLOCKS:", blocks) + + // Raw paragraphs + var paragraphs = [] + for (var i = 0; i < blocks.length; i++) { + var blockParagraphs = blocks[i].paragraphs; + for (var j = 0; j < blockParagraphs.length; j++) { + const para = blockParagraphs[j]; + // print("new paragraph", JSON.stringify(para)) + paragraphs.push(para); + } + } + root.rawParagraphs = [...paragraphs]; + + // print("RAW PARAGRAPHS", paragraphs) + + // Coherent paragraphs + // (raw data can be as granular as symbols) + // We're interested in paragraph level of granularity as it's good for translations + for (var i = 0; i < paragraphs.length; i++) { + const paragraph = paragraphs[i]; + const words = paragraph.words; + var strList = [] + for (var j = 0; j < words.length; j++) { + const symbols = words[j].symbols; + for (var k = 0; k < symbols.length; k++) { + const sym = symbols[k]; + strList.push(sym.text); + // print("CHAR:", JSON.stringify(sym, null, 2)); + // Breaks + // Reference: https://docs.cloud.google.com/vision/docs/reference/rpc/google.cloud.vision.v1#breaktype + if (sym.property?.detectedBreak.type == "SPACE" || sym.property?.detectedBreak.type == "UNKNOWN") { + strList.push(" "); + } else if (sym.property?.detectedBreak.type == "SURE_SPACE") { + strList.push("　"); + } else if (sym.property?.detectedBreak.type == "EOL_SURE_SPACE" || sym.property?.detectedBreak.type == "LINE_BREAK") { + strList.push("\n"); + } else if (sym.property?.detectedBreak.type == "HYPHEN") { + strList.push("-\n"); + } + } + } + // print("STR LIST:", strList) + paragraphs[i].text = strList.join("").trim(); + // print("PARA TEXT:", paragraphs[i].text) + } + root.coherentParagraphs = paragraphs + // print("COHERENT PARAGRAPHS", JSON.stringify(paragraphs)) + } +} diff --git a/dots/.config/quickshell/ii/modules/common/widgets/MaskMultiEffect.qml b/dots/.config/quickshell/ii/modules/common/widgets/MaskMultiEffect.qml new file mode 100644 index 000000000..18e2177d4 --- /dev/null +++ b/dots/.config/quickshell/ii/modules/common/widgets/MaskMultiEffect.qml @@ -0,0 +1,9 @@ +import QtQuick +import QtQuick.Effects + +// Note: You still have to set sizes yourself +MultiEffect { + maskEnabled: true + maskThresholdMin: 0.5 + maskSpreadAtMin: 1 +} diff --git a/dots/.config/quickshell/ii/modules/common/widgets/SqueezedAnnotationStyledText.qml b/dots/.config/quickshell/ii/modules/common/widgets/SqueezedAnnotationStyledText.qml new file mode 100644 index 000000000..25990cd42 --- /dev/null +++ b/dots/.config/quickshell/ii/modules/common/widgets/SqueezedAnnotationStyledText.qml @@ -0,0 +1,74 @@ +pragma ComponentBehavior: Bound +import QtQuick +import qs.modules.common + +// Annotation similar to how Google Lens does it. +Item { + id: root + + property real scaleFactor: 1.0 + property alias font: textWidget.font + property alias color: textWidget.color + property string text: "" + + property bool rotate90: false + property real maxFontPixelSize: 100 + visible: false + + Component.onCompleted: updateText() + onTextChanged: updateText() + + property bool searching: false + property real searchPixelSize: Appearance.font.pixelSize.small + property real renderPixelSize: Appearance.font.pixelSize.small + font.pixelSize: searching ? searchPixelSize : (renderPixelSize * scaleFactor) + + function updateText() { + // Do we rotate? + + root.rotate90 = false; + const textAspectRatio = textMetrics.width / textMetrics.height + const areaAspectRatio = root.width / root.height + if ((textAspectRatio > 1 && areaAspectRatio < 1) || (textAspectRatio < 1 && areaAspectRatio > 1)) { + root.rotate90 = true; + } + const targetWidth = (root.rotate90 ? root.height : root.width) / root.scaleFactor; + const targetHeight = (root.rotate90 ? root.width : root.height) / root.scaleFactor; + + // Binary search to find the correct font size + var lower = 0 + var upper = maxFontPixelSize + root.searching = true; + while (upper - lower > 0.00001) { + var mid = (lower + upper) / 2; + // print("bin searching", mid, "target", targetWidth, targetHeight, "actual", textWidget.contentWidth, textWidget.contentHeight); + root.searchPixelSize = mid + if (textWidget.contentHeight > targetHeight) { + upper = mid + } else { + lower = mid + } + } + root.renderPixelSize = lower + root.searching = false; + root.visible = true + } + + TextMetrics { + id: textMetrics + text: root.text + font: root.font + } + + StyledText { + id: textWidget + + anchors.centerIn: parent + width: root.rotate90 ? parent.height : parent.width + text: root.text + rotation: root.rotate90 ? 90 : 0 + + renderType: Text.QtRendering + wrapMode: Text.Wrap + } +} diff --git a/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTextOverlay.qml b/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTextOverlay.qml new file mode 100644 index 000000000..cd1c6a202 --- /dev/null +++ b/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTextOverlay.qml @@ -0,0 +1,313 @@ +pragma ComponentBehavior: Bound + +import QtQuick +import QtQuick.Effects +import Qt5Compat.GraphicalEffects +import Quickshell + +import qs +import qs.modules.common +import qs.modules.common.functions +import qs.modules.common.models.gCloud +import qs.modules.common.utils +import qs.modules.common.widgets +import qs.services + +Item { + id: root + + property double scaleFactor: 1 + property color overlayColor: "#BB000000" + property color textColor: "white" + required property string screenshotPath + + readonly property string wikiLink: "https://ii.clsty.link/en/ii-qs/02usage/#setting-it-up" // TODO: write a page for this + readonly property string textColorDetectionScriptPath: Quickshell.shellPath("scripts/images/text-color-venv.sh") + + property bool loading: true + property var visionParagraphs: [] + property list translationKeys: [] + property var translation: ({}) + + function translate(s: string): string { + return translation[s] ?? s; + } + + property bool error: false + function showError() { + error = true; + } + + Component.onCompleted: { + if (GoogleCloud.tokenReady && GoogleCloud.tokenError) { + root.showError(); + } + cloudVision.annotateImage(screenshotPath); + } + + Connections { + target: GoogleCloud + function onTokenChanged() { + if (GoogleCloud.tokenReady && !GoogleCloud.tokenError) { + root.error = false; + cloudVision.annotateImage(root.screenshotPath); + } + } + } + + Rectangle { + id: loadingOverlay + anchors.fill: parent + opacity: root.loading ? 1 : 0 + Behavior on opacity { + animation: Appearance.animation.elementMoveSmall.numberAnimation.createObject(this) + } + color: root.overlayColor + + Column { + visible: !root.error + anchors.centerIn: parent + spacing: 10 * root.scaleFactor + MaterialLoadingIndicator { + anchors.horizontalCenter: parent.horizontalCenter + implicitSize: 100 * root.scaleFactor + scale: 1 + ((1 - loadingOverlay.opacity) * 0.5) * root.scaleFactor + } + StyledText { + anchors.horizontalCenter: parent.horizontalCenter + text: { + if (cloudVision.state == GCloudVision.State.Uploading) + return Translation.tr("Uploading image"); + else if (cloudVision.state == GCloudVision.State.Processing) + return Translation.tr("Reading image"); + else if (cloudVision.state == GCloudVision.State.Error) + return Translation.tr("Error"); + else if (cloudTrans.state == GCloudTranslate.State.Preparing) + return Translation.tr("Getting ready to translate"); + else if (cloudTrans.state == GCloudTranslate.State.Processing) + return Translation.tr("Translating"); + else + return " "; + } + font.pixelSize: Appearance.font.pixelSize.small * root.scaleFactor + animateChange: true + color: root.textColor + } + } + + Column { + visible: root.error + anchors.centerIn: parent + spacing: 10 * root.scaleFactor + + MaterialShapeWrappedMaterialSymbol { + anchors.horizontalCenter: parent.horizontalCenter + text: "exclamation" + iconSize: 80 * root.scaleFactor + padding: 6 * root.scaleFactor + color: Appearance.colors.colError + colSymbol: Appearance.colors.colOnError + shape: MaterialShape.Shape.Sunny + } + StyledText { + anchors.horizontalCenter: parent.horizontalCenter + horizontalAlignment: Text.AlignHCenter + textFormat: Text.MarkdownText + text: `**${Translation.tr("Screen Translator")}**\n\n${Translation.tr("Set your Google Cloud service account key")}\n\n__[${Translation.tr("See how on the wiki")}](${root.wikiLink})__` + font.pixelSize: Appearance.font.pixelSize.small * root.scaleFactor + color: root.textColor + MouseArea { + anchors.fill: parent + cursorShape: Qt.PointingHandCursor + onClicked: { + Qt.openUrlExternally(root.wikiLink) + GlobalStates.screenTranslatorOpen = false + } + } + } + } + } + + GCloudVisionResult { + id: gcr + } + + GCloudVision { + id: cloudVision + onError: { + root.showError(); + } + onFinished: { + gcr.initializeWithData(outputData); + root.visionParagraphs = gcr.coherentParagraphs; + // print(gcr.coherentParagraphs) + root.translationKeys = gcr.coherentParagraphs.map(p => p.text); + // print("TRANSLATION KEYS:", JSON.stringify(root.translationKeys)); + cloudTrans.translateStrings(root.translationKeys); + } + } + + GCloudTranslate { + id: cloudTrans + onFinished: { + var values = outputData.translations.map(translation => translation.translatedText); + const keys = root.translationKeys; + root.translation = ({}); + for (var i = 0; i < keys.length; i++) { + Object.assign(root.translation, { + [keys[i]]: values[i] + }); + } + // print("TRANSLATION:", JSON.stringify(root.translation)); + root.loading = false; + } + } + + property real windowWidth: QsWindow.window.screen.width + property real windowHeight: QsWindow.window.screen.height + + StyledImage { + id: screenshotImage + z: 1 + asynchronous: false + width: root.windowWidth + height: root.windowHeight + sourceSize: Qt.size(root.windowWidth, root.windowHeight) + source: Qt.resolvedUrl(root.screenshotPath) + visible: false + } + + Item { + id: blurMaskItem + z: 2 + width: root.windowWidth + height: root.windowHeight + layer.enabled: true + visible: false + Repeater { + model: root.loading ? [] : root.visionParagraphs + delegate: VisionBoundingBoxRect { + readonly property string text: modelData.text + readonly property string translatedText: root.translate(text) + visible: translatedText != text + scaleFactor: 1 + } + } + } + + // I no longer need these but they were a fucking pain in the ass to figure out so they're staying + // GaussianBlur { + // id: blurredImage + // z: 3 + // width: root.windowWidth + // height: root.windowHeight + // transformOrigin: Item.TopLeft + // scale: root.scaleFactor + // source: screenshotImage + // radius: 10 + // samples: radius * 2 + 1 + // visible: false + // } + // MultiEffect { + // id: blurredImage + // z: 3 + // source: screenshotImage + // width: root.windowWidth + // height: root.windowHeight + // transformOrigin: Item.TopLeft + // scale: root.scaleFactor + + // blurEnabled: true + // blur: 1 + // blurMax: 64 + // visible: false + // } + + MaskMultiEffect { + z: 4 + implicitWidth: parent.width + implicitHeight: parent.height + width: parent.width + height: parent.height + + // Mask + source: screenshotImage + maskSource: blurMaskItem + + // Blur + blurEnabled: true + blur: 1 + blurMax: 50 + blurMultiplier: root.scaleFactor + autoPaddingEnabled: false + } + + Item { + id: textItems + z: 999 + Repeater { + model: root.loading ? [] : root.visionParagraphs + // An entry looks like this: + delegate: TextItem {} + } + } + + component VisionBoundingBoxRect: Rectangle { + required property var modelData + property real scaleFactor: root.scaleFactor + property list boundingVertices: modelData.boundingBox.vertices + property real unscaledX: boundingVertices[0].x + property real unscaledY: boundingVertices[0].y + property real unscaledWidth: boundingVertices[1].x - boundingVertices[0].x + property real unscaledHeight: boundingVertices[3].y - boundingVertices[0].y + x: unscaledX * scaleFactor + y: unscaledY * scaleFactor + width: unscaledWidth * scaleFactor + height: unscaledHeight * scaleFactor + radius: 4 + } + + component TextItem: VisionBoundingBoxRect { + id: ti + // {"boundingPoly": {"vertices": [{"x": 536,"y": 236},{"x": 583,"y": 236},{"x": 583,"y": 262},{"x": 536,"y": 262}]},"description": "宮坂"} + readonly property string text: modelData.text + readonly property string translatedText: root.translate(text) + visible: translatedText != text + + color: ColorUtils.transparentize(Appearance.colors.colSecondaryContainer, 0.4) + Behavior on color { + animation: Appearance.animation.elementMoveFast.colorAnimation.createObject(this) + } + + Loader { + active: ti.visible + sourceComponent: MultiTurnProcess { + Component.onCompleted: { + runSequence([ // + [ // + "bash", "-c", // + `magick ${StringUtils.shellSingleQuoteEscape(root.screenshotPath)} +repage -crop ${StringUtils.shellSingleQuoteEscape(ti.unscaledWidth)}x${StringUtils.shellSingleQuoteEscape(ti.unscaledHeight)}+${StringUtils.shellSingleQuoteEscape(ti.unscaledX)}+${StringUtils.shellSingleQuoteEscape(ti.unscaledY)} png:- | ${root.textColorDetectionScriptPath}` + ], + (out => { + var colorData = JSON.parse(out); + ti.color = ColorUtils.transparentize(colorData.background, 0.4); + tiText.color = colorData.text; + }) + ]); + } + } + } + + SqueezedAnnotationStyledText { + id: tiText + width: parent.width + height: parent.height + text: ti.translatedText + scaleFactor: root.scaleFactor + + Behavior on color { + animation: Appearance.animation.elementMoveFast.colorAnimation.createObject(this) + } + } + } +} diff --git a/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslator.qml b/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslator.qml new file mode 100644 index 000000000..39db82909 --- /dev/null +++ b/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslator.qml @@ -0,0 +1,41 @@ +pragma ComponentBehavior: Bound +import qs +import QtQuick +import Quickshell +import Quickshell.Io +import Quickshell.Hyprland + +Scope { + id: root + + function dismiss() { + GlobalStates.screenTranslatorOpen = false + } + + Loader { + id: translatorLoader + active: GlobalStates.screenTranslatorOpen + + sourceComponent: ScreenTranslatorPanel { + onDismiss: root.dismiss() + } + } + + function translate() { + GlobalStates.screenTranslatorOpen = true + } + + IpcHandler { + target: "screenTranslator" + + function translate() { + root.translate() + } + } + + GlobalShortcut { + name: "screenTranslate" + description: "Translates screen content" + onPressed: root.translate() + } +} diff --git a/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslatorPanel.qml b/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslatorPanel.qml new file mode 100644 index 000000000..ac4070ede --- /dev/null +++ b/dots/.config/quickshell/ii/modules/ii/screenTranslator/ScreenTranslatorPanel.qml @@ -0,0 +1,225 @@ +pragma ComponentBehavior: Bound + +import QtQuick +import QtQuick.Layouts +import Quickshell +import Quickshell.Wayland + +import qs.modules.common +import qs.modules.common.utils +import qs.modules.common.widgets +import qs.services + +PanelWindow { + id: root + + // Interface + signal dismiss + + // Window props + visible: false + // color: Appearance.colors.colLayer0 + color: "black" + WlrLayershell.namespace: "quickshell:regionSelector" + WlrLayershell.layer: WlrLayer.Overlay + WlrLayershell.keyboardFocus: WlrKeyboardFocus.OnDemand + exclusionMode: ExclusionMode.Ignore + anchors { + left: true + right: true + top: true + bottom: true + } + + // Config + readonly property string screenshotDir: Directories.screenshotTemp + readonly property string screenshotPath: `${root.screenshotDir}/image-${screen.name}` + + // Preparation + property bool screenshotReady: false + + function performTranslation() { + screenshotReady = true; + } + + TempScreenshotProcess { + id: screenshotProc + running: true + screen: root.screen + screenshotDir: root.screenshotDir + screenshotPath: root.screenshotPath + onExited: (_, __) => { + root.visible = true; + root.performTranslation(); + } + } + + // Actual content + property real scale: 1.0 + property real contentX: 0 + property real contentY: 0 + + MouseArea { + anchors.fill: parent + clip: true + + property real lastX: 0 + property real lastY: 0 + + cursorShape: Qt.SizeAllCursor + + onPressed: mouse => { + lastX = mouse.x; + lastY = mouse.y; + } + + onPositionChanged: mouse => { + if (pressed) { + root.contentX += (mouse.x - lastX); + root.contentY += (mouse.y - lastY); + lastX = mouse.x; + lastY = mouse.y; + } + } + + onWheel: event => { + const zoomFactor = event.angleDelta.y > 0 ? 1.1 : 0.9; + const oldScale = root.scale; + const newScale = Math.min(Math.max(0.1, oldScale * zoomFactor), 5); + + if (newScale !== oldScale) { + // Determine mouse position relative to the content's unscaled origin + const localX = (event.x - root.contentX) / oldScale; + const localY = (event.y - root.contentY) / oldScale; + + // Apply zoom + root.scale = newScale; + + // Shift offsets to keep the same local point under the cursor + root.contentX = event.x - (localX * newScale); + root.contentY = event.y - (localY * newScale); + } + } + + ScreencopyView { // Freeze screen + id: screencopy + width: parent.width + height: parent.height + + x: root.contentX + y: root.contentY + scale: root.scale + transformOrigin: Item.TopLeft + + live: false + captureSource: root.screen + } + + Loader { + width: parent.width * root.scale + height: parent.height * root.scale + + x: root.contentX + y: root.contentY + + active: root.screenshotReady + sourceComponent: ScreenTextOverlay { + screenshotPath: root.screenshotPath + scaleFactor: root.scale + } + } + } + + Row { + anchors { + horizontalCenter: parent.horizontalCenter + bottom: parent.bottom + bottomMargin: -height + } + Behavior on anchors.bottomMargin { + animation: Appearance.animation.elementMove.numberAnimation.createObject(this) + } + Component.onCompleted: { + anchors.bottomMargin = 8; + } + + spacing: 6 + + Toolbar { + id: toolbar + focus: root.visible + Keys.onPressed: event => { // Esc to close + if (event.key === Qt.Key_Escape) { + root.dismiss(); + } + } + spacing: 0 + + IconToolbarButton { + id: sleepButton + onClicked: { + toggled = !toggled + if (toggled) keyInput.forceActiveFocus() + } + text: "key" + + StyledToolTip { + z: 9999 + text: Translation.tr("Key input") + } + } + + Revealer { + reveal: sleepButton.toggled + Layout.fillHeight: true + + RowLayout { + anchors.left: parent.left + spacing: 6 + Item {} // extra padding + ToolbarTextField { + id: keyInput + implicitWidth: 400 + placeholderText: Translation.tr("Paste service account key JSON here") + inputMethodHints: Qt.ImhSensitiveData + onAccepted: submit() + + function submit() { + const success = GoogleCloud.setKeyJson(text); + if (!success) { + invalidJsonAnimation.restart(); + } else { + text = ""; + sleepButton.toggled = false; + } + } + + ErrorShakeAnimation { + id: invalidJsonAnimation + target: keyInput + } + } + IconToolbarButton { + id: submitButton + onClicked: keyInput.submit() + text: "check" + toggled: keyInput.text.length > 0 + + StyledToolTip { + z: 9999 + text: Translation.tr("Confirm") + } + } + } + } + } + + ToolbarPairedFab { + iconText: "close" + onClicked: root.dismiss() + StyledToolTip { + text: Translation.tr("Close") + } + } + } +} diff --git a/dots/.config/quickshell/ii/panelFamilies/IllogicalImpulseFamily.qml b/dots/.config/quickshell/ii/panelFamilies/IllogicalImpulseFamily.qml index f4ffda651..61a901fa2 100644 --- a/dots/.config/quickshell/ii/panelFamilies/IllogicalImpulseFamily.qml +++ b/dots/.config/quickshell/ii/panelFamilies/IllogicalImpulseFamily.qml @@ -15,6 +15,7 @@ import qs.modules.ii.overview import qs.modules.ii.polkit import qs.modules.ii.regionSelector import qs.modules.ii.screenCorners +import qs.modules.ii.screenTranslator import qs.modules.ii.sessionScreen import qs.modules.ii.sidebarLeft import qs.modules.ii.sidebarRight @@ -37,6 +38,7 @@ Scope { PanelLoader { component: Polkit {} } PanelLoader { component: RegionSelector {} } PanelLoader { component: ScreenCorners {} } + PanelLoader { component: ScreenTranslator {} } PanelLoader { component: SessionScreen {} } PanelLoader { component: SidebarLeft {} } PanelLoader { component: SidebarRight {} } diff --git a/dots/.config/quickshell/ii/panelFamilies/WaffleFamily.qml b/dots/.config/quickshell/ii/panelFamilies/WaffleFamily.qml index 67d35de55..1953703b4 100644 --- a/dots/.config/quickshell/ii/panelFamilies/WaffleFamily.qml +++ b/dots/.config/quickshell/ii/panelFamilies/WaffleFamily.qml @@ -20,6 +20,7 @@ import qs.modules.waffle.taskView import qs.modules.ii.cheatsheet import qs.modules.ii.onScreenKeyboard import qs.modules.ii.overlay +import qs.modules.ii.screenTranslator import qs.modules.ii.wallpaperSelector Scope { @@ -40,5 +41,6 @@ Scope { PanelLoader { component: Cheatsheet {} } PanelLoader { component: OnScreenKeyboard {} } PanelLoader { component: Overlay {} } + PanelLoader { component: ScreenTranslator {} } PanelLoader { component: WallpaperSelector {} } } diff --git a/dots/.config/quickshell/ii/scripts/images/text-color-venv.sh b/dots/.config/quickshell/ii/scripts/images/text-color-venv.sh new file mode 100755 index 000000000..f673bd687 --- /dev/null +++ b/dots/.config/quickshell/ii/scripts/images/text-color-venv.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source $(eval echo $ILLOGICAL_IMPULSE_VIRTUAL_ENV)/bin/activate +"$SCRIPT_DIR/text_color.py" "$@" +deactivate diff --git a/dots/.config/quickshell/ii/scripts/images/text_color.py b/dots/.config/quickshell/ii/scripts/images/text_color.py new file mode 100755 index 000000000..3fcff8d10 --- /dev/null +++ b/dots/.config/quickshell/ii/scripts/images/text_color.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# Disclaimer: This script was ai-generated and went through minimal revision. + +import cv2 +import numpy as np +import json +import sys + +def to_hex(color): + return "#{:02x}{:02x}{:02x}".format(int(color[0]), int(color[1]), int(color[2])) + +def get_color_from_stdin(): + # Read raw bytes from stdin + input_data = sys.stdin.buffer.read() + if not input_data: + return {"error": "No data received via stdin"} + + # Convert bytes to numpy array and decode to image + nparr = np.frombuffer(input_data, np.uint8) + img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + + if img is None: + return {"error": "Could not decode image data"} + + img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + h, w, _ = img_rgb.shape + + # 1. Sample corner pixels (The background anchors) + corners = np.array([ + img_rgb[0, 0], + img_rgb[0, w-1], + img_rgb[h-1, 0], + img_rgb[h-1, w-1] + ]) + + # 2. Determine single dominant background + # Using median handles noise/gradients better than a simple average + bg_color = np.median(corners, axis=0).astype(int) + + # 3. Find the Text Color + pixels = img_rgb.reshape(-1, 3).astype(int) + distances = np.linalg.norm(pixels - bg_color, axis=1) + + # Take the 95th percentile of pixels furthest from background + threshold = np.percentile(distances, 95) + text_pixels = pixels[distances >= threshold] + + if len(text_pixels) == 0: + text_color = [255, 255, 255] # Fallback + else: + text_color = np.median(text_pixels, axis=0).astype(int) + + return { + "background": to_hex(bg_color), + "text": to_hex(text_color) + } + +if __name__ == "__main__": + result = get_color_from_stdin() + print(json.dumps(result)) \ No newline at end of file diff --git a/dots/.config/quickshell/ii/services/GoogleCloud.qml b/dots/.config/quickshell/ii/services/GoogleCloud.qml new file mode 100644 index 000000000..e90ae5dfd --- /dev/null +++ b/dots/.config/quickshell/ii/services/GoogleCloud.qml @@ -0,0 +1,93 @@ +pragma Singleton +pragma ComponentBehavior: Bound +import QtQuick +import Quickshell +import qs.modules.common.utils + +Singleton { + id: root + + property var keyContent: ({}) + property string keyProjectId: keyContent.project_id + property bool keyError: false + property bool keyReady: false + property string token: "" + property bool tokenError: false + property bool tokenReady: false + readonly property string projectId: keyProjectId + + readonly property bool loaded: keyReady && tokenReady + + readonly property string tokenForKeyScriptPath: Quickshell.shellPath("services/gCloud/token-from-key-venv.sh") + + function load() { + // Dummy for init + } + + function setKeyJson(str: string): bool { + try { + var keyData = JSON.parse(str) + KeyringStorage.setNestedField(["googleCloud", "serviceAccountKey"], keyData); + return true; + } catch(e) { + return false; + } + } + + function getToken() { + if (root.keyError) { + root.tokenError = true; + root.tokenReady = true; + return; + } + tokenProc.runSequence([(() => { // prep token fetcher + tokenProc.environment.SERVICE_KEY_CONTENT = JSON.stringify(root.keyContent); + tokenProc.command = [ // + "bash", "-c" // + , `${tokenForKeyScriptPath} "$SERVICE_KEY_CONTENT"`]; + }), [] // run token fetcher + , (out => { + if (out.startsWith("Error")) { + root.tokenError = true; + } else { + root.tokenError = false; + root.token = out.trim(); + } + root.tokenReady = true; + })]); + } + + function loadKeyIfPossible() { + if (KeyringStorage.loaded) { + root.keyContent = KeyringStorage.keyringData?.googleCloud?.serviceAccountKey; + if (!root.keyContent?.project_id) { + root.keyError = true; + } else { + root.keyError = false; + root.keyProjectId = root.keyContent.project_id; + } + root.keyReady = true; + root.getToken(); + } else { + KeyringStorage.fetchKeyringData(); + } + } + + Component.onCompleted: { + loadKeyIfPossible(); + } + + Connections { + target: KeyringStorage + function onLoadedChanged() { + root.loadKeyIfPossible(); + } + function onDataChanged() { + root.loadKeyIfPossible(); + } + } + + MultiTurnProcess { + id: tokenProc + } +} diff --git a/dots/.config/quickshell/ii/services/KeyringStorage.qml b/dots/.config/quickshell/ii/services/KeyringStorage.qml index ae49496d2..7e716d459 100644 --- a/dots/.config/quickshell/ii/services/KeyringStorage.qml +++ b/dots/.config/quickshell/ii/services/KeyringStorage.qml @@ -15,6 +15,8 @@ import QtQuick; Singleton { id: root + signal dataChanged() + property bool loaded: false property var keyringData: ({}) @@ -82,6 +84,7 @@ Singleton { if (saveData.running) { // console.log("[KeyringStorage] Saving with command: '" + saveData.command.join("' '") + "'"); saveData.write(JSON.stringify(root.keyringData)); + root.dataChanged() stdinEnabled = false // End input stream } } diff --git a/dots/.config/quickshell/ii/services/gCloud/token-from-key-venv.sh b/dots/.config/quickshell/ii/services/gCloud/token-from-key-venv.sh new file mode 100755 index 000000000..fd9e255e8 --- /dev/null +++ b/dots/.config/quickshell/ii/services/gCloud/token-from-key-venv.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source $(eval echo $ILLOGICAL_IMPULSE_VIRTUAL_ENV)/bin/activate +"$SCRIPT_DIR/token_from_key.py" "$@" +deactivate diff --git a/dots/.config/quickshell/ii/services/gCloud/token_from_key.py b/dots/.config/quickshell/ii/services/gCloud/token_from_key.py new file mode 100755 index 000000000..f40a50661 --- /dev/null +++ b/dots/.config/quickshell/ii/services/gCloud/token_from_key.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import sys +import json +import google.auth.transport.requests +import google.oauth2.service_account + +def get_token(json_str): + try: + # Load the string into a dictionary + info = json.loads(json_str) + + # Initialize credentials + creds = google.oauth2.service_account.Credentials.from_service_account_info(info) + scoped_creds = creds.with_scopes(['https://www.googleapis.com/auth/cloud-platform']) + + # Refresh to get the access token + request = google.auth.transport.requests.Request() + scoped_creds.refresh(request) + + print(scoped_creds.token) + except Exception as e: + sys.stderr.write(f"Error: {str(e)}\n") + sys.exit(1) + +if __name__ == "__main__": + if len(sys.argv) < 2: + sys.stderr.write("Usage: python3 get_token.py ''\n") + sys.exit(1) + + get_token(sys.argv[1]) diff --git a/sdata/uv/requirements.in b/sdata/uv/requirements.in index 277a461cb..58023b915 100644 --- a/sdata/uv/requirements.in +++ b/sdata/uv/requirements.in @@ -16,3 +16,5 @@ pygobject tqdm numpy opencv-contrib-python +google-auth +requests diff --git a/sdata/uv/requirements.txt b/sdata/uv/requirements.txt index f58e87d3d..cdfd65503 100644 --- a/sdata/uv/requirements.txt +++ b/sdata/uv/requirements.txt @@ -1,63 +1,83 @@ # This file was autogenerated by uv via the following command: -# uv pip compile sdata/uv/requirements.in -o sdata/uv/requirements.txt +# uv pip compile requirements.in -o requirements.txt build==1.2.2.post1 - # via -r sdata/uv/requirements.in + # via -r requirements.in +certifi==2026.2.25 + # via requests cffi==1.17.1 - # via pywayland + # via + # cryptography + # pywayland +charset-normalizer==3.4.7 + # via requests click==8.2.1 - # via -r sdata/uv/requirements.in + # via -r requirements.in +cryptography==46.0.0 + # via google-auth dbus-python==1.4.0 # via kde-material-you-colors +google-auth==2.49.1 + # via -r requirements.in +idna==3.11 + # via requests kde-material-you-colors==1.10.1 - # via -r sdata/uv/requirements.in + # via -r requirements.in libsass==0.23.0 - # via -r sdata/uv/requirements.in + # via -r requirements.in loguru==0.7.3 - # via -r sdata/uv/requirements.in + # via -r requirements.in material-color-utilities==0.2.1 - # via -r sdata/uv/requirements.in + # via -r requirements.in materialyoucolor==2.0.10 # via - # -r sdata/uv/requirements.in + # -r requirements.in # kde-material-you-colors numpy==2.2.2 # via - # -r sdata/uv/requirements.in + # -r requirements.in # kde-material-you-colors # material-color-utilities # opencv-contrib-python opencv-contrib-python==4.12.0.88 - # via -r sdata/uv/requirements.in + # via -r requirements.in packaging==24.2 # via # build # setuptools-scm pillow==11.1.0 # via - # -r sdata/uv/requirements.in + # -r requirements.in # kde-material-you-colors # material-color-utilities psutil==6.1.1 - # via -r sdata/uv/requirements.in + # via -r requirements.in +pyasn1==0.6.3 + # via pyasn1-modules +pyasn1-modules==0.4.2 + # via google-auth pycairo==1.28.0 # via - # -r sdata/uv/requirements.in + # -r requirements.in # pygobject pycparser==2.22 # via cffi pygobject==3.52.3 - # via -r sdata/uv/requirements.in + # via -r requirements.in pyproject-hooks==1.2.0 # via build pywayland==0.4.18 - # via -r sdata/uv/requirements.in + # via -r requirements.in +requests==2.33.1 + # via -r requirements.in setproctitle==1.3.4 - # via -r sdata/uv/requirements.in + # via -r requirements.in setuptools==80.9.0 # via setuptools-scm setuptools-scm==8.1.0 - # via -r sdata/uv/requirements.in + # via -r requirements.in tqdm==4.67.1 - # via -r sdata/uv/requirements.in + # via -r requirements.in +urllib3==2.6.3 + # via requests wheel==0.45.1 - # via -r sdata/uv/requirements.in + # via -r requirements.in