From 513d612b262458ad456de4c3be4fae413a0a351d Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Wed, 13 Mar 2024 19:03:02 +0530 Subject: [PATCH 1/9] test git push --- src/scripts/extensions/firefox/manifest.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scripts/extensions/firefox/manifest.json b/src/scripts/extensions/firefox/manifest.json index 41ea1a76..dd6d7db4 100644 --- a/src/scripts/extensions/firefox/manifest.json +++ b/src/scripts/extensions/firefox/manifest.json @@ -27,7 +27,8 @@ "tabs", "webRequest", "storage", - "webNavigation" + "webNavigation", + "nativeMessaging" ], "applications": { From c70a987bf0473999c916840bf1acc96f806fb1a3 Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Wed, 13 Mar 2024 19:19:33 +0530 Subject: [PATCH 2/9] changes to communicate with memory bar --- startMemory.bat | 1 + webClipperNext.json | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 startMemory.bat create mode 100644 webClipperNext.json diff --git a/startMemory.bat b/startMemory.bat new file mode 100644 index 00000000..79b57a90 --- /dev/null +++ b/startMemory.bat @@ -0,0 +1 @@ +start onenote.exe /memoryWindow \ No newline at end of file diff --git a/webClipperNext.json b/webClipperNext.json new file mode 100644 index 00000000..1f81909b --- /dev/null +++ b/webClipperNext.json @@ -0,0 +1,9 @@ +{ + "name": "com.microsoft.onenote.stickynotes", + "description": "Native host for OneNote Extension", + "path": "D:\\WebClipper\\startMemory.bat", + "type": "stdio", + "allowed_origins": [ + "chrome-extension://kdnabjiachpmkglfekldaoanlbhnlgbi/" + ] + } \ No newline at end of file From 7ff32f479650bea4ccdaf293818599b43e94f7a2 Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Sat, 16 Mar 2024 02:56:33 +0530 Subject: [PATCH 3/9] changes for memory bar to communicate with extension --- gulpfile.js | 16 +++++- .../extensions/chrome/chromeExtension.ts | 37 +++++++++++++ src/scripts/extensions/chrome/manifest.json | 8 ++- src/scripts/extensions/contentScript.ts | 53 +++++++++++++++++++ src/scripts/extensions/firefox/manifest.json | 3 +- startMemory.bat | 1 - webClipperNext.json => winManifest.json | 4 +- 7 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 src/scripts/extensions/contentScript.ts delete mode 100644 startMemory.bat rename webClipperNext.json => winManifest.json (52%) diff --git a/gulpfile.js b/gulpfile.js index ce11d101..699daff9 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -229,6 +229,13 @@ gulp.task("bundleAppendIsInstalledMarker", function () { return merge(tasks); }); +gulp.task("bundleContentScript", function () { + var extensionRoot = PATHS.BUILDROOT + "scripts/extensions/"; + var files = ["contentScript.js"]; + var tasks = generateBrowserifyTasks(extensionRoot, files); + return merge(tasks); +}); + gulp.task("bundleClipperUI", function () { var extensionRoot = PATHS.BUILDROOT + "scripts/clipperUI/"; var files = ["clipper.js", "pageNav.js", "localeSpecificTasks.js", "unsupportedBrowser.js"]; @@ -299,6 +306,7 @@ gulp.task("bundleTests", function () { gulp.task("bundle", function(callback) { runSequence( "bundleAppendIsInstalledMarker", + "bundleContentScript", "bundleClipperUI", "bundleLogManager", "bundleBookmarklet", @@ -512,6 +520,10 @@ function exportChromeJS() { PATHS.BUNDLEROOT + "appendIsInstalledMarker.js" ]).pipe(concat("appendIsInstalledMarker.js")).pipe(gulp.dest(targetDir)); + var contentScriptTask = gulp.src([ + PATHS.BUNDLEROOT + "contentScript.js" + ]).pipe(concat("contentScript.js")).pipe(gulp.dest(targetDir)); + var chromeExtensionTask = gulp.src([ targetDir + "logManager.js", targetDir + "oneNoteApi.min.js", @@ -537,9 +549,9 @@ function exportChromeJS() { ]).pipe(concat("chromePageNavInject.js")).pipe(gulp.dest(targetDir)); if (commonTask) { - return merge(commonTask, appendIsInstalledMarkerTask, chromeExtensionTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); + return merge(commonTask, appendIsInstalledMarkerTask, contentScriptTask, chromeExtensionTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); } - return merge(chromeExtensionTask, appendIsInstalledMarkerTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); + return merge(chromeExtensionTask, appendIsInstalledMarkerTask, contentScriptTask, chromeDebugLoggingInjectTask, chromeInjectTask, chromePageNavInjectTask); } function exportChromeCSS() { diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index e6c1b544..081412ee 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -2,6 +2,43 @@ import {ClientType} from "../../clientType"; import {WebExtension} from "../webExtensionBase/webExtension"; +function sendMessageToContentScript(tabs, msg) { + if (tabs.length > 0) { + chrome.tabs.sendMessage(tabs[0].id, { bMessage: msg }, function (response) { + console.log(response.cMessage); + }); + } +} + +let port = chrome.runtime.connectNative('com.microsoft.onenote.stickynotes'); + +port.onMessage.addListener((response) => { + console.log('Received from native application: ' + JSON.stringify(response)); + console.log('window.location.href as known to the current background script is: ' + window.location.href); + /** + * This is a background script and cannot access the DOM of the current page. + * To access the DOM of the current page, we need to send a message to the + * content script. + */ + chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { + console.log('Getting window.location.href from content script...'); + sendMessageToContentScript(tabs, "GET_WINDOW_LOCATION_HREF"); + console.log('Getting video id from content script...'); + sendMessageToContentScript(tabs, "GET_VIDEO_ID"); + console.log('Getting stream player from content script...'); + sendMessageToContentScript(tabs, "GET_STREAM_PLAYER"); + }); + + +}); + +port.onDisconnect.addListener(() => { + console.log('Disconnected'); +}); + +// let message = { text: 'Hello from the extension!' }; +// port.postMessage(message); + WebExtension.browser = chrome; let clipperBackground = new WebExtension(ClientType.ChromeExtension, { debugLoggingInjectUrl: "chromeDebugLoggingInject.js", diff --git a/src/scripts/extensions/chrome/manifest.json b/src/scripts/extensions/chrome/manifest.json index 30718585..777e2007 100644 --- a/src/scripts/extensions/chrome/manifest.json +++ b/src/scripts/extensions/chrome/manifest.json @@ -13,6 +13,11 @@ "js": ["appendIsInstalledMarker.js"], "run_at": "document_start", "all_frames": true + }, { + "matches": ["https://www.youtube.com/*", "https://youtube.com/*"], + "js": ["contentScript.js"], + "run_at": "document_end", + "all_frames": true }], "web_accessible_resources": [ @@ -27,7 +32,8 @@ "tabs", "webRequest", "storage", - "webNavigation" + "webNavigation", + "nativeMessaging" ], "content_security_policy": "script-src 'self'; object-src 'self'", diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts new file mode 100644 index 00000000..517acd1b --- /dev/null +++ b/src/scripts/extensions/contentScript.ts @@ -0,0 +1,53 @@ +console.log("Content script loaded"); + +// Listen for a message from the background script +chrome.runtime.onMessage.addListener( + function(request, sender, sendResponse) { + console.log(JSON.stringify(request)); + if (request === undefined) { + sendResponse({ cMessage: "Invalid request received by content script" }); + } + switch (request.bMessage) { + case "GET_WINDOW_LOCATION_HREF": + sendResponse({ cMessage: window.location.href }); + break; + case "GET_VIDEO_ID": + sendResponse({ cMessage: getVideoId() }); + break; + case "GET_STREAM_PLAYER": + sendResponse({ cMessage: (getStreamPlayer() as HTMLElement).outerHTML }); + break; + default: + sendResponse({ cMessage: "Invalid message received by content script" }); + } + }); + +function getStreamPlayer() { + return document.getElementsByClassName("video-stream")[0]; +} + +function parseParams(href) { + const noHash = href.split("#")[0]; + const paramString = noHash.split("?")[1]; + const params = {}; + if (paramString) { + const paramsArray = paramString.split("&"); + for (const kv of paramsArray) { + const tmparr = kv.split("="); + params[tmparr[0]] = tmparr[1]; + } + } + return params; +} + +function getVideoId() { + if (window.location.pathname == "/watch") { + return parseParams(window.location.href)["v"]; + } else if (window.location.pathname.indexOf("/embed/") == 0) { + return window.location.pathname.substring("/embed/".length); + } else { + return null; + } +} + +console.log("Content script finished executing"); diff --git a/src/scripts/extensions/firefox/manifest.json b/src/scripts/extensions/firefox/manifest.json index dd6d7db4..41ea1a76 100644 --- a/src/scripts/extensions/firefox/manifest.json +++ b/src/scripts/extensions/firefox/manifest.json @@ -27,8 +27,7 @@ "tabs", "webRequest", "storage", - "webNavigation", - "nativeMessaging" + "webNavigation" ], "applications": { diff --git a/startMemory.bat b/startMemory.bat deleted file mode 100644 index 79b57a90..00000000 --- a/startMemory.bat +++ /dev/null @@ -1 +0,0 @@ -start onenote.exe /memoryWindow \ No newline at end of file diff --git a/webClipperNext.json b/winManifest.json similarity index 52% rename from webClipperNext.json rename to winManifest.json index 1f81909b..a94b0d6f 100644 --- a/webClipperNext.json +++ b/winManifest.json @@ -1,9 +1,9 @@ { "name": "com.microsoft.onenote.stickynotes", "description": "Native host for OneNote Extension", - "path": "D:\\WebClipper\\startMemory.bat", + "path": "C:\\Program Files\\Microsoft Office\\root\\Office16\\onenote.exe", "type": "stdio", "allowed_origins": [ - "chrome-extension://kdnabjiachpmkglfekldaoanlbhnlgbi/" + "chrome-extension://gopaeppdaekmbdnampgofgnfdcfojdbn/" ] } \ No newline at end of file From c10322a193fed5c1061b4e48df9323859169d73f Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Sat, 16 Mar 2024 04:06:40 +0530 Subject: [PATCH 4/9] added comments --- src/scripts/extensions/chrome/chromeExtension.ts | 8 ++++++-- src/scripts/extensions/contentScript.ts | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index 081412ee..9ff79c8c 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -10,11 +10,14 @@ function sendMessageToContentScript(tabs, msg) { } } +/****** START CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ + let port = chrome.runtime.connectNative('com.microsoft.onenote.stickynotes'); port.onMessage.addListener((response) => { console.log('Received from native application: ' + JSON.stringify(response)); console.log('window.location.href as known to the current background script is: ' + window.location.href); + /****** START CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ /** * This is a background script and cannot access the DOM of the current page. * To access the DOM of the current page, we need to send a message to the @@ -28,8 +31,7 @@ port.onMessage.addListener((response) => { console.log('Getting stream player from content script...'); sendMessageToContentScript(tabs, "GET_STREAM_PLAYER"); }); - - + /****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ }); port.onDisconnect.addListener(() => { @@ -39,6 +41,8 @@ port.onDisconnect.addListener(() => { // let message = { text: 'Hello from the extension!' }; // port.postMessage(message); +/****** END CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ + WebExtension.browser = chrome; let clipperBackground = new WebExtension(ClientType.ChromeExtension, { debugLoggingInjectUrl: "chromeDebugLoggingInject.js", diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts index 517acd1b..8ca2d95b 100644 --- a/src/scripts/extensions/contentScript.ts +++ b/src/scripts/extensions/contentScript.ts @@ -1,5 +1,7 @@ console.log("Content script loaded"); +/****** START CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ + // Listen for a message from the background script chrome.runtime.onMessage.addListener( function(request, sender, sendResponse) { @@ -22,6 +24,8 @@ chrome.runtime.onMessage.addListener( } }); +/****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ + function getStreamPlayer() { return document.getElementsByClassName("video-stream")[0]; } From bb8a3f283c8018d6739a4db872c43222cad68a52 Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Mon, 18 Mar 2024 17:32:25 +0530 Subject: [PATCH 5/9] get deep youtube url --- src/scripts/extensions/chrome/chromeExtension.ts | 4 ++-- src/scripts/extensions/contentScript.ts | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index 9ff79c8c..ac2205cf 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -24,8 +24,8 @@ port.onMessage.addListener((response) => { * content script. */ chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { - console.log('Getting window.location.href from content script...'); - sendMessageToContentScript(tabs, "GET_WINDOW_LOCATION_HREF"); + console.log('Getting youtube url from content script...'); + sendMessageToContentScript(tabs, "GET_YOUTUBE_URL"); console.log('Getting video id from content script...'); sendMessageToContentScript(tabs, "GET_VIDEO_ID"); console.log('Getting stream player from content script...'); diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts index 8ca2d95b..04abaf66 100644 --- a/src/scripts/extensions/contentScript.ts +++ b/src/scripts/extensions/contentScript.ts @@ -10,8 +10,8 @@ chrome.runtime.onMessage.addListener( sendResponse({ cMessage: "Invalid request received by content script" }); } switch (request.bMessage) { - case "GET_WINDOW_LOCATION_HREF": - sendResponse({ cMessage: window.location.href }); + case "GET_YOUTUBE_URL": + sendResponse({ cMessage: getYoutubeURL() }); break; case "GET_VIDEO_ID": sendResponse({ cMessage: getVideoId() }); @@ -54,4 +54,13 @@ function getVideoId() { } } +function getYoutubeURL() { + var player = getStreamPlayer() as HTMLVideoElement; + var time = player.currentTime; + var path = document.location.href; + path = path.split("?")[0]; + path += "?v=" + getVideoId() + "&t=" + Math.floor(time) + "s"; + return path; +} + console.log("Content script finished executing"); From 301f76fa5cef4a468512f97f50fb12a041c98649 Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Mon, 18 Mar 2024 23:16:54 +0530 Subject: [PATCH 6/9] send messages back to memory bar --- .../extensions/chrome/chromeExtension.ts | 18 +++++++++++++----- src/scripts/extensions/contentScript.ts | 7 +++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index ac2205cf..1a9dfb60 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -5,14 +5,23 @@ import {WebExtension} from "../webExtensionBase/webExtension"; function sendMessageToContentScript(tabs, msg) { if (tabs.length > 0) { chrome.tabs.sendMessage(tabs[0].id, { bMessage: msg }, function (response) { - console.log(response.cMessage); + if (msg === "GET_ALL") { + sendMessageToNativeApplication(response); + } else { + console.log(response.cMessage); + } }); } } /****** START CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ -let port = chrome.runtime.connectNative('com.microsoft.onenote.stickynotes'); +let hostName = 'com.microsoft.onenote.stickynotes'; +let port = chrome.runtime.connectNative(hostName); + +function sendMessageToNativeApplication(msg) { + port.postMessage(msg); +} port.onMessage.addListener((response) => { console.log('Received from native application: ' + JSON.stringify(response)); @@ -30,6 +39,8 @@ port.onMessage.addListener((response) => { sendMessageToContentScript(tabs, "GET_VIDEO_ID"); console.log('Getting stream player from content script...'); sendMessageToContentScript(tabs, "GET_STREAM_PLAYER"); + console.log('Getting all details from content script and sending back to native application...'); + sendMessageToContentScript(tabs, "GET_ALL"); }); /****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ }); @@ -38,9 +49,6 @@ port.onDisconnect.addListener(() => { console.log('Disconnected'); }); -// let message = { text: 'Hello from the extension!' }; -// port.postMessage(message); - /****** END CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ WebExtension.browser = chrome; diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts index 04abaf66..845703bb 100644 --- a/src/scripts/extensions/contentScript.ts +++ b/src/scripts/extensions/contentScript.ts @@ -19,6 +19,13 @@ chrome.runtime.onMessage.addListener( case "GET_STREAM_PLAYER": sendResponse({ cMessage: (getStreamPlayer() as HTMLElement).outerHTML }); break; + case "GET_ALL": + sendResponse({ + youtubeURL: getYoutubeURL(), + videoId: getVideoId(), + streamPlayer: (getStreamPlayer() as HTMLElement).outerHTML + }); + break; default: sendResponse({ cMessage: "Invalid message received by content script" }); } From 3f810d6a5a16b7f40344d48d6d5ed8ec44e3da9e Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Tue, 19 Mar 2024 18:21:11 +0530 Subject: [PATCH 7/9] content script changes to allow it to be reused for all urls --- .../extensions/chrome/chromeExtension.ts | 4 ++-- src/scripts/extensions/chrome/manifest.json | 2 +- src/scripts/extensions/contentScript.ts | 24 ++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index 1a9dfb60..5b3bedad 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -5,7 +5,7 @@ import {WebExtension} from "../webExtensionBase/webExtension"; function sendMessageToContentScript(tabs, msg) { if (tabs.length > 0) { chrome.tabs.sendMessage(tabs[0].id, { bMessage: msg }, function (response) { - if (msg === "GET_ALL") { + if (msg === "GET_ALL_YOUTUBE_DETAILS") { sendMessageToNativeApplication(response); } else { console.log(response.cMessage); @@ -40,7 +40,7 @@ port.onMessage.addListener((response) => { console.log('Getting stream player from content script...'); sendMessageToContentScript(tabs, "GET_STREAM_PLAYER"); console.log('Getting all details from content script and sending back to native application...'); - sendMessageToContentScript(tabs, "GET_ALL"); + sendMessageToContentScript(tabs, "GET_ALL_YOUTUBE_DETAILS"); }); /****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ }); diff --git a/src/scripts/extensions/chrome/manifest.json b/src/scripts/extensions/chrome/manifest.json index 777e2007..593b79b5 100644 --- a/src/scripts/extensions/chrome/manifest.json +++ b/src/scripts/extensions/chrome/manifest.json @@ -14,7 +14,7 @@ "run_at": "document_start", "all_frames": true }, { - "matches": ["https://www.youtube.com/*", "https://youtube.com/*"], + "matches": [""], "js": ["contentScript.js"], "run_at": "document_end", "all_frames": true diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts index 845703bb..f6bfcd35 100644 --- a/src/scripts/extensions/contentScript.ts +++ b/src/scripts/extensions/contentScript.ts @@ -4,26 +4,30 @@ console.log("Content script loaded"); // Listen for a message from the background script chrome.runtime.onMessage.addListener( - function(request, sender, sendResponse) { + function (request, sender, sendResponse) { + if (window.self !== window.top) { + // We're in an iframe, ignore the message + return; + } console.log(JSON.stringify(request)); if (request === undefined) { sendResponse({ cMessage: "Invalid request received by content script" }); } switch (request.bMessage) { case "GET_YOUTUBE_URL": - sendResponse({ cMessage: getYoutubeURL() }); + sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL() }); break; case "GET_VIDEO_ID": - sendResponse({ cMessage: getVideoId() }); + sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getVideoId() }); break; case "GET_STREAM_PLAYER": - sendResponse({ cMessage: (getStreamPlayer() as HTMLElement).outerHTML }); + sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML }); break; - case "GET_ALL": + case "GET_ALL_YOUTUBE_DETAILS": sendResponse({ - youtubeURL: getYoutubeURL(), - videoId: getVideoId(), - streamPlayer: (getStreamPlayer() as HTMLElement).outerHTML + youtubeURL: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL(), + videoId: !isYoutube() ? "NOT_YOUTUBE" : getVideoId(), + streamPlayer: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML }); break; default: @@ -33,6 +37,10 @@ chrome.runtime.onMessage.addListener( /****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ +function isYoutube() { + return document.location.href.indexOf("youtube.com") !== -1; +} + function getStreamPlayer() { return document.getElementsByClassName("video-stream")[0]; } From 577d99e9fbb835ec72bc60c71043259584ac4851 Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Tue, 19 Mar 2024 23:13:00 +0530 Subject: [PATCH 8/9] send document title to native application --- src/scripts/extensions/contentScript.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts index f6bfcd35..47da8b51 100644 --- a/src/scripts/extensions/contentScript.ts +++ b/src/scripts/extensions/contentScript.ts @@ -26,6 +26,7 @@ chrome.runtime.onMessage.addListener( case "GET_ALL_YOUTUBE_DETAILS": sendResponse({ youtubeURL: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL(), + documentTitle: getDocumentTitle(), videoId: !isYoutube() ? "NOT_YOUTUBE" : getVideoId(), streamPlayer: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML }); @@ -41,6 +42,10 @@ function isYoutube() { return document.location.href.indexOf("youtube.com") !== -1; } +function getDocumentTitle() { + return document.title; +} + function getStreamPlayer() { return document.getElementsByClassName("video-stream")[0]; } From 5217a8d33b3e2b53d69a99d8e675b1720ae597c0 Mon Sep 17 00:00:00 2001 From: Arpan Kundu Date: Thu, 21 Mar 2024 00:30:06 +0530 Subject: [PATCH 9/9] changes to add youtube screenshot and transcript --- .../extensions/chrome/chromeExtension.ts | 28 ++++- src/scripts/extensions/contentScript.ts | 110 +++++++++++++++++- 2 files changed, 127 insertions(+), 11 deletions(-) diff --git a/src/scripts/extensions/chrome/chromeExtension.ts b/src/scripts/extensions/chrome/chromeExtension.ts index 5b3bedad..07ce7d81 100644 --- a/src/scripts/extensions/chrome/chromeExtension.ts +++ b/src/scripts/extensions/chrome/chromeExtension.ts @@ -5,15 +5,31 @@ import {WebExtension} from "../webExtensionBase/webExtension"; function sendMessageToContentScript(tabs, msg) { if (tabs.length > 0) { chrome.tabs.sendMessage(tabs[0].id, { bMessage: msg }, function (response) { - if (msg === "GET_ALL_YOUTUBE_DETAILS") { - sendMessageToNativeApplication(response); - } else { + if (msg !== "GET_ALL_YOUTUBE_DETAILS") { console.log(response.cMessage); } }); } } +/****** START CODE TO COMMUNICATE FROM CONTENT TO BACKGROUND SCRIPT ******/ + +chrome.runtime.onMessage.addListener( + function (request, sender, sendResponse) { + // Received all details from content script, now send back to native application + sendMessageToNativeApplication({ + youtubeURL: request.youtubeURL, + documentTitle: request.documentTitle, + videoId: request.videoId, + streamPlayer: request.streamPlayer, + pictureOfVideo: request.pictureOfVideo, + transcript: request.transcript + }); + } +); + +/****** END CODE TO COMMUNICATE FROM CONTENT TO BACKGROUND SCRIPT ******/ + /****** START CODE TO COMMUNICATE BETWEEN EXTENSION AND NATIVE APPLICATION ******/ let hostName = 'com.microsoft.onenote.stickynotes'; @@ -26,23 +42,25 @@ function sendMessageToNativeApplication(msg) { port.onMessage.addListener((response) => { console.log('Received from native application: ' + JSON.stringify(response)); console.log('window.location.href as known to the current background script is: ' + window.location.href); - /****** START CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ + /****** START CODE TO COMMUNICATE FROM BACKGROUND TO CONTENT SCRIPT ******/ /** * This is a background script and cannot access the DOM of the current page. * To access the DOM of the current page, we need to send a message to the * content script. */ chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { + /* console.log('Getting youtube url from content script...'); sendMessageToContentScript(tabs, "GET_YOUTUBE_URL"); console.log('Getting video id from content script...'); sendMessageToContentScript(tabs, "GET_VIDEO_ID"); console.log('Getting stream player from content script...'); sendMessageToContentScript(tabs, "GET_STREAM_PLAYER"); + */ console.log('Getting all details from content script and sending back to native application...'); sendMessageToContentScript(tabs, "GET_ALL_YOUTUBE_DETAILS"); }); - /****** END CODE TO COMMUNICATE BETWEEN BACKGROUND AND CONTENT SCRIPTS ******/ + /****** END CODE TO COMMUNICATE FROM BACKGROUND TO CONTENT SCRIPT ******/ }); port.onDisconnect.addListener(() => { diff --git a/src/scripts/extensions/contentScript.ts b/src/scripts/extensions/contentScript.ts index 47da8b51..c46cbdf8 100644 --- a/src/scripts/extensions/contentScript.ts +++ b/src/scripts/extensions/contentScript.ts @@ -14,6 +14,7 @@ chrome.runtime.onMessage.addListener( sendResponse({ cMessage: "Invalid request received by content script" }); } switch (request.bMessage) { + /* case "GET_YOUTUBE_URL": sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL() }); break; @@ -23,13 +24,18 @@ chrome.runtime.onMessage.addListener( case "GET_STREAM_PLAYER": sendResponse({ cMessage: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML }); break; + */ case "GET_ALL_YOUTUBE_DETAILS": - sendResponse({ - youtubeURL: !isYoutube() ? "NOT_YOUTUBE" : getYoutubeURL(), - documentTitle: getDocumentTitle(), - videoId: !isYoutube() ? "NOT_YOUTUBE" : getVideoId(), - streamPlayer: !isYoutube() ? "NOT_YOUTUBE" : (getStreamPlayer() as HTMLElement).outerHTML - }); + if (!isYoutube()) { + chrome.runtime.sendMessage({ + youtubeURL: "NOT_YOUTUBE", + documentTitle: getDocumentTitle(), + pictureOfVideo: "NOT_YOUTUBE", + transcript: "NOT_YOUTUBE" + }); + } else { + getTranscriptAndReturnAllYoutubeDetails(); + } break; default: sendResponse({ cMessage: "Invalid message received by content script" }); @@ -50,6 +56,98 @@ function getStreamPlayer() { return document.getElementsByClassName("video-stream")[0]; } +function getPictureOfVideo() { + const player = getStreamPlayer() as HTMLVideoElement; + var canvas = document.createElement("canvas"); + canvas.width = player.videoWidth; + canvas.height = player.videoHeight; + canvas.getContext("2d").drawImage(player, 0, 0, canvas.width, canvas.height); + + /****** START IMAGE COMPRESSION LOGIC ******/ + + // Create a new canvas element + const newCanvas = document.createElement('canvas'); + + // Set the dimensions of the new canvas to the desired size + newCanvas.width = canvas.width / 10; // Reduce the width + newCanvas.height = canvas.height / 10; // Reduce the height + + // Get the 2D context of the new canvas + const ctx = newCanvas.getContext('2d'); + + // Draw the image from the original canvas onto the new canvas + ctx.drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, newCanvas.width, newCanvas.height); + + /****** END IMAGE COMPRESSION LOGIC ******/ + + // Get the data URL of the image from the new canvas + const img = newCanvas.toDataURL("image/png"); + return img; +} + +function getTranscriptAroundCurrentTimestamp(fullTranscript) { + var player = getStreamPlayer() as HTMLVideoElement; + var time = player.currentTime; + + let result = ""; + + for (let i = 0; i < fullTranscript.length; i++) { + if (fullTranscript[i][0] < time - 15) { + continue; + } + if (fullTranscript[i][0] > time + 15) { + break; + } + // Include transcripts within 10 seconds of the current timestamp + result += fullTranscript[i][1] + " "; + } + + return result; +} + +function getTranscriptAndReturnAllYoutubeDetails() { + const injectedCode = ` +h2 = document.createElement('h2') +h2.id = 'ytTranscript' +h2.innerText = ytInitialPlayerResponse.captions.playerCaptionsTracklistRenderer.captionTracks[0].baseUrl +document.getElementById('content').appendChild(h2) +`; + var script = document.createElement("script"); + script.textContent = injectedCode; + (document.head).appendChild(script); + var ytTranscript = document.getElementById('ytTranscript'); + var subsUrl; + if (ytTranscript) { + subsUrl = document.getElementById('ytTranscript').innerText; + let xhr = new XMLHttpRequest(); + xhr.open('GET', subsUrl); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4 && xhr.status === 200) { + // Manipulate this code to get the timestamp as well + let xml = new DOMParser().parseFromString(xhr.responseText, 'text/xml'); + let textNodes = Array.prototype.slice.call(xml.getElementsByTagName('text')); + let subsText = [] + for (const t of textNodes) { + var obj = []; + const start = t.getAttribute('start'); + const duration = t.getAttribute('dur'); + obj.push(start) + obj.push(t.textContent) + subsText.push(obj) + } + console.log(JSON.stringify(subsText)); + chrome.runtime.sendMessage({ + youtubeURL: getYoutubeURL(), + documentTitle: getDocumentTitle(), + pictureOfVideo: getPictureOfVideo(), + transcript: getTranscriptAroundCurrentTimestamp(subsText) + }); + } + } + xhr.send(); + } +} + function parseParams(href) { const noHash = href.split("#")[0]; const paramString = noHash.split("?")[1];