Updated Gmail API search endpoint. ALlow searching by threadId and em…

…ailId
postalsys · May 14, 2024 · 734c1e6 · 734c1e6
1 parent 8d13a94
commit 734c1e6
Show file tree

Hide file tree

Showing 11 changed files with 261 additions and 88 deletions.
diff --git a/data/google-crawlers.json b/data/google-crawlers.json
@@ -1,12 +1,9 @@
 {
-    "creationTime": "2024-04-30T22:00:37.000000",
+    "creationTime": "2024-05-07T22:00:49.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"
         },
-        {
-            "ipv6Prefix": "2001:4860:4801:2009::/64"
-        },
         {
             "ipv6Prefix": "2001:4860:4801:200c::/64"
         },
@@ -445,9 +442,6 @@
         {
             "ipv4Prefix": "66.249.92.32/27"
         },
-        {
-            "ipv4Prefix": "66.249.92.64/27"
-        },
         {
             "ipv4Prefix": "66.249.92.96/27"
         },

diff --git a/lib/api-client/gmail-client.js b/lib/api-client/gmail-client.js
@@ -70,9 +70,21 @@ class PageCursor {
     }
 
     constructor(cursorStr) {
+        this.type = 'gmail';
         this.cursorList = [];
         this.cursorStr = '';
         if (cursorStr) {
+            let splitPos = cursorStr.indexOf('_');
+            if (splitPos >= 0) {
+                let cursorType = cursorStr.substring(0, splitPos);
+                cursorStr = cursorStr.substring(splitPos + 1);
+                if (cursorType && this.type !== cursorType) {
+                    let error = new Error('Invalid cursor');
+                    error.code = 'InvalidCursorType';
+                    throw error;
+                }
+            }
+
             try {
                 this.cursorList = msgpack.decode(Buffer.from(cursorStr, 'base64url'));
                 this.cursorStr = cursorStr;
@@ -88,7 +100,7 @@ class PageCursor {
     }
 
     currentPage() {
-        if (this.cursorList.length <= 1) {
+        if (this.cursorList.length < 1) {
             return { page: 0, cursor: '', pageCursor: '' };
         }
 
@@ -103,15 +115,15 @@ class PageCursor {
         // if nextPageCursor is an array, then it will be flattened, have to push instead
         let cursorListCopy = this.cursorList.concat([]);
         cursorListCopy.push(encodedCursor);
-        return msgpack.encode(cursorListCopy).toString('base64url');
+        return this.type + '_' + msgpack.encode(cursorListCopy).toString('base64url');
     }
 
     prevPageCursor() {
         if (this.cursorList.length <= 1) {
             return null;
         }
 
-        return msgpack.encode(this.cursorList.slice(0, this.cursorList.length - 1)).toString('base64url');
+        return this.type + '_' + msgpack.encode(this.cursorList.slice(0, this.cursorList.length - 1)).toString('base64url');
     }
 
     encodeCursorValue(cursor) {
@@ -287,7 +299,87 @@ class GmailClient extends BaseClient {
             requestQuery.labelIds = [label.id];
         }
 
+        let messageList = [];
+
         if (query.search) {
+            if (query.search.emailId) {
+                // Return only a single matching email
+
+                let messageEntry = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${query.search.emailId}`, 'get', {
+                    format: 'full'
+                });
+                if (messageEntry) {
+                    messageList.push(this.formatMessage(messageEntry, { path }));
+                }
+
+                let messageCount = messageList.length;
+                let pages = Math.ceil(messageCount / pageSize);
+
+                return {
+                    total: messageCount,
+                    page: 0,
+                    pages,
+                    nextPageCursor: null,
+                    prevPageCursor: null,
+                    messages: messageList
+                };
+            }
+
+            if (query.search.threadId) {
+                // Threading is a special case
+                let threadListingResult = await this.oAuth2Client.request(
+                    accessToken,
+                    `${GMAIL_API_BASE}/gmail/v1/users/me/threads/${query.search.threadId}`,
+                    'get',
+                    {
+                        format: 'full'
+                    }
+                );
+
+                let messageCount = threadListingResult?.messages?.length || 0;
+                let currentPage = pageCursor.currentPage();
+
+                console.log('CURRENT PAGE', query.cursor, currentPage);
+
+                let nextPageToken = null;
+                if (messageCount > pageSize) {
+                    let pageStart = 0;
+
+                    if (currentPage?.cursor) {
+                        pageStart = Number(currentPage.cursor) || 0;
+                    }
+
+                    if (pageStart + pageSize < messageCount) {
+                        nextPageToken = (pageStart + pageSize).toString(10);
+                    }
+
+                    // extract messages for the current page only
+                    threadListingResult.messages = threadListingResult.messages.slice(pageStart, pageStart + pageSize, messageCount);
+                }
+
+                if (threadListingResult?.messages) {
+                    for (let entry of threadListingResult.messages) {
+                        messageList.push(this.formatMessage(entry, { path }));
+                    }
+                }
+
+                let pages = Math.ceil(messageCount / pageSize);
+                let nextPageCursor = pageCursor.nextPageCursor(nextPageToken);
+                let prevPageCursor = pageCursor.prevPageCursor();
+
+                console.log('THREAD LISTING');
+                console.log(JSON.stringify(messageList, false, 2));
+
+                return {
+                    total: messageCount,
+                    page: currentPage.page,
+                    pages,
+                    nextPageCursor,
+                    prevPageCursor,
+                    messages: messageList
+                };
+            }
+
             // NB! Might throw if using unsupported search terms
             const preparedQuery = this.prepareQuery(query.search);
             if (preparedQuery) {
@@ -302,11 +394,10 @@ class GmailClient extends BaseClient {
 
         console.log('REQUEST QUERY', requestQuery);
 
-        let messageList = [];
         let listingResult = await this.oAuth2Client.request(accessToken, `${GMAIL_API_BASE}/gmail/v1/users/me/messages`, 'get', requestQuery);
         let messageCount = listingResult.resultSizeEstimate;
 
-        let pages = Math.ceil(messageCount / pageSize) || 1;
+        let pages = Math.ceil(messageCount / pageSize);
 
         let nextPageCursor = pageCursor.nextPageCursor(listingResult.nextPageToken);
         let prevPageCursor = pageCursor.prevPageCursor();
@@ -851,8 +942,6 @@ class GmailClient extends BaseClient {
 
         const queryParts = [];
 
-        // FIXME: emailId, threadId
-
         // not supported search terms
         for (let disabledKey of ['seq', 'uid', 'paths', 'answered', 'deleted', 'draft']) {
             if (disabledKey in search) {
@@ -924,9 +1013,25 @@ if (/gmail-client\.js$/.test(process.argv[1])) {
         console.log(mailboxes);
 
         let messages;
-        let cursor;
+        let cursor; //= 'gmail_kcQIAwAAAAAAAAA';
+
+        while (Date.now()) {
+            messages = await gmailClient.listMessages({
+                path: 'INBOX',
+                pageSize: 3,
+                search: {
+                    emailId: '18f765767efa70c0',
+                    threadId: '18f4281215decd54' /*from: '[email protected]'*/
+                },
+                cursor
+            });
 
-        messages = await gmailClient.listMessages({ path: 'INBOX', search: { from: '[email protected]' }, cursor });
+            console.log(messages);
+            if (!messages.nextPageCursor) {
+                break;
+            }
+            cursor = messages.nextPageCursor;
+        }
 
         let deleted = false;