Compress sync data (#4951)

# What this PR does
- Compresses sync data being sent to engine
- Minor fix to log messages when JSON parse errors occur

## Which issue(s) this PR closes

Related to [issue link here]

<!--
*Note*: If you want the issue to be auto-closed once the PR is merged,
change "Related to" to "Closes" in the line above.
If you have more than one GitHub issue that this PR closes, be sure to
preface
each issue link with a [closing
keyword](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/using-keywords-in-issues-and-pull-requests#linking-a-pull-request-to-an-issue).
This ensures that the issue(s) are auto-closed once the PR has been
merged.
-->

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

engine/apps/grafana_plugin/serializers/sync_data.py

@@ -22,7 +22,7 @@ class SyncUserSerializer(serializers.Serializer):
     login = serializers.CharField()
     email = serializers.CharField()
     role = serializers.CharField()
-    avatar_url = serializers.CharField()
+    avatar_url = serializers.CharField(allow_blank=True)
     permissions = SyncPermissionSerializer(many=True, allow_empty=True, allow_null=True)
     teams = serializers.ListField(child=serializers.IntegerField(), allow_empty=True, allow_null=True)
 

engine/apps/grafana_plugin/tests/test_sync_v2.py

@@ -1,3 +1,6 @@
+import gzip
+import json
+from dataclasses import asdict
 from unittest.mock import patch
 
 import pytest
@@ -6,6 +9,7 @@
 from rest_framework.test import APIClient
 
 from apps.api.permissions import LegacyAccessControlRole
+from apps.grafana_plugin.sync_data import SyncData, SyncSettings, SyncUser
 from apps.grafana_plugin.tasks.sync_v2 import start_sync_organizations_v2
 
 
@@ -76,3 +80,59 @@ def test_skip_org_without_api_token(make_organization, api_token, sync_called):
         ) as mock_sync:
             start_sync_organizations_v2()
             assert mock_sync.called == sync_called
+
+
+@pytest.mark.parametrize("format", [("json"), ("gzip")])
+@pytest.mark.django_db
+def test_sync_v2_content_encoding(
+    make_organization_and_user_with_plugin_token, make_user_auth_headers, settings, format
+):
+    organization, user, token = make_organization_and_user_with_plugin_token()
+    settings.LICENSE = settings.CLOUD_LICENSE_NAME
+    client = APIClient()
+    headers = make_user_auth_headers(None, token, organization=organization)
+
+    data = SyncData(
+        users=[
+            SyncUser(
+                id=user.user_id,
+                name=user.username,
+                login=user.username,
+                email=user.email,
+                role="Admin",
+                avatar_url="",
+                permissions=[],
+                teams=[],
+            )
+        ],
+        teams=[],
+        team_members={},
+        settings=SyncSettings(
+            stack_id=organization.stack_id,
+            org_id=organization.org_id,
+            license=settings.CLOUD_LICENSE_NAME,
+            oncall_api_url="http://localhost",
+            oncall_token="",
+            grafana_url="http://localhost",
+            grafana_token="fake_token",
+            rbac_enabled=False,
+            incident_enabled=False,
+            incident_backend_url="",
+            labels_enabled=False,
+        ),
+    )
+
+    payload = asdict(data)
+    headers["HTTP_Content-Type"] = "application/json"
+    url = reverse("grafana-plugin:sync-v2")
+    with patch("apps.grafana_plugin.views.sync_v2.apply_sync_data") as mock_sync:
+        if format == "gzip":
+            headers["HTTP_Content-Encoding"] = "gzip"
+            json_data = json.dumps(payload)
+            payload = gzip.compress(json_data.encode("utf-8"))
+            response = client.generic("POST", url, data=payload, **headers)
+        else:
+            response = client.post(url, format=format, data=payload, **headers)
+
+        assert response.status_code == status.HTTP_200_OK
+        mock_sync.assert_called()

engine/apps/grafana_plugin/views/sync_v2.py

@@ -1,3 +1,5 @@
+import gzip
+import json
 import logging
 from dataclasses import asdict, is_dataclass
 
@@ -25,7 +27,14 @@ class SyncV2View(APIView):
     authentication_classes = (BasePluginAuthentication,)
 
     def do_sync(self, request: Request) -> Organization:
-        serializer = SyncDataSerializer(data=request.data)
+        if request.headers.get("Content-Encoding") == "gzip":
+            gzip_data = gzip.GzipFile(fileobj=request).read()
+            decoded_data = gzip_data.decode("utf-8")
+            data = json.loads(decoded_data)
+        else:
+            data = request.data
+
+        serializer = SyncDataSerializer(data=data)
         if not serializer.is_valid():
             raise SyncException(serializer.errors)
 

grafana-plugin/pkg/plugin/debug.go

@@ -95,3 +95,8 @@ func (a *App) handleDebugStats(w http.ResponseWriter, req *http.Request) {
 	}
 	w.WriteHeader(http.StatusOK)
 }
+
+func (a *App) handleDebugUnlock(w http.ResponseWriter, req *http.Request) {
+	a.OnCallSyncCache.syncMutex.Unlock()
+	w.WriteHeader(http.StatusOK)
+}

grafana-plugin/pkg/plugin/permissions.go

@@ -41,7 +41,7 @@ func (a *App) GetPermissions(settings *OnCallPluginSettings, onCallUser *OnCallU
 	var permissions []OnCallPermission
 	err = json.Unmarshal(body, &permissions)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	if res.StatusCode == 200 {
@@ -88,7 +88,7 @@ func (a *App) GetAllPermissions(settings *OnCallPluginSettings) (map[string]map[
 	var permissions map[string]map[string]interface{}
 	err = json.Unmarshal(body, &permissions)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	if res.StatusCode == 200 {

grafana-plugin/pkg/plugin/resources.go

@@ -132,6 +132,7 @@ func (a *App) registerRoutes(mux *http.ServeMux) {
 	//mux.HandleFunc("/debug/settings", a.handleDebugSettings)
 	//mux.HandleFunc("/debug/permissions", a.handleDebugPermissions)
 	//mux.HandleFunc("/debug/stats", a.handleDebugStats)
+	//mux.HandleFunc("/debug/unlock", a.handleDebugUnlock)
 
 	mux.HandleFunc("/", a.handleInternalApi)
 }

grafana-plugin/pkg/plugin/settings.go

@@ -268,7 +268,7 @@ func (a *App) GetOtherPluginSettings(settings *OnCallPluginSettings, pluginID st
 	var result map[string]interface{}
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	return result, nil

grafana-plugin/pkg/plugin/sync.go

@@ -2,6 +2,7 @@ package plugin
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"errors"
@@ -136,6 +137,16 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error {
 		return fmt.Errorf("error marshalling JSON: %v", err)
 	}
 
+	var syncDataBuffer bytes.Buffer
+	gzipWriter := gzip.NewWriter(&syncDataBuffer)
+	_, err = gzipWriter.Write(onCallSyncJsonData)
+	if err != nil {
+		return fmt.Errorf("error writing sync data to gzip writer: %v", err)
+	}
+	if err := gzipWriter.Close(); err != nil {
+		return fmt.Errorf("error closing gzip writer: %v", err)
+	}
+
 	syncURL, err := url.JoinPath(onCallPluginSettings.OnCallAPIURL, "api/internal/v1/plugin/v2/sync")
 	if err != nil {
 		return fmt.Errorf("error joining path: %v", err)
@@ -146,7 +157,7 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error {
 		return fmt.Errorf("error parsing path: %v", err)
 	}
 
-	syncReq, err := http.NewRequest("POST", parsedSyncURL.String(), bytes.NewBuffer(onCallSyncJsonData))
+	syncReq, err := http.NewRequest("POST", parsedSyncURL.String(), &syncDataBuffer)
 	if err != nil {
 		return fmt.Errorf("error creating request: %v", err)
 	}
@@ -156,6 +167,7 @@ func (a *App) makeSyncRequest(ctx context.Context, forceSend bool) error {
 		return err
 	}
 	syncReq.Header.Set("Content-Type", "application/json")
+	syncReq.Header.Set("Content-Encoding", "gzip")
 
 	res, err := a.httpClient.Do(syncReq)
 	if err != nil {

grafana-plugin/pkg/plugin/teams.go

@@ -70,7 +70,7 @@ func (a *App) GetTeamsForUser(settings *OnCallPluginSettings, onCallUser *OnCall
 	var result []Team
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	if res.StatusCode == 200 {
@@ -115,7 +115,7 @@ func (a *App) GetAllTeams(settings *OnCallPluginSettings) ([]OnCallTeam, error)
 	var result Teams
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	if res.StatusCode == 200 {
@@ -161,7 +161,7 @@ func (a *App) GetTeamsMembersForTeam(settings *OnCallPluginSettings, onCallTeam
 	var result []OrgUser
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	if res.StatusCode == 200 {

grafana-plugin/pkg/plugin/users.go

@@ -233,7 +233,7 @@ func (a *App) GetAllUsers(settings *OnCallPluginSettings) ([]OnCallUser, error)
 	var result []OrgUser
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, body)
+		return nil, fmt.Errorf("failed to parse JSON response: %v body=%v", err, string(body))
 	}
 
 	if res.StatusCode == 200 {