refactor: refactoring connectors

plugins/connectors/google_drive/api.go

@@ -81,8 +81,8 @@ func (h *Plugin) oAuthRedirect(w http.ResponseWriter, req *http.Request, _ httpr
 	}
 
 	//// Access custom arguments
-	tenantID := customArgs["tenant"]
-	userID := customArgs["user"]
+	//tenantID := customArgs["tenant"]
+	//userID := customArgs["user"]
 	redirectPath := customArgs["redirect"]
 
 	// Extract the code from the query parameters
@@ -98,7 +98,7 @@ func (h *Plugin) oAuthRedirect(w http.ResponseWriter, req *http.Request, _ httpr
 	}
 
 	datasource := common.DataSource{}
-	datasource.ID = util.GetUUID()
+	datasource.ID = util.GetUUID() //TODO routing to single task, if connect multi-times
 	datasource.Type = "connector"
 	datasource.Name = "My Google Drive" //TODO, input from user
 	datasource.Connector = common.ConnectorConfig{
@@ -113,25 +113,26 @@ func (h *Plugin) oAuthRedirect(w http.ResponseWriter, req *http.Request, _ httpr
 		panic(err)
 	}
 
-	err = h.saveToken(tenantID, userID, token)
-	if err != nil {
-		panic(err)
-	}
+	//err = h.saveToken(tenantID, userID, token)
+	//if err != nil {
+	//	panic(err)
+	//}
 
 	newRedirectUrl := util.JoinPath(redirectPath, "?source=google_drive")
 	h.Redirect(w, req, newRedirectUrl)
 }
 
-func (h *Plugin) getTenantKey(tenantID, userID string) string {
-	return strings.Join([]string{tenantID, userID}, ",")
+func (h *Plugin) getTenantKey(tenantID, userID, datasourceID string) string {
+	return strings.Join([]string{tenantID, userID, datasourceID}, ",")
 }
 
 func (h *Plugin) reset(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
 	//from context
 	tenantID := "test"
 	userID := "test"
+	datasourceID := h.GetParameter(req, "datasource")
 
-	tenantKey := h.getTenantKey(tenantID, userID)
+	tenantKey := h.getTenantKey(tenantID, userID, datasourceID)
 	err := kv.DeleteKey("/connector/google_drive/lastModifiedTime", []byte(tenantKey))
 	if err != nil {
 		panic(err)
@@ -144,44 +145,18 @@ func (h *Plugin) reset(w http.ResponseWriter, req *http.Request, _ httprouter.Pa
 	h.WriteAckOKJSON(w)
 }
 
-func (this *Plugin) saveLastModifiedTime(tenantID, userID string, lastModifiedTime string) error {
-	tenantKey := this.getTenantKey(tenantID, userID)
+func (this *Plugin) saveLastModifiedTime(tenantID, userID, datasourceID string, lastModifiedTime string) error {
+	tenantKey := this.getTenantKey(tenantID, userID, datasourceID)
 	err := kv.AddValue("/connector/google_drive/lastModifiedTime", []byte(tenantKey), []byte(lastModifiedTime))
 	return err
 }
 
-func (this *Plugin) getLastModifiedTime(tenantID, userID string) (string, error) {
-	tenantKey := this.getTenantKey(tenantID, userID)
+func (this *Plugin) getLastModifiedTime(tenantID, userID, datasourceID string) (string, error) {
+	tenantKey := this.getTenantKey(tenantID, userID, datasourceID)
 	data, err := kv.GetValue("/connector/google_drive/lastModifiedTime", []byte(tenantKey))
 	if err != nil {
 		return "", err
 	}
 
 	return string(data), nil
 }
-
-func (this *Plugin) saveToken(tenantID, userID string, token *oauth2.Token) error {
-	tenantKey := this.getTenantKey(tenantID, userID)
-	err := kv.AddValue("/connector/google_drive/token", []byte(tenantKey), util.MustToJSONBytes(token))
-	return err
-}
-
-func (this *Plugin) getToken(tenantID, userID string) (bool, *oauth2.Token, error) {
-	tenantKey := this.getTenantKey(tenantID, userID)
-	data, err := kv.GetValue("/connector/google_drive/token", []byte(tenantKey))
-	if err != nil {
-		return false, nil, err
-	}
-
-	if data != nil {
-		obj := oauth2.Token{}
-		err = util.FromJSONBytes(data, &obj)
-		if err != nil {
-			return false, nil, err
-		}
-
-		return true, &obj, nil
-	}
-
-	return false, nil, nil
-}

plugins/connectors/google_drive/files.go

@@ -30,12 +30,40 @@ func getIcon(fileType string) string {
 		return "presentation"
 	case "application/vnd.google-apps.spreadsheet":
 		return "spreadsheet"
+	case "application/vnd.google-apps.drawing":
+		return "drawing"
+	case "application/vnd.google-apps.folder":
+		return "folder"
+	case "application/vnd.google-apps.fusiontable":
+		return "fusiontable"
+	case "application/vnd.google-apps.jam":
+		return "jam"
+	case "application/vnd.google-apps.map":
+		return "map"
+	case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": // MS Excel
+		return "ms_excel"
+	case "application/vnd.openxmlformats-officedocument.presentationml.presentation": // MS PowerPoint
+		return "ms_powerpoint"
+	case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": // MS Word
+		return "ms_word"
+	case "application/vnd.google-apps.script":
+		return "script"
+	case "application/vnd.google-apps.site":
+		return "site"
+	case "application/vnd.google-apps.video":
+		return "video"
+	case "application/zip":
+		return "zip"
+	case "image/jpeg", "image/png", "image/gif", "image/tiff", "image/bmp": // Image formats
+		return "photo"
+	case "audio/mpeg", "audio/wav", "audio/mp3", "audio/ogg": // Audio formats
+		return "audio"
 	default:
-		return "default" // default fallback
+		return "default" // Default fallback
 	}
 }
 
-func (this *Plugin) startIndexingFiles(tenantID, userID string, tok *oauth2.Token) {
+func (this *Plugin) startIndexingFiles(tenantID, userID, datasourceID string, tok *oauth2.Token) {
 	var filesProcessed = 0
 	defer func() {
 		if !global.Env().IsDebug {
@@ -68,7 +96,7 @@ func (this *Plugin) startIndexingFiles(tenantID, userID string, tok *oauth2.Toke
 	var query string
 
 	//get last access time from kv
-	lastModifiedTimeStr, _ := this.getLastModifiedTime(tenantID, userID)
+	lastModifiedTimeStr, _ := this.getLastModifiedTime(tenantID, userID, datasourceID)
 
 	log.Tracef("get last modified time: %v", lastModifiedTimeStr)
 
@@ -216,7 +244,7 @@ func (this *Plugin) startIndexingFiles(tenantID, userID string, tok *oauth2.Toke
 		if lastModifyTime != nil {
 			// Save the lastModifyTime (for example, in a KV store or file)
 			lastModifiedTimeStr = lastModifyTime.Format(time.RFC3339Nano)
-			err := this.saveLastModifiedTime(tenantID, userID, lastModifiedTimeStr)
+			err := this.saveLastModifiedTime(tenantID, userID, lastModifiedTimeStr, datasourceID)
 			if err != nil {
 				panic(err)
 			}

plugins/connectors/google_drive/plugin.go

@@ -185,7 +185,7 @@ func (this *Plugin) fetch_google_drive(connector *common.Connector, datasource *
 			log.Warnf("skip invalid google_drive token: %v", tok)
 		} else {
 			log.Debug("start processing google drive files")
-			this.startIndexingFiles(tenantID, userID, &tok)
+			this.startIndexingFiles(tenantID, userID, datasource.ID, &tok)
 			log.Debug("finished process google drive files")
 		}
 	}

plugins/connectors/yuque/collect.go

@@ -32,7 +32,7 @@ func get(path, token string) *util.Result {
 }
 
 func (this *Plugin) getIconKey(category, iconType string) string {
-	return fmt.Sprintf("%v_%v", strings.TrimSpace(strings.ToLower(category)), strings.TrimSpace(strings.ToLower(iconType)))
+	return strings.TrimSpace(strings.ToLower(iconType))
 }
 
 func (this *Plugin) cleanupIconName(name string) string {