Skip to content

Commit

Permalink
refactor: refactoring connectors
Browse files Browse the repository at this point in the history
  • Loading branch information
@medcl
medcl committed Jan 19, 2025
1 parent 9521630 commit 356dd39
Show file tree
Hide file tree
Showing 8 changed files with 202 additions and 128 deletions.
25 changes: 4 additions & 21 deletions coco.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,34 +184,17 @@ web:

connector:
google_drive: # get your token here: https://developers.google.com/drive/api/quickstart/go
enabled: true
enabled: false
queue:
name: indexing_documents
# credential_file: credentials.json
credential:
"client_id": "XXXX"
"project_id": "XXXX"
"auth_uri": "https://accounts.google.com/o/oauth2/auth"
"token_uri": "https://oauth2.googleapis.com/token"
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs"
"client_secret": "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"
"redirect_uris": "http://localhost:9000/connector/google_drive/oauth_redirect"
"javascript_origins": [ "http://localhost:9000" ]
interval: 10s
credential_file: credentials.json
interval: 60s
skip_invalid_token: true
yuque:
enabled: false
queue:
name: indexing_documents
token: $[[keystore.YUQUE_TOKEN]]
interval: 10s
indexing_books: true
indexing_docs: true
indexing_groups: true
indexing_users: true
include_private_book: false
include_private_doc: false
skip_invalid_token: true
interval: 60s
hugo_site:
enabled: false
interval: 60s
Expand Down
4 changes: 2 additions & 2 deletions docs/content.en/docs/references/connectors/google_drive.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ Below is the config parameters supported by this connector.
|-------------------------------|--------------|--------------------------------------------------------------------------------------------------|
| `enabled` | `bool` | Set to `true` to enable the Google Drive Connector. |
| `queue.name` | `string` | Specifies the queue name for indexing documents. |
| `interval` | `duration` | Interval for polling the Google Drive API. |
| `skip_invalid_token` | `bool` | Skip errors caused by invalid tokens if set to `true`. |
| `credential` | `object` | Inline Google Drive API credentials. |
| `credential_file` | `string` | Path to the `credentials.json` file (optional if `credential` is used). |
| `client_id` | `string` | Google Drive client ID obtained from Google API Console. |
Expand All @@ -110,8 +112,6 @@ Below is the config parameters supported by this connector.
| `client_secret` | `string` | Client secret for Google Drive API. |
| `redirect_uris` | `string` | Callback URI for authorization responses. |
| `javascript_origins` | `[]string` | List of allowed JavaScript origins for the application. |
| `interval` | `duration` | Interval for polling the Google Drive API. |
| `skip_invalid_token` | `bool` | Skip errors caused by invalid tokens if set to `true`. |

> **Notes**:
> - Use either `credential_file` or `credential` for providing credentials.
Expand Down
19 changes: 19 additions & 0 deletions docs/content.en/docs/references/connectors/hugo_site.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,25 @@ curl -XPUT http://localhost:9000/connector/hugo_site?replace=true -d '{

> Use `hugo_site` as a unique identifier, as it is a builtin connector.
## Update coco-server's config

Below is an example configuration for enabling the Hugo Site Drive Connector in coco-server:

```shell
connector:
hugo_site:
enabled: true
interval: 60s
queue:
name: indexing_documents
```
### Explanation of Config Parameters

| **Field** | **Type** | **Description** |
|-----------------|-----------|---------------------------------------------------------------------------------|
| `enabled` | `boolean` | Enables or disables the Hugo Site connector. Set to `true` to activate it. |
| `interval` | `string` | Specifies the time interval (e.g., `60s`) at which the connector will check for updates. |
| `queue.name` | `string` | Defines the name of the queue where indexing tasks will be added. |

## Use the Hugo Site Connector

Expand Down
75 changes: 74 additions & 1 deletion docs/content.en/docs/references/connectors/yuque.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,77 @@ curl -XPUT http://localhost:9000/connector/yuque?replace=true -d '{
```


> Use `yuque` as a unique identifier, as it is a builtin connector.
> Use `yuque` as a unique identifier, as it is a builtin connector.

## Update coco-server's config

Below is an example configuration for enabling the Yuque Connector in coco-server:

```shell
connector:
yuque:
enabled: true
queue:
name: indexing_documents
interval: 10s
```

### Explanation of Config Parameters

| **Field** | **Type** | **Description** |
|-----------------|-----------|---------------------------------------------------------------------------------|
| `enabled` | `boolean` | Enables or disables the Hugo Site connector. Set to `true` to activate it. |
| `interval` | `string` | Specifies the time interval (e.g., `60s`) at which the connector will check for updates. |
| `queue.name` | `string` | Defines the name of the queue where indexing tasks will be added. |

## Use the Yuque Connector

The Yuque Connector allows you to index data from your Yuque account into your system. Follow these steps to set it up:

### Obtain Yuque API Token

Before using this connector, you need to obtain your Yuque API token. Refer to the official [Yuque API documentation](https://www.yuque.com/yuque/developer/api) for instructions.

### Example Request

Here is an example request to configure the Yuque Connector:

```shell
curl -H 'Content-Type: application/json' -XPOST http://localhost:9000/datasource/ -d '
{
"name": "My Yuque",
"type": "connector",
"connector": {
"id": "yuque",
"config": {
"token": "your_yuque_api_token",
"include_private_book": false,
"include_private_doc": false,
"indexing_books": true,
"indexing_docs": true,
"indexing_users": true,
"indexing_groups": true
}
}
}'
```

## Supported Config Parameters for Yuque Connector

Below are the configuration parameters supported by the Yuque Connector:

| **Field** | **Type** | **Description** |
|--------------------------|-----------|--------------------------------------------------------------------------------------------------|
| `token` | `string` | Your Yuque API token. This is required to access Yuque's API. |
| `include_private_book` | `bool` | Whether to include private books in indexing. Defaults to `false`. |
| `include_private_doc` | `bool` | Whether to include private documents in indexing. Defaults to `false`. |
| `indexing_books` | `bool` | Whether to index books in Yuque. Defaults to `false`. |
| `indexing_docs` | `bool` | Whether to index documents in Yuque. Defaults to `false`. |
| `indexing_users` | `bool` | Whether to index user data from Yuque. Defaults to `false`. |
| `indexing_groups` | `bool` | Whether to index group data from Yuque. Defaults to `false`. |

### Notes

- Set `token` to your valid Yuque API token to enable the connector.
- Boolean parameters like `include_private_book`, `indexing_books`, etc., allow you to customize the scope of indexing.
5 changes: 4 additions & 1 deletion plugins/connectors/google_drive/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ func (this *Plugin) Setup() {
this.Queue = &queue.QueueConfig{Name: "indexing_documents"}
}

this.Queue = queue.SmartGetOrInitConfig(this.Queue)

if this.CredentialFile != "" {
b, err := os.ReadFile(this.CredentialFile)
if err != nil {
Expand Down Expand Up @@ -179,7 +181,8 @@ func (this *Plugin) fetch_google_drive(connector *common.Connector, datasource *
if !this.SkipInvalidToken && !tok.Valid() {
panic("token is invalid")
}
log.Warnf("skip invalid token: %v", tok)
//TODO, update the datasource's config, avoid access before fix
log.Warnf("skip invalid google_drive token: %v", tok)
} else {
log.Debug("start processing google drive files")
this.startIndexingFiles(tenantID, userID, &tok)
Expand Down
2 changes: 2 additions & 0 deletions plugins/connectors/hugo_site/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ func (this *Plugin) Setup() {
if this.Queue == nil {
this.Queue = &queue.QueueConfig{Name: "indexing_documents"}
}

this.Queue = queue.SmartGetOrInitConfig(this.Queue)
}

// ParseTimestamp safely parses a timestamp string into a *time.Time.
Expand Down
38 changes: 19 additions & 19 deletions plugins/connectors/yuque/collect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,15 @@ func (this *Plugin) save(obj interface{}) {
if global.Env().IsDebug {
log.Tracef(string(data))
}
err := queue.Push(this.cfg.Queue, data)
err := queue.Push(this.Queue, data)
if err != nil {
panic(err)
}
}

func (this *Plugin) collect() {
func (this *Plugin) collect(cfg *YuqueConfig) {

token := this.cfg.Token
token := cfg.Token

if token == "" {
panic("invalid yuque token")
Expand All @@ -81,19 +81,19 @@ func (this *Plugin) collect() {
log.Infof("start collecting for %v", currentUser.Group.Login)

//get users in group
if this.cfg.IndexingUsers || this.cfg.IndexingGroups {
this.collectUsers(currentUser.Group.Login, token)
if cfg.IndexingUsers || cfg.IndexingGroups {
this.collectUsers(currentUser.Group.Login, token, cfg)
}

//get all books
if this.cfg.IndexingBooks || this.cfg.IndexingDocs {
this.collectBooks(currentUser.Group.Login, token)
if cfg.IndexingBooks || cfg.IndexingDocs {
this.collectBooks(currentUser.Group.Login, token, cfg)
}

log.Infof("finished collecting for %v", currentUser.Group.Login)
}

func (this *Plugin) collectBooks(login, token string) {
func (this *Plugin) collectBooks(login, token string, cfg *YuqueConfig) {

const limit = 100
offset := 0
Expand Down Expand Up @@ -125,7 +125,7 @@ func (this *Plugin) collectBooks(login, token string) {

bookID := bookDetail.Book.ID

if this.cfg.IndexingBooks && (bookDetail.Book.Public > 0 || (this.cfg.IncludePrivateBook)) {
if cfg.IndexingBooks && (bookDetail.Book.Public > 0 || (cfg.IncludePrivateBook)) {

//index books
document := common.Document{
Expand Down Expand Up @@ -174,8 +174,8 @@ func (this *Plugin) collectBooks(login, token string) {
}

//get docs in repo
if this.cfg.IndexingDocs {
this.collectDocs(login, bookID, token)
if cfg.IndexingDocs {
this.collectDocs(login, bookID, token, cfg)
}
}

Expand All @@ -188,7 +188,7 @@ func (this *Plugin) collectBooks(login, token string) {

}

func (this *Plugin) collectDocs(login string, bookID int64, token string) {
func (this *Plugin) collectDocs(login string, bookID int64, token string, cfg *YuqueConfig) {

const limit = 100
offset := 0
Expand All @@ -211,9 +211,9 @@ func (this *Plugin) collectDocs(login string, bookID int64, token string) {
log.Infof("fetched %v docs for %v, book: %v, offset: %v, total: %v", len(doc.Docs), login, bookID, offset, doc.Meta.Total)

for _, doc := range doc.Docs {
if this.cfg.IndexingDocs && (doc.Public > 0 || (this.cfg.IncludePrivateDoc)) {
if cfg.IndexingDocs && (doc.Public > 0 || (cfg.IncludePrivateDoc)) {
//get doc details
this.collectDocDetails(bookID, doc.ID, token)
this.collectDocDetails(bookID, doc.ID, token, cfg)
} else {
log.Debug("skip doc:", doc.Title, ",", doc.Public)
}
Expand All @@ -228,7 +228,7 @@ func (this *Plugin) collectDocs(login string, bookID int64, token string) {

}

func (this *Plugin) collectDocDetails(bookID int64, docID int64, token string) {
func (this *Plugin) collectDocDetails(bookID int64, docID int64, token string, cfg *YuqueConfig) {

res := get(fmt.Sprintf("/api/v2/repos/%v/docs/%v", bookID, docID), token)
doc := struct {
Expand All @@ -240,7 +240,7 @@ func (this *Plugin) collectDocDetails(bookID int64, docID int64, token string) {
panic(err)
}

if this.cfg.IndexingDocs && (doc.Doc.Public > 0 || (this.cfg.IncludePrivateDoc)) {
if cfg.IndexingDocs && (doc.Doc.Public > 0 || (cfg.IncludePrivateDoc)) {
//index doc
document := common.Document{
Source: common.DataSourceReference{
Expand Down Expand Up @@ -301,7 +301,7 @@ func (this *Plugin) collectDocDetails(bookID int64, docID int64, token string) {
}
}

func (this *Plugin) collectUsers(login, token string) {
func (this *Plugin) collectUsers(login, token string, cfg *YuqueConfig) {
const pageSize = 100
offset := 0

Expand All @@ -324,7 +324,7 @@ func (this *Plugin) collectUsers(login, token string) {
var idPrefix, docType string
var metadata util.MapStr

if groupUser.User != nil && this.cfg.IndexingUsers {
if groupUser.User != nil && cfg.IndexingUsers {
idPrefix, docType = "yuque-user", groupUser.User.Type
metadata = util.MapStr{
"user_id": groupUser.User.ID,
Expand Down Expand Up @@ -353,7 +353,7 @@ func (this *Plugin) collectUsers(login, token string) {
document.Updated = &groupUser.User.UpdatedAt
document.Metadata = metadata

} else if groupUser.Group != nil && this.cfg.IndexingGroups {
} else if groupUser.Group != nil && cfg.IndexingGroups {
idPrefix, docType = "yuque-group", groupUser.Group.Type
metadata = util.MapStr{
"user_id": groupUser.Group.ID,
Expand Down
Loading

0 comments on commit 356dd39

Please sign in to comment.