From d24560cfa7dce1e901b0a458630ee3359fd8dd8d Mon Sep 17 00:00:00 2001 From: traut Date: Wed, 8 Jan 2025 14:04:59 +0100 Subject: [PATCH 1/5] Renaming and extending the attributes of a data source --- internal/builtin/data_rss.go | 66 ++++++++++++++++------- internal/builtin/data_rss_test.go | 85 +++++++++++++++++++++++------- plugin/ast/v1/ast.pb.go | 2 +- plugin/pluginapi/v1/content.pb.go | 2 +- plugin/pluginapi/v1/cty.pb.go | 2 +- plugin/pluginapi/v1/data.pb.go | 2 +- plugin/pluginapi/v1/dataspec.pb.go | 2 +- plugin/pluginapi/v1/hcl.pb.go | 2 +- plugin/pluginapi/v1/plugin.pb.go | 2 +- plugin/pluginapi/v1/schema.pb.go | 2 +- 10 files changed, 123 insertions(+), 44 deletions(-) diff --git a/internal/builtin/data_rss.go b/internal/builtin/data_rss.go index 5c107ef7..f310af08 100644 --- a/internal/builtin/data_rss.go +++ b/internal/builtin/data_rss.go @@ -27,9 +27,9 @@ import ( ) const ( - defaultRequestTimeout = 30 * time.Second - defaultUserAgent = "blackstork-rss/0.0.1" - defaultOnlyItemsAfterTimeFormat = "2006-01-02T15:04:05Z" + defaultRequestTimeout = 30 * time.Second + defaultUserAgent = "blackstork-rss/0.0.1" + defaultItemTimeFormat = "2006-01-02T15:04:05Z" ) // https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ @@ -82,7 +82,7 @@ func makeRSSDataSource() *plugin.DataSource { `, defaultUserAgent), }, { - Name: "fill_in_max_items", + Name: "max_items_to_fill", Type: cty.Number, ExampleVal: cty.BoolVal(false), Constraints: constraint.NonNull, @@ -93,11 +93,19 @@ func makeRSSDataSource() *plugin.DataSource { `, }, { - Name: "only_items_after_time", + Name: "items_after", Type: cty.String, ExampleVal: cty.StringVal("2024-12-23T00:00:00Z"), Doc: ` - Return only items after a specified date time, in the format "%Y-%m-%dT%H:%M:%S%Z". + Return only items published after a specified timestamp. The timestamp format is "%Y-%m-%dT%H:%M:%S%Z". + `, + }, + { + Name: "items_before", + Type: cty.String, + ExampleVal: cty.StringVal("2024-12-23T00:00:00Z"), + Doc: ` + Return only items published before a specified timestamp. The timestamp format is "%Y-%m-%dT%H:%M:%S%Z". `, }, }, @@ -137,7 +145,7 @@ func makeRSSDataSource() *plugin.DataSource { } } -func filterItems(ctx context.Context, feed *gofeed.Feed, from time.Time) *gofeed.Feed { +func filterItems(feed *gofeed.Feed, after time.Time, before time.Time) *gofeed.Feed { filteredItems := make([]*gofeed.Item, 0) for i := range feed.Items { @@ -149,10 +157,15 @@ func filterItems(ctx context.Context, feed *gofeed.Feed, from time.Time) *gofeed itemTime = item.UpdatedParsed } else if item.PublishedParsed != nil { itemTime = item.PublishedParsed + } else { + continue } - if itemTime == nil { + + if !after.IsZero() && itemTime.Before(after) { continue - } else if itemTime.Before(from) { + } + + if !before.IsZero() && itemTime.After(before) { continue } @@ -243,8 +256,10 @@ func fetchRSSData(ctx context.Context, params *plugin.RetrieveDataParams) (plugi fillInContent := params.Args.GetAttrVal("fill_in_content").True() useBrowserUserAgent := params.Args.GetAttrVal("use_browser_user_agent").True() - fillInMaxItems, _ := params.Args.GetAttrVal("fill_in_max_items").AsBigFloat().Int64() - onlyItemsAfterTimeAttr := params.Args.GetAttrVal("only_items_after_time") + fillInMaxItems, _ := params.Args.GetAttrVal("max_items_to_fill").AsBigFloat().Int64() + + itemsAfterTimeAttr := params.Args.GetAttrVal("items_after") + itemsBeforeTimeAttr := params.Args.GetAttrVal("items_before") userAgent := defaultUserAgent if useBrowserUserAgent { @@ -274,11 +289,25 @@ func fetchRSSData(ctx context.Context, params *plugin.RetrieveDataParams) (plugi }} } - var fromTime time.Time - if !onlyItemsAfterTimeAttr.IsNull() { - fromTime, err = time.Parse(defaultOnlyItemsAfterTimeFormat, onlyItemsAfterTimeAttr.AsString()) + var afterTime time.Time + if !itemsAfterTimeAttr.IsNull() { + afterTime, err = time.Parse(defaultItemTimeFormat, itemsAfterTimeAttr.AsString()) + if err != nil { + errorMsg := "Can't parse the value in `items_after` argument" + log.ErrorContext(ctx, errorMsg, "err", err) + return nil, diagnostics.Diag{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: errorMsg, + Detail: err.Error(), + }} + } + } + + var beforeTime time.Time + if !itemsBeforeTimeAttr.IsNull() { + beforeTime, err = time.Parse(defaultItemTimeFormat, itemsBeforeTimeAttr.AsString()) if err != nil { - errorMsg := "Can't parse the value in `only_items_after_time` argument" + errorMsg := "Can't parse the value in `items_before` argument" log.ErrorContext(ctx, errorMsg, "err", err) return nil, diagnostics.Diag{&hcl.Diagnostic{ Severity: hcl.DiagError, @@ -288,15 +317,16 @@ func fetchRSSData(ctx context.Context, params *plugin.RetrieveDataParams) (plugi } } - if !fromTime.IsZero() { + if !afterTime.IsZero() || !beforeTime.IsZero() { oldItemsCount := len(feed.Items) - feed = filterItems(ctx, feed, fromTime) + feed = filterItems(feed, afterTime, beforeTime) log.InfoContext( ctx, "Feed items filtered", "old_items_count", oldItemsCount, "new_items_count", len(feed.Items), - "only_items_after_time", fromTime, + "items_after", afterTime, + "items_before", beforeTime, ) } diff --git a/internal/builtin/data_rss_test.go b/internal/builtin/data_rss_test.go index 324bf73a..8e51d279 100644 --- a/internal/builtin/data_rss_test.go +++ b/internal/builtin/data_rss_test.go @@ -140,7 +140,16 @@ func Test_fetchRSSData(t *testing.T) { }, } - ValidRssDataTimeFilter := plugindata.Map{ + ValidRssDataAfterBeforeTimeFiltered := plugindata.Map{ + "description": plugindata.String("This is an example of an RSS feed"), + "link": plugindata.String("http://www.example.com/main.html"), + "pub_date": plugindata.String("Sun, 6 Sep 2009 16:20:12 +0000"), + "title": plugindata.String("RSS Title"), + "pub_timestamp": plugindata.Number(1252254012), + "items": plugindata.List{}, + } + + ValidRssDataAfterTimeFiltered := plugindata.Map{ "description": plugindata.String("This is an example of an RSS feed"), "link": plugindata.String("http://www.example.com/main.html"), "pub_date": plugindata.String("Sun, 6 Sep 2009 16:20:12 +0000"), @@ -159,6 +168,25 @@ func Test_fetchRSSData(t *testing.T) { }, } + ValidRssDataBeforeimeFiltered := plugindata.Map{ + "description": plugindata.String("This is an example of an RSS feed"), + "link": plugindata.String("http://www.example.com/main.html"), + "pub_date": plugindata.String("Sun, 6 Sep 2009 16:20:12 +0000"), + "title": plugindata.String("RSS Title"), + "pub_timestamp": plugindata.Number(1252254012), + "items": plugindata.List{ + plugindata.Map{ + "description": plugindata.String("Here is some text containing an interesting description."), + "guid": plugindata.String("7bd204c6-1655-4c27-aeee-53f933c5395f"), + "link": plugindata.String(addr + "content/1"), + "pub_date": plugindata.String("Sun, 6 Sep 2009 16:20:23 +0000"), + "title": plugindata.String("Example entry 1"), + "pub_timestamp": plugindata.Number(1252254023), + "content": plugindata.String(""), + }, + }, + } + ValidRssWithContent := plugindata.Map{ "description": plugindata.String("This is an example of an RSS feed"), "link": plugindata.String("http://www.example.com/main.html"), @@ -175,7 +203,7 @@ func Test_fetchRSSData(t *testing.T) { "pub_timestamp": plugindata.Number(1252447200), // Note, go-readability wraps content in a `div`. We can use `article.TextContent` if // we want only the text content returned - "content": plugindata.String("
test-content\n
"), + "content": plugindata.String("
test-content\n
"), }, plugindata.Map{ "description": plugindata.String("Here is some text containing an interesting description."), @@ -208,13 +236,14 @@ func Test_fetchRSSData(t *testing.T) { } tt := []struct { - name string - url string - only_items_after_time string - fill_in_content bool - fill_in_max_items int - auth *gofeed.Auth - expected result + name string + url string + items_after string + items_before string + fill_in_content bool + max_items_to_fill int + auth *gofeed.Auth + expected result }{ { name: "valid_rss", @@ -231,18 +260,35 @@ func Test_fetchRSSData(t *testing.T) { }, }, { - name: "valid_rss_with_time_filter", - url: "data/basic.rss", - only_items_after_time: "2009-09-07T00:00:00Z", + name: "valid_rss_items_after", + url: "data/basic.rss", + items_after: "2009-09-07T00:00:00Z", expected: result{ - Data: ValidRssDataTimeFilter, + Data: ValidRssDataAfterTimeFiltered, + }, + }, + { + name: "valid_rss_items_after_before", + url: "data/basic.rss", + items_after: "2009-09-07T00:00:00Z", + items_before: "2009-09-08T00:00:00Z", + expected: result{ + Data: ValidRssDataAfterBeforeTimeFiltered, + }, + }, + { + name: "valid_rss_items_before", + url: "data/basic.rss", + items_before: "2009-09-08T00:00:00Z", + expected: result{ + Data: ValidRssDataBeforeimeFiltered, }, }, { name: "valid_rss_with_fill_in", url: "data/basic.rss", fill_in_content: true, - fill_in_max_items: 1, + max_items_to_fill: 1, expected: result{ Data: ValidRssWithContent, }, @@ -354,13 +400,16 @@ func Test_fetchRSSData(t *testing.T) { dec := plugintest.NewTestDecoder(t, p.DataSources["rss"].Args). SetAttr("url", cty.StringVal(addr+tc.url)) - if tc.only_items_after_time != "" { - dec = dec.SetAttr("only_items_after_time", cty.StringVal(tc.only_items_after_time)) + if tc.items_after != "" { + dec = dec.SetAttr("items_after", cty.StringVal(tc.items_after)) + } + if tc.items_before != "" { + dec = dec.SetAttr("items_before", cty.StringVal(tc.items_before)) } dec = dec.SetAttr("fill_in_content", cty.BoolVal(tc.fill_in_content)) - if tc.fill_in_max_items != 0 { - dec = dec.SetAttr("fill_in_max_items", cty.NumberIntVal(int64(tc.fill_in_max_items))) + if tc.max_items_to_fill != 0 { + dec = dec.SetAttr("max_items_to_fill", cty.NumberIntVal(int64(tc.max_items_to_fill))) } if tc.auth != nil { diff --git a/plugin/ast/v1/ast.pb.go b/plugin/ast/v1/ast.pb.go index c4a1eb0f..c6a3d800 100644 --- a/plugin/ast/v1/ast.pb.go +++ b/plugin/ast/v1/ast.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: ast/v1/ast.proto diff --git a/plugin/pluginapi/v1/content.pb.go b/plugin/pluginapi/v1/content.pb.go index 21488d2d..6cdce1d2 100644 --- a/plugin/pluginapi/v1/content.pb.go +++ b/plugin/pluginapi/v1/content.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/content.proto diff --git a/plugin/pluginapi/v1/cty.pb.go b/plugin/pluginapi/v1/cty.pb.go index cfb000b9..6cefc6db 100644 --- a/plugin/pluginapi/v1/cty.pb.go +++ b/plugin/pluginapi/v1/cty.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/cty.proto diff --git a/plugin/pluginapi/v1/data.pb.go b/plugin/pluginapi/v1/data.pb.go index caf7325a..fe4a0828 100644 --- a/plugin/pluginapi/v1/data.pb.go +++ b/plugin/pluginapi/v1/data.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/data.proto diff --git a/plugin/pluginapi/v1/dataspec.pb.go b/plugin/pluginapi/v1/dataspec.pb.go index cc6265bb..9d2c52c4 100644 --- a/plugin/pluginapi/v1/dataspec.pb.go +++ b/plugin/pluginapi/v1/dataspec.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/dataspec.proto diff --git a/plugin/pluginapi/v1/hcl.pb.go b/plugin/pluginapi/v1/hcl.pb.go index 7f6c1c2e..e98a5032 100644 --- a/plugin/pluginapi/v1/hcl.pb.go +++ b/plugin/pluginapi/v1/hcl.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/hcl.proto diff --git a/plugin/pluginapi/v1/plugin.pb.go b/plugin/pluginapi/v1/plugin.pb.go index 6fcb5e0e..fd24ca68 100644 --- a/plugin/pluginapi/v1/plugin.pb.go +++ b/plugin/pluginapi/v1/plugin.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/plugin.proto diff --git a/plugin/pluginapi/v1/schema.pb.go b/plugin/pluginapi/v1/schema.pb.go index 15306c93..efdd945a 100644 --- a/plugin/pluginapi/v1/schema.pb.go +++ b/plugin/pluginapi/v1/schema.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.1 +// protoc-gen-go v1.36.2 // protoc (unknown) // source: pluginapi/v1/schema.proto From a87e797c4473f596ee34af7f74c11bd031bb8bee Mon Sep 17 00:00:00 2001 From: traut Date: Wed, 8 Jan 2025 19:58:36 +0100 Subject: [PATCH 2/5] Making dedenting explicit --- .../atlassian/data-sources/jira_issues.md | 14 +- .../builtin/content-providers/blockquote.md | 1 + .../plugins/builtin/content-providers/code.md | 4 +- .../builtin/content-providers/frontmatter.md | 1 + .../builtin/content-providers/image.md | 4 +- .../plugins/builtin/content-providers/list.md | 4 +- .../builtin/content-providers/table.md | 2 + .../plugins/builtin/content-providers/text.md | 1 + .../builtin/content-providers/title.md | 3 + docs/plugins/builtin/content-providers/toc.md | 2 + docs/plugins/builtin/data-sources/csv.md | 7 +- docs/plugins/builtin/data-sources/http.md | 5 + docs/plugins/builtin/data-sources/json.md | 9 +- docs/plugins/builtin/data-sources/rss.md | 31 +++- docs/plugins/builtin/data-sources/txt.md | 7 +- docs/plugins/builtin/data-sources/yaml.md | 27 ++-- docs/plugins/builtin/publishers/hub.md | 2 + docs/plugins/builtin/publishers/local_file.md | 1 + .../data-sources/falcon_cspm_ioms.md | 5 +- .../data-sources/falcon_detection_details.md | 5 +- .../falcon_discover_host_details.md | 5 +- .../data-sources/falcon_intel_indicators.md | 5 +- .../data-sources/falcon_vulnerabilities.md | 5 +- .../data-sources/elastic_security_cases.md | 1 + .../elastic/data-sources/elasticsearch.md | 1 + .../github/data-sources/github_issues.md | 2 + docs/plugins/github/publishers/github_gist.md | 1 + docs/plugins/graphql/data-sources/graphql.md | 2 + .../data-sources/hackerone_reports.md | 2 + docs/plugins/iris/data-sources/iris_alerts.md | 2 + docs/plugins/iris/data-sources/iris_cases.md | 2 + .../content-providers/azure_openai_text.md | 4 + .../microsoft/data-sources/microsoft_graph.md | 3 + .../data-sources/microsoft_security.md | 3 + .../data-sources/microsoft_security_query.md | 3 + .../microsoft_sentinel_incidents.md | 6 + docs/plugins/misp/data-sources/misp_events.md | 3 + .../misp/publishers/misp_event_reports.md | 4 + .../openai/content-providers/openai_text.md | 2 + docs/plugins/opencti/data-sources/opencti.md | 2 + docs/plugins/plugins.json | 5 +- .../postgresql/data-sources/postgresql.md | 2 + docs/plugins/snyk/data-sources/snyk_issues.md | 1 + .../splunk/data-sources/splunk_search.md | 2 + docs/plugins/sqlite/data-sources/sqlite.md | 5 +- .../data-sources/terraform_state_local.md | 1 + .../data-sources/virustotal_api_usage.md | 1 + internal/builtin/content_title.go | 21 +-- internal/builtin/content_toc.go | 17 ++- internal/builtin/data_http.go | 25 ++-- internal/builtin/data_json.go | 46 +++--- internal/builtin/data_rss.go | 41 ++--- internal/builtin/data_txt.go | 39 ++--- .../data_falcon_intel_indicators.go | 4 +- internal/github/data_github_issues.go | 53 ++++--- plugin/dataspec/attr_spec.go | 141 +++++++++++------- plugin/dataspec/attr_spec_doc_comment.gotmpl | 41 +++++ plugin/dataspec/block_spec.go | 5 +- plugin/dataspec/genutil.go | 13 +- tools/docgen/main.go | 2 +- 60 files changed, 440 insertions(+), 218 deletions(-) create mode 100644 plugin/dataspec/attr_spec_doc_comment.gotmpl diff --git a/docs/plugins/atlassian/data-sources/jira_issues.md b/docs/plugins/atlassian/data-sources/jira_issues.md index 8ed5dbb3..fd62e916 100644 --- a/docs/plugins/atlassian/data-sources/jira_issues.md +++ b/docs/plugins/atlassian/data-sources/jira_issues.md @@ -44,6 +44,7 @@ config data jira_issues { # # Required string. # Must be non-empty + # # For example: domain = "some string" @@ -51,6 +52,7 @@ config data jira_issues { # # Required string. # Must be non-empty + # # For example: account_email = "some string" @@ -58,6 +60,7 @@ config data jira_issues { # # Required string. # Must be non-empty + # # For example: api_token = "some string" } @@ -73,34 +76,37 @@ data jira_issues { # # Optional string. # Must be one of: "renderedFields", "names", "schema", "changelog" + # # For example: # expand = "names" - # + # # Default value: expand = null # A list of fields to return for each issue. # # Optional list of string. + # # For example: # fields = ["*all"] - # + # # Default value: fields = null # A JQL expression. For performance reasons, this field requires a bounded query. A bounded query is a query with a search restriction. # # Optional string. + # # For example: # jql = "order by key desc" - # + # # Default value: jql = null # A list of up to 5 issue properties to include in the results. # # Optional list of string. - # Must have a length of at most 5 + # Must contain no more than 5 elements. # Default value: properties = [] diff --git a/docs/plugins/builtin/content-providers/blockquote.md b/docs/plugins/builtin/content-providers/blockquote.md index 67b0e6ba..901255f5 100644 --- a/docs/plugins/builtin/content-providers/blockquote.md +++ b/docs/plugins/builtin/content-providers/blockquote.md @@ -32,6 +32,7 @@ The content provider supports the following execution arguments: ```hcl content blockquote { # Required string. + # # For example: value = "Text to be formatted as a quote" } diff --git a/docs/plugins/builtin/content-providers/code.md b/docs/plugins/builtin/content-providers/code.md index d205587d..c88cbb97 100644 --- a/docs/plugins/builtin/content-providers/code.md +++ b/docs/plugins/builtin/content-providers/code.md @@ -32,15 +32,17 @@ The content provider supports the following execution arguments: ```hcl content code { # Required string. + # # For example: value = "Text to be formatted as a code block" # Specifiy the language for syntax highlighting # # Optional string. + # # For example: # language = "python3" - # + # # Default value: language = "" } diff --git a/docs/plugins/builtin/content-providers/frontmatter.md b/docs/plugins/builtin/content-providers/frontmatter.md index 29979b3f..cea91330 100644 --- a/docs/plugins/builtin/content-providers/frontmatter.md +++ b/docs/plugins/builtin/content-providers/frontmatter.md @@ -42,6 +42,7 @@ content frontmatter { # # Required jq queriable. # Must be non-empty + # # For example: content = { key = "arbitrary value" diff --git a/docs/plugins/builtin/content-providers/image.md b/docs/plugins/builtin/content-providers/image.md index 07b2f8f0..024d4185 100644 --- a/docs/plugins/builtin/content-providers/image.md +++ b/docs/plugins/builtin/content-providers/image.md @@ -33,13 +33,15 @@ The content provider supports the following execution arguments: content image { # Required string. # Must be non-empty + # # For example: src = "https://example.com/img.png" # Optional string. + # # For example: # alt = "Text description of the image" - # + # # Default value: alt = null } diff --git a/docs/plugins/builtin/content-providers/list.md b/docs/plugins/builtin/content-providers/list.md index 2ba101b4..f80ea7a7 100644 --- a/docs/plugins/builtin/content-providers/list.md +++ b/docs/plugins/builtin/content-providers/list.md @@ -34,9 +34,10 @@ content list { # Go template for the item of the list # # Optional string. + # # For example: # item_template = "[{{.Title}}]({{.URL}})" - # + # # Default value: item_template = "{{.}}" @@ -49,6 +50,7 @@ content list { # # Required list of jq queriable. # Must be non-empty + # # For example: items = ["First item", "Second item", "Third item"] } diff --git a/docs/plugins/builtin/content-providers/table.md b/docs/plugins/builtin/content-providers/table.md index 98aea182..38b3e3e2 100644 --- a/docs/plugins/builtin/content-providers/table.md +++ b/docs/plugins/builtin/content-providers/table.md @@ -16,6 +16,7 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "table" "content provider" >}} ## Description + Produces a table. Each cell template has access to the data context and the following variables: @@ -51,6 +52,7 @@ content table { # # Required list of object. # Must be non-empty + # # For example: columns = [{ header = "1st column header template" diff --git a/docs/plugins/builtin/content-providers/text.md b/docs/plugins/builtin/content-providers/text.md index 99257029..6db6acc3 100644 --- a/docs/plugins/builtin/content-providers/text.md +++ b/docs/plugins/builtin/content-providers/text.md @@ -34,6 +34,7 @@ content text { # A string to render. Can use go template syntax. # # Required string. + # # For example: value = "Hello world!" } diff --git a/docs/plugins/builtin/content-providers/title.md b/docs/plugins/builtin/content-providers/title.md index c24f6e99..c1e62a1c 100644 --- a/docs/plugins/builtin/content-providers/title.md +++ b/docs/plugins/builtin/content-providers/title.md @@ -16,11 +16,13 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "title" "content provider" >}} ## Description + Produces a title. The title size after calculations must be in an interval [0; 5] inclusive, where 0 corresponds to the largest size (`

`) and 5 corresponds to (`

`) + The content provider is built-in, which means it's a part of `fabric` binary. It's available out-of-the-box, no installation required. @@ -37,6 +39,7 @@ content title { # Title content # # Required string. + # # For example: value = "Vulnerability Report" diff --git a/docs/plugins/builtin/content-providers/toc.md b/docs/plugins/builtin/content-providers/toc.md index bbad5b49..fd5b1773 100644 --- a/docs/plugins/builtin/content-providers/toc.md +++ b/docs/plugins/builtin/content-providers/toc.md @@ -16,11 +16,13 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "toc" "content provider" >}} ## Description + Produces table of contents. Inspects the rendered document for headers of a certain size and creates a linked table of contents + The content provider is built-in, which means it's a part of `fabric` binary. It's available out-of-the-box, no installation required. diff --git a/docs/plugins/builtin/data-sources/csv.md b/docs/plugins/builtin/data-sources/csv.md index ea7518d2..b4d88794 100644 --- a/docs/plugins/builtin/data-sources/csv.md +++ b/docs/plugins/builtin/data-sources/csv.md @@ -16,6 +16,7 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "csv" "data source" >}} ## Description + Loads CSV files with the names that match provided `glob` pattern or a single file from a provided path. Either `glob` or `path` argument must be set. @@ -89,18 +90,20 @@ data csv { # A glob pattern to select CSV files to read # # Optional string. + # # For example: # glob = "path/to/file*.csv" - # + # # Default value: glob = null # A file path to a CSV file to read # # Optional string. + # # For example: # path = "path/to/file.csv" - # + # # Default value: path = null } diff --git a/docs/plugins/builtin/data-sources/http.md b/docs/plugins/builtin/data-sources/http.md index 77ea8d4c..e687aa8f 100644 --- a/docs/plugins/builtin/data-sources/http.md +++ b/docs/plugins/builtin/data-sources/http.md @@ -16,6 +16,7 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "http" "data source" >}} ## Description + Loads data from a URL. At the moment, the data source accepts only responses with UTF-8 charset and parses only responses @@ -25,6 +26,7 @@ If MIME type of the response is `text/csv` or `application/json`, the response content will be parsed and returned as a JSON structure (similar to the behaviour of CSV and JSON data sources). Otherwise, the response content will be returned as text + The data source is built-in, which means it's a part of `fabric` binary. It's available out-of-the-box, no installation required. ## Configuration @@ -42,12 +44,14 @@ data http { # Optional basic_auth { # Required string. + # # For example: username = "user@example.com" # Note: avoid storing credentials in the templates. Use environment variables instead. # # Required string. + # # For example: password = "passwd" } @@ -57,6 +61,7 @@ data http { # # Required string. # Must be non-empty + # # For example: url = "https://example.localhost/file.json" diff --git a/docs/plugins/builtin/data-sources/json.md b/docs/plugins/builtin/data-sources/json.md index f7ebdb44..0927c0eb 100644 --- a/docs/plugins/builtin/data-sources/json.md +++ b/docs/plugins/builtin/data-sources/json.md @@ -16,6 +16,7 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "json" "data source" >}} ## Description + Loads JSON files with the names that match provided `glob` pattern or a single file from provided `path`value. Either `glob` or `path` argument must be set. @@ -30,7 +31,7 @@ When `glob` argument is specified, the data source returns a list of dicts that "file_name": "file-a.json", "content": { "foo": "bar" - } + } }, { "file_path": "path/file-b.json", @@ -57,18 +58,20 @@ data json { # A glob pattern to select JSON files to read # # Optional string. + # # For example: # glob = "path/to/file*.json" - # + # # Default value: glob = null # A file path to a JSON file to read # # Optional string. + # # For example: # path = "path/to/file.json" - # + # # Default value: path = null } diff --git a/docs/plugins/builtin/data-sources/rss.md b/docs/plugins/builtin/data-sources/rss.md index 86c9bd16..e6885bf0 100644 --- a/docs/plugins/builtin/data-sources/rss.md +++ b/docs/plugins/builtin/data-sources/rss.md @@ -16,10 +16,12 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "rss" "data source" >}} ## Description + Fetches RSS / Atom / JSON feed from a provided URL. The full content of the items can be fetched and added to the feed. The data source supports basic authentication. + The data source is built-in, which means it's a part of `fabric` binary. It's available out-of-the-box, no installation required. ## Configuration @@ -37,18 +39,21 @@ data rss { # Optional basic_auth { # Required string. + # # For example: username = "user@example.com" # Note: avoid storing credentials in the templates. Use environment variables instead. # # Required string. + # # For example: password = "passwd" } # Required string. + # # For example: url = "https://www.elastic.co/security-labs/rss/feed.xml" @@ -69,19 +74,31 @@ data rss { # # Optional number. # Must be >= 0 + # + # For example: + # max_items_to_fill = 10 + # + # Default value: + max_items_to_fill = 10 + + # Return only items published after a specified timestamp. The timestamp format is "%Y-%m-%dT%H:%M:%S%Z". + # + # Optional string. + # # For example: - # fill_in_max_items = false - # + # items_after = "2024-12-23T00:00:00Z" + # # Default value: - fill_in_max_items = 10 + items_after = null - # Return only items after a specified date time, in the format "%Y-%m-%dT%H:%M:%S%Z". + # Return only items published before a specified timestamp. The timestamp format is "%Y-%m-%dT%H:%M:%S%Z". # # Optional string. + # # For example: - # only_items_after_time = "2024-12-23T00:00:00Z" - # + # items_before = "2024-12-23T00:00:00Z" + # # Default value: - only_items_after_time = null + items_before = null } ``` \ No newline at end of file diff --git a/docs/plugins/builtin/data-sources/txt.md b/docs/plugins/builtin/data-sources/txt.md index 542c0c3f..705405a1 100644 --- a/docs/plugins/builtin/data-sources/txt.md +++ b/docs/plugins/builtin/data-sources/txt.md @@ -16,6 +16,7 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "txt" "data source" >}} ## Description + Loads TXT files with the names that match provided `glob` pattern or a single file from a provided path. Either `glob` or `path` argument must be set. @@ -52,18 +53,20 @@ data txt { # A glob pattern to select TXT files to read # # Optional string. + # # For example: # glob = "path/to/file*.txt" - # + # # Default value: glob = null # A file path to a TXT file to read # # Optional string. + # # For example: # path = "path/to/file.txt" - # + # # Default value: path = null } diff --git a/docs/plugins/builtin/data-sources/yaml.md b/docs/plugins/builtin/data-sources/yaml.md index 69058566..2c2ffa48 100644 --- a/docs/plugins/builtin/data-sources/yaml.md +++ b/docs/plugins/builtin/data-sources/yaml.md @@ -16,6 +16,7 @@ type: docs {{< plugin-resource-header "blackstork/builtin" "builtin" "v0.4.2" "yaml" "data source" >}} ## Description + Loads YAML files with the names that match provided `glob` pattern or a single file from provided `path`value. Either `glob` or `path` argument must be set. @@ -26,18 +27,18 @@ When `glob` argument is specified, the data source returns a list of dicts that ```json [ { - "file_path": "path/file-a.yaml", - "file_name": "file-a.yaml", - "content": { - "foo": "bar" - } + "file_path": "path/file-a.yaml", + "file_name": "file-a.yaml", + "content": { + "foo": "bar" + } }, { - "file_path": "path/file-b.yaml", - "file_name": "file-b.yaml", - "content": [ - {"x": "y"} - ] + "file_path": "path/file-b.yaml", + "file_name": "file-b.yaml", + "content": [ + {"x": "y"} + ] } ] ``` @@ -57,18 +58,20 @@ data yaml { # A glob pattern to select YAML files to read # # Optional string. + # # For example: # glob = "path/to/file*.yaml" - # + # # Default value: glob = null # A file path to a YAML file to read # # Optional string. + # # For example: # path = "path/to/file.yaml" - # + # # Default value: path = null } diff --git a/docs/plugins/builtin/publishers/hub.md b/docs/plugins/builtin/publishers/hub.md index 8aa12588..63ed2290 100644 --- a/docs/plugins/builtin/publishers/hub.md +++ b/docs/plugins/builtin/publishers/hub.md @@ -36,6 +36,7 @@ config publish hub { # # Required string. # Must be non-empty + # # For example: api_url = "some string" @@ -43,6 +44,7 @@ config publish hub { # # Required string. # Must be non-empty + # # For example: api_token = "some string" } diff --git a/docs/plugins/builtin/publishers/local_file.md b/docs/plugins/builtin/publishers/local_file.md index 07914910..2bd3b8b5 100644 --- a/docs/plugins/builtin/publishers/local_file.md +++ b/docs/plugins/builtin/publishers/local_file.md @@ -43,6 +43,7 @@ publish local_file { # Path to the file # # Required string. + # # For example: path = "dist/output.md" } diff --git a/docs/plugins/crowdstrike/data-sources/falcon_cspm_ioms.md b/docs/plugins/crowdstrike/data-sources/falcon_cspm_ioms.md index bafe03ba..27624a9c 100644 --- a/docs/plugins/crowdstrike/data-sources/falcon_cspm_ioms.md +++ b/docs/plugins/crowdstrike/data-sources/falcon_cspm_ioms.md @@ -44,6 +44,7 @@ config data falcon_cspm_ioms { # # Required string. # Must be non-empty + # # For example: client_id = "some string" @@ -51,6 +52,7 @@ config data falcon_cspm_ioms { # # Required string. # Must be non-empty + # # For example: client_secret = "some string" @@ -64,9 +66,10 @@ config data falcon_cspm_ioms { # # Optional string. # Must be one of: "autodiscover", "us-1", "us-2", "eu-1", "us-gov-1", "gov1" + # # For example: # client_cloud = "us-1" - # + # # Default value: client_cloud = null } diff --git a/docs/plugins/crowdstrike/data-sources/falcon_detection_details.md b/docs/plugins/crowdstrike/data-sources/falcon_detection_details.md index 42f53115..92ad4b50 100644 --- a/docs/plugins/crowdstrike/data-sources/falcon_detection_details.md +++ b/docs/plugins/crowdstrike/data-sources/falcon_detection_details.md @@ -44,6 +44,7 @@ config data falcon_detection_details { # # Required string. # Must be non-empty + # # For example: client_id = "some string" @@ -51,6 +52,7 @@ config data falcon_detection_details { # # Required string. # Must be non-empty + # # For example: client_secret = "some string" @@ -64,9 +66,10 @@ config data falcon_detection_details { # # Optional string. # Must be one of: "autodiscover", "us-1", "us-2", "eu-1", "us-gov-1", "gov1" + # # For example: # client_cloud = "us-1" - # + # # Default value: client_cloud = null } diff --git a/docs/plugins/crowdstrike/data-sources/falcon_discover_host_details.md b/docs/plugins/crowdstrike/data-sources/falcon_discover_host_details.md index 6081315c..a17ab30b 100644 --- a/docs/plugins/crowdstrike/data-sources/falcon_discover_host_details.md +++ b/docs/plugins/crowdstrike/data-sources/falcon_discover_host_details.md @@ -44,6 +44,7 @@ config data falcon_discover_host_details { # # Required string. # Must be non-empty + # # For example: client_id = "some string" @@ -51,6 +52,7 @@ config data falcon_discover_host_details { # # Required string. # Must be non-empty + # # For example: client_secret = "some string" @@ -64,9 +66,10 @@ config data falcon_discover_host_details { # # Optional string. # Must be one of: "autodiscover", "us-1", "us-2", "eu-1", "us-gov-1", "gov1" + # # For example: # client_cloud = "us-1" - # + # # Default value: client_cloud = null } diff --git a/docs/plugins/crowdstrike/data-sources/falcon_intel_indicators.md b/docs/plugins/crowdstrike/data-sources/falcon_intel_indicators.md index 2b03eced..5761caa7 100644 --- a/docs/plugins/crowdstrike/data-sources/falcon_intel_indicators.md +++ b/docs/plugins/crowdstrike/data-sources/falcon_intel_indicators.md @@ -44,6 +44,7 @@ config data falcon_intel_indicators { # # Required string. # Must be non-empty + # # For example: client_id = "some string" @@ -51,6 +52,7 @@ config data falcon_intel_indicators { # # Required string. # Must be non-empty + # # For example: client_secret = "some string" @@ -64,9 +66,10 @@ config data falcon_intel_indicators { # # Optional string. # Must be one of: "autodiscover", "us-1", "us-2", "eu-1", "us-gov-1", "gov1" + # # For example: # client_cloud = "us-1" - # + # # Default value: client_cloud = null } diff --git a/docs/plugins/crowdstrike/data-sources/falcon_vulnerabilities.md b/docs/plugins/crowdstrike/data-sources/falcon_vulnerabilities.md index cbeabbca..d0a0ed9d 100644 --- a/docs/plugins/crowdstrike/data-sources/falcon_vulnerabilities.md +++ b/docs/plugins/crowdstrike/data-sources/falcon_vulnerabilities.md @@ -44,6 +44,7 @@ config data falcon_vulnerabilities { # # Required string. # Must be non-empty + # # For example: client_id = "some string" @@ -51,6 +52,7 @@ config data falcon_vulnerabilities { # # Required string. # Must be non-empty + # # For example: client_secret = "some string" @@ -64,9 +66,10 @@ config data falcon_vulnerabilities { # # Optional string. # Must be one of: "autodiscover", "us-1", "us-2", "eu-1", "us-gov-1", "gov1" + # # For example: # client_cloud = "us-1" - # + # # Default value: client_cloud = null } diff --git a/docs/plugins/elastic/data-sources/elastic_security_cases.md b/docs/plugins/elastic/data-sources/elastic_security_cases.md index 5bfe8a51..51afc29f 100644 --- a/docs/plugins/elastic/data-sources/elastic_security_cases.md +++ b/docs/plugins/elastic/data-sources/elastic_security_cases.md @@ -38,6 +38,7 @@ The data source supports the following configuration arguments: ```hcl config data elastic_security_cases { # Required string. + # # For example: kibana_endpoint_url = "some string" diff --git a/docs/plugins/elastic/data-sources/elasticsearch.md b/docs/plugins/elastic/data-sources/elasticsearch.md index 13603545..7d688da2 100644 --- a/docs/plugins/elastic/data-sources/elasticsearch.md +++ b/docs/plugins/elastic/data-sources/elasticsearch.md @@ -78,6 +78,7 @@ The data source supports the following execution arguments: ```hcl data elasticsearch { # Required string. + # # For example: index = "some string" diff --git a/docs/plugins/github/data-sources/github_issues.md b/docs/plugins/github/data-sources/github_issues.md index 325a1c8b..50baa271 100644 --- a/docs/plugins/github/data-sources/github_issues.md +++ b/docs/plugins/github/data-sources/github_issues.md @@ -41,6 +41,7 @@ config data github_issues { # # Required string. # Must be non-empty + # # For example: github_token = "some string" } @@ -56,6 +57,7 @@ data github_issues { # # Required string. # Must be non-empty + # # For example: repository = "blackstork-io/fabric" diff --git a/docs/plugins/github/publishers/github_gist.md b/docs/plugins/github/publishers/github_gist.md index 71c3ae48..b504e042 100644 --- a/docs/plugins/github/publishers/github_gist.md +++ b/docs/plugins/github/publishers/github_gist.md @@ -48,6 +48,7 @@ The publisher supports the following configuration arguments: ```hcl config publish github_gist { # Required string. + # # For example: github_token = "some string" } diff --git a/docs/plugins/graphql/data-sources/graphql.md b/docs/plugins/graphql/data-sources/graphql.md index bbdebc93..16acc5e3 100644 --- a/docs/plugins/graphql/data-sources/graphql.md +++ b/docs/plugins/graphql/data-sources/graphql.md @@ -38,6 +38,7 @@ The data source supports the following configuration arguments: ```hcl config data graphql { # Required string. + # # For example: url = "some string" @@ -54,6 +55,7 @@ The data source supports the following execution arguments: ```hcl data graphql { # Required string. + # # For example: query = "some string" } diff --git a/docs/plugins/hackerone/data-sources/hackerone_reports.md b/docs/plugins/hackerone/data-sources/hackerone_reports.md index 1172a8d5..05e052d8 100644 --- a/docs/plugins/hackerone/data-sources/hackerone_reports.md +++ b/docs/plugins/hackerone/data-sources/hackerone_reports.md @@ -38,10 +38,12 @@ The data source supports the following configuration arguments: ```hcl config data hackerone_reports { # Required string. + # # For example: api_username = "some string" # Required string. + # # For example: api_token = "some string" } diff --git a/docs/plugins/iris/data-sources/iris_alerts.md b/docs/plugins/iris/data-sources/iris_alerts.md index 197e4464..d9954b36 100644 --- a/docs/plugins/iris/data-sources/iris_alerts.md +++ b/docs/plugins/iris/data-sources/iris_alerts.md @@ -44,6 +44,7 @@ config data iris_alerts { # # Required string. # Must be non-empty + # # For example: api_url = "some string" @@ -51,6 +52,7 @@ config data iris_alerts { # # Required string. # Must be non-empty + # # For example: api_key = "some string" diff --git a/docs/plugins/iris/data-sources/iris_cases.md b/docs/plugins/iris/data-sources/iris_cases.md index 59fb1a57..b8cfee63 100644 --- a/docs/plugins/iris/data-sources/iris_cases.md +++ b/docs/plugins/iris/data-sources/iris_cases.md @@ -44,6 +44,7 @@ config data iris_cases { # # Required string. # Must be non-empty + # # For example: api_url = "some string" @@ -51,6 +52,7 @@ config data iris_cases { # # Required string. # Must be non-empty + # # For example: api_key = "some string" diff --git a/docs/plugins/microsoft/content-providers/azure_openai_text.md b/docs/plugins/microsoft/content-providers/azure_openai_text.md index 34c7823f..c237c019 100644 --- a/docs/plugins/microsoft/content-providers/azure_openai_text.md +++ b/docs/plugins/microsoft/content-providers/azure_openai_text.md @@ -39,14 +39,17 @@ The content provider supports the following configuration arguments: ```hcl config content azure_openai_text { # Required string. + # # For example: api_key = "some string" # Required string. + # # For example: resource_endpoint = "some string" # Required string. + # # For example: deployment_name = "some string" @@ -63,6 +66,7 @@ The content provider supports the following execution arguments: ```hcl content azure_openai_text { # Required string. + # # For example: prompt = "Summarize the following text: {{.vars.text_to_summarize}}" diff --git a/docs/plugins/microsoft/data-sources/microsoft_graph.md b/docs/plugins/microsoft/data-sources/microsoft_graph.md index 39a5c1fb..66f9622b 100644 --- a/docs/plugins/microsoft/data-sources/microsoft_graph.md +++ b/docs/plugins/microsoft/data-sources/microsoft_graph.md @@ -43,6 +43,7 @@ config data microsoft_graph { # The Azure client ID # # Required string. + # # For example: client_id = "some string" @@ -55,6 +56,7 @@ config data microsoft_graph { # The Azure tenant ID # # Required string. + # # For example: tenant_id = "some string" @@ -93,6 +95,7 @@ data microsoft_graph { # The endpoint to query # # Required string. + # # For example: endpoint = "/users" diff --git a/docs/plugins/microsoft/data-sources/microsoft_security.md b/docs/plugins/microsoft/data-sources/microsoft_security.md index 8e95945d..558d0fc3 100644 --- a/docs/plugins/microsoft/data-sources/microsoft_security.md +++ b/docs/plugins/microsoft/data-sources/microsoft_security.md @@ -43,6 +43,7 @@ config data microsoft_security { # The Azure client ID # # Required string. + # # For example: client_id = "some string" @@ -55,6 +56,7 @@ config data microsoft_security { # The Azure tenant ID # # Required string. + # # For example: tenant_id = "some string" @@ -87,6 +89,7 @@ data microsoft_security { # API endpoint to query # # Required string. + # # For example: endpoint = "/users" diff --git a/docs/plugins/microsoft/data-sources/microsoft_security_query.md b/docs/plugins/microsoft/data-sources/microsoft_security_query.md index 1d8bf4fb..59bb4ff5 100644 --- a/docs/plugins/microsoft/data-sources/microsoft_security_query.md +++ b/docs/plugins/microsoft/data-sources/microsoft_security_query.md @@ -43,6 +43,7 @@ config data microsoft_security_query { # The Azure client ID # # Required string. + # # For example: client_id = "some string" @@ -55,6 +56,7 @@ config data microsoft_security_query { # The Azure tenant ID # # Required string. + # # For example: tenant_id = "some string" @@ -87,6 +89,7 @@ data microsoft_security_query { # Advanced hunting query to run # # Required string. + # # For example: query = "DeviceRegistryEvents | where Timestamp >= ago(30d) | where isnotempty(RegistryKey) and isnotempty(RegistryValueName) | limit 5" } diff --git a/docs/plugins/microsoft/data-sources/microsoft_sentinel_incidents.md b/docs/plugins/microsoft/data-sources/microsoft_sentinel_incidents.md index ee9adced..eacd3bb3 100644 --- a/docs/plugins/microsoft/data-sources/microsoft_sentinel_incidents.md +++ b/docs/plugins/microsoft/data-sources/microsoft_sentinel_incidents.md @@ -43,36 +43,42 @@ config data microsoft_sentinel_incidents { # The Azure client ID # # Required string. + # # For example: client_id = "some string" # The Azure client secret # # Required string. + # # For example: client_secret = "some string" # The Azure tenant ID # # Required string. + # # For example: tenant_id = "some string" # The Azure subscription ID # # Required string. + # # For example: subscription_id = "some string" # The Azure resource group name # # Required string. + # # For example: resource_group_name = "some string" # The Azure workspace name # # Required string. + # # For example: workspace_name = "some string" } diff --git a/docs/plugins/misp/data-sources/misp_events.md b/docs/plugins/misp/data-sources/misp_events.md index 88cc70be..2c4a2662 100644 --- a/docs/plugins/misp/data-sources/misp_events.md +++ b/docs/plugins/misp/data-sources/misp_events.md @@ -44,6 +44,7 @@ config data misp_events { # # Required string. # Must be non-empty + # # For example: api_key = "some string" @@ -51,6 +52,7 @@ config data misp_events { # # Required string. # Must be non-empty + # # For example: base_url = "some string" @@ -69,6 +71,7 @@ The data source supports the following execution arguments: ```hcl data misp_events { # Required string. + # # For example: value = "some string" diff --git a/docs/plugins/misp/publishers/misp_event_reports.md b/docs/plugins/misp/publishers/misp_event_reports.md index b3dbb559..b0ea01b7 100644 --- a/docs/plugins/misp/publishers/misp_event_reports.md +++ b/docs/plugins/misp/publishers/misp_event_reports.md @@ -50,6 +50,7 @@ config publish misp_event_reports { # # Required string. # Must be non-empty + # # For example: api_key = "some string" @@ -57,6 +58,7 @@ config publish misp_event_reports { # # Required string. # Must be non-empty + # # For example: base_url = "some string" @@ -79,11 +81,13 @@ The publisher supports the following execution arguments: publish misp_event_reports { # Required string. # Must be non-empty + # # For example: event_id = "some string" # Required string. # Must be non-empty + # # For example: name = "some string" diff --git a/docs/plugins/openai/content-providers/openai_text.md b/docs/plugins/openai/content-providers/openai_text.md index 4f3ac336..6e3fedc6 100644 --- a/docs/plugins/openai/content-providers/openai_text.md +++ b/docs/plugins/openai/content-providers/openai_text.md @@ -43,6 +43,7 @@ config content openai_text { system_prompt = null # Required string. + # # For example: api_key = "some string" @@ -59,6 +60,7 @@ The content provider supports the following execution arguments: ```hcl content openai_text { # Required string. + # # For example: prompt = "Summarize the following text: {{.vars.text_to_summarize}}" diff --git a/docs/plugins/opencti/data-sources/opencti.md b/docs/plugins/opencti/data-sources/opencti.md index d966f875..268b7eaa 100644 --- a/docs/plugins/opencti/data-sources/opencti.md +++ b/docs/plugins/opencti/data-sources/opencti.md @@ -38,6 +38,7 @@ The data source supports the following configuration arguments: ```hcl config data opencti { # Required string. + # # For example: graphql_url = "some string" @@ -54,6 +55,7 @@ The data source supports the following execution arguments: ```hcl data opencti { # Required string. + # # For example: graphql_query = "some string" } diff --git a/docs/plugins/plugins.json b/docs/plugins/plugins.json index d9f06e77..6928953a 100644 --- a/docs/plugins/plugins.json +++ b/docs/plugins/plugins.json @@ -121,8 +121,9 @@ "type": "data-source", "arguments": [ "fill_in_content", - "fill_in_max_items", - "only_items_after_time", + "items_after", + "items_before", + "max_items_to_fill", "url", "use_browser_user_agent" ] diff --git a/docs/plugins/postgresql/data-sources/postgresql.md b/docs/plugins/postgresql/data-sources/postgresql.md index 044f17dd..f3424d51 100644 --- a/docs/plugins/postgresql/data-sources/postgresql.md +++ b/docs/plugins/postgresql/data-sources/postgresql.md @@ -39,6 +39,7 @@ The data source supports the following configuration arguments: config data postgresql { # Required string. # Must be non-empty + # # For example: database_url = "some string" } @@ -52,6 +53,7 @@ The data source supports the following execution arguments: data postgresql { # Required string. # Must be non-empty + # # For example: sql_query = "some string" diff --git a/docs/plugins/snyk/data-sources/snyk_issues.md b/docs/plugins/snyk/data-sources/snyk_issues.md index ef7e4a68..24eae2ea 100644 --- a/docs/plugins/snyk/data-sources/snyk_issues.md +++ b/docs/plugins/snyk/data-sources/snyk_issues.md @@ -44,6 +44,7 @@ config data snyk_issues { # # Required string. # Must be non-empty + # # For example: api_key = "some string" } diff --git a/docs/plugins/splunk/data-sources/splunk_search.md b/docs/plugins/splunk/data-sources/splunk_search.md index d5bd33b6..7c070350 100644 --- a/docs/plugins/splunk/data-sources/splunk_search.md +++ b/docs/plugins/splunk/data-sources/splunk_search.md @@ -38,6 +38,7 @@ The data source supports the following configuration arguments: ```hcl config data splunk_search { # Required string. + # # For example: auth_token = "some string" @@ -58,6 +59,7 @@ The data source supports the following execution arguments: ```hcl data splunk_search { # Required string. + # # For example: search_query = "some string" diff --git a/docs/plugins/sqlite/data-sources/sqlite.md b/docs/plugins/sqlite/data-sources/sqlite.md index 3fa76986..496e68f1 100644 --- a/docs/plugins/sqlite/data-sources/sqlite.md +++ b/docs/plugins/sqlite/data-sources/sqlite.md @@ -39,6 +39,7 @@ The data source supports the following configuration arguments: config data sqlite { # Required string. # Must be non-empty + # # For example: database_uri = "some string" } @@ -53,15 +54,17 @@ data sqlite { # SQL query to execute # # Required string. + # # For example: sql_query = "some string" # A tuple (or list) of strings, numbers, or booleans to be used as arguments in the SQL query # # Optional any type. + # # For example: # sql_args = ["example argument", 2, false] - # + # # Default value: sql_args = null } diff --git a/docs/plugins/terraform/data-sources/terraform_state_local.md b/docs/plugins/terraform/data-sources/terraform_state_local.md index e84e7bbb..ee106112 100644 --- a/docs/plugins/terraform/data-sources/terraform_state_local.md +++ b/docs/plugins/terraform/data-sources/terraform_state_local.md @@ -42,6 +42,7 @@ The data source supports the following execution arguments: ```hcl data terraform_state_local { # Required string. + # # For example: path = "some string" } diff --git a/docs/plugins/virustotal/data-sources/virustotal_api_usage.md b/docs/plugins/virustotal/data-sources/virustotal_api_usage.md index 6081c1ce..0fb2ce19 100644 --- a/docs/plugins/virustotal/data-sources/virustotal_api_usage.md +++ b/docs/plugins/virustotal/data-sources/virustotal_api_usage.md @@ -39,6 +39,7 @@ The data source supports the following configuration arguments: config data virustotal_api_usage { # Required string. # Must be non-empty + # # For example: api_key = "some string" } diff --git a/internal/builtin/content_title.go b/internal/builtin/content_title.go index 3560718d..c335a10d 100644 --- a/internal/builtin/content_title.go +++ b/internal/builtin/content_title.go @@ -9,6 +9,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/dataspec/constraint" @@ -38,29 +39,29 @@ func makeTitleContentProvider() *plugin.ContentProvider { Type: cty.Number, Constraints: constraint.Integer, DefaultVal: cty.NullVal(cty.Number), - Doc: ` - Sets the absolute size of the title. - If ` + "`null`" + ` – absoulute title size is determined from the document structure - `, + Doc: utils.Dedent(` + Sets the absolute size of the title. + If ` + "`null`" + ` – absoulute title size is determined from the document structure + `), }, { Name: "relative_size", Type: cty.Number, Constraints: constraint.Integer, DefaultVal: cty.NumberIntVal(0), - Doc: ` - Adjusts the absolute size of the title. - The value (which may be negative) is added to the ` + "`absolute_size`" + ` to produce the final title size - `, + Doc: utils.Dedent(` + Adjusts the absolute size of the title. + The value (which may be negative) is added to the ` + "`absolute_size`" + ` to produce the final title size + `), }, }, }, - Doc: ` + Doc: utils.Dedent(` Produces a title. The title size after calculations must be in an interval [0; 5] inclusive, where 0 corresponds to the largest size (` + "`

`" + `) and 5 corresponds to (` + "`

`" + `) - `, + `), } } diff --git a/internal/builtin/content_toc.go b/internal/builtin/content_toc.go index d2c687df..e3222ee3 100644 --- a/internal/builtin/content_toc.go +++ b/internal/builtin/content_toc.go @@ -10,6 +10,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/dataspec/constraint" @@ -47,12 +48,12 @@ func makeTOCContentProvider() *plugin.ContentProvider { { Name: "scope", Type: cty.String, - Doc: ` - Scope of the headers to evaluate. - "document" – look for headers in the whole document - "section" – look for headers only in the current section - "auto" – behaves as "section" if the "toc" block is inside of a section; else – behaves as "document" - `, + Doc: utils.Dedent(` + Scope of the headers to evaluate. + "document" – look for headers in the whole document + "section" – look for headers only in the current section + "auto" – behaves as "section" if the "toc" block is inside of a section; else – behaves as "document" + `), OneOf: []cty.Value{ cty.StringVal("document"), cty.StringVal("section"), @@ -64,12 +65,12 @@ func makeTOCContentProvider() *plugin.ContentProvider { }, InvocationOrder: plugin.InvocationOrderEnd, ContentFunc: genTOC, - Doc: ` + Doc: utils.Dedent(` Produces table of contents. Inspects the rendered document for headers of a certain size and creates a linked table of contents - `, + `), } } diff --git a/internal/builtin/data_http.go b/internal/builtin/data_http.go index 2f865af9..3d7e8ea1 100644 --- a/internal/builtin/data_http.go +++ b/internal/builtin/data_http.go @@ -19,6 +19,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/internal/builtin/utils" + u "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/pkg/diagnostics" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" @@ -35,9 +36,9 @@ func makeHTTPDataSource(version string) *plugin.DataSource { Header: dataspec.HeadersSpec{ dataspec.ExactMatcher{"basic_auth"}, }, - Doc: ` + Doc: u.Dedent(` Basic authentication credentials to be used for HTTP request. - `, + `), Attrs: []*dataspec.AttrSpec{ { Name: "username", @@ -49,9 +50,9 @@ func makeHTTPDataSource(version string) *plugin.DataSource { Name: "password", Type: cty.String, ExampleVal: cty.StringVal("passwd"), - Doc: ` + Doc: u.Dedent(` Note: avoid storing credentials in the templates. Use environment variables instead. - `, + `), Constraints: constraint.RequiredNonNull, }, }, @@ -102,16 +103,16 @@ func makeHTTPDataSource(version string) *plugin.DataSource { }, }, }, - Doc: ` - Loads data from a URL. + Doc: u.Dedent(` + Loads data from a URL. - At the moment, the data source accepts only responses with UTF-8 charset and parses only responses - with MIME types ` + "`text/csv`" + ` or ` + "`application/json`" + `. + At the moment, the data source accepts only responses with UTF-8 charset and parses only responses + with MIME types ` + "`text/csv`" + ` or ` + "`application/json`" + `. - If MIME type of the response is ` + "`text/csv`" + ` or ` + "`application/json`" + `, the response - content will be parsed and returned as a JSON structure (similar to the behaviour of CSV and JSON data - sources). Otherwise, the response content will be returned as text - `, + If MIME type of the response is ` + "`text/csv`" + ` or ` + "`application/json`" + `, the response + content will be parsed and returned as a JSON structure (similar to the behaviour of CSV and JSON data + sources). Otherwise, the response content will be returned as text + `), } } diff --git a/internal/builtin/data_json.go b/internal/builtin/data_json.go index 2a53481a..8acbd36f 100644 --- a/internal/builtin/data_json.go +++ b/internal/builtin/data_json.go @@ -12,6 +12,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/plugindata" @@ -36,32 +37,33 @@ func makeJSONDataSource() *plugin.DataSource { }, }, }, - Doc: ` - Loads JSON files with the names that match provided ` + "`glob`" + ` pattern or a single file from provided ` + "`path`" + `value. + Doc: utils.Dedent(` + Loads JSON files with the names that match provided ` + "`glob`" + ` pattern or a single file from provided ` + "`path`" + `value. - Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set. + Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set. - When ` + "`path`" + ` argument is specified, the data source returns only the content of a file. - When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example: + When ` + "`path`" + ` argument is specified, the data source returns only the content of a file. + When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example: - ` + "```json" + ` - [ - { - "file_path": "path/file-a.json", - "file_name": "file-a.json", - "content": { - "foo": "bar" - } - }, - { - "file_path": "path/file-b.json", - "file_name": "file-b.json", - "content": [ - {"x": "y"} + ` + "```json" + ` + [ + { + "file_path": "path/file-a.json", + "file_name": "file-a.json", + "content": { + "foo": "bar" + } + }, + { + "file_path": "path/file-b.json", + "file_name": "file-b.json", + "content": [ + {"x": "y"} + ] + } ] - } - ] - ` + "```", + ` + "```", + ), } } diff --git a/internal/builtin/data_rss.go b/internal/builtin/data_rss.go index f310af08..1b141c23 100644 --- a/internal/builtin/data_rss.go +++ b/internal/builtin/data_rss.go @@ -67,46 +67,49 @@ func makeRSSDataSource() *plugin.DataSource { Type: cty.Bool, DefaultVal: cty.BoolVal(false), Constraints: constraint.NonNull, - Doc: ` + Doc: utils.Dedent(` If the full content should be added when it's not present in the feed items. - `, + `), }, { Name: "use_browser_user_agent", Type: cty.Bool, DefaultVal: cty.BoolVal(false), Constraints: constraint.NonNull, - Doc: fmt.Sprintf(` - If the data source should pretend to be a browser while fetching the feed and the feed items. - If set to "false", the default user-agent value "%s" will be used. - `, defaultUserAgent), + Doc: utils.Dedent( + fmt.Sprintf(` + If the data source should pretend to be a browser while fetching the feed and the feed items. + If set to "false", the default user-agent value "%s" will be used. + `, + defaultUserAgent), + ), }, { Name: "max_items_to_fill", Type: cty.Number, - ExampleVal: cty.BoolVal(false), + ExampleVal: cty.NumberIntVal(10), Constraints: constraint.NonNull, MinInclusive: cty.NumberIntVal(0), DefaultVal: cty.NumberIntVal(10), - Doc: ` + Doc: utils.Dedent(` Maximum number of items to fill the content in per feed. - `, + `), }, { Name: "items_after", Type: cty.String, ExampleVal: cty.StringVal("2024-12-23T00:00:00Z"), - Doc: ` + Doc: utils.Dedent(` Return only items published after a specified timestamp. The timestamp format is "%Y-%m-%dT%H:%M:%S%Z". - `, + `), }, { Name: "items_before", Type: cty.String, ExampleVal: cty.StringVal("2024-12-23T00:00:00Z"), - Doc: ` + Doc: utils.Dedent(` Return only items published before a specified timestamp. The timestamp format is "%Y-%m-%dT%H:%M:%S%Z". - `, + `), }, }, Blocks: []*dataspec.BlockSpec{ @@ -114,9 +117,9 @@ func makeRSSDataSource() *plugin.DataSource { Header: dataspec.HeadersSpec{ dataspec.ExactMatcher{"basic_auth"}, }, - Doc: ` + Doc: utils.Dedent(` Basic authentication credentials to be used in a HTTP request fetching RSS feed. - `, + `), Attrs: []*dataspec.AttrSpec{ { Name: "username", @@ -128,20 +131,20 @@ func makeRSSDataSource() *plugin.DataSource { Name: "password", Type: cty.String, ExampleVal: cty.StringVal("passwd"), - Doc: ` + Doc: utils.Dedent(` Note: avoid storing credentials in the templates. Use environment variables instead. - `, + `), Constraints: constraint.RequiredNonNull, }, }, }, }, }, - Doc: ` + Doc: utils.Dedent(` Fetches RSS / Atom / JSON feed from a provided URL. The full content of the items can be fetched and added to the feed. The data source supports basic authentication. - `, + `), } } diff --git a/internal/builtin/data_txt.go b/internal/builtin/data_txt.go index b4a0daf8..9a2a5775 100644 --- a/internal/builtin/data_txt.go +++ b/internal/builtin/data_txt.go @@ -11,6 +11,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/plugindata" @@ -35,27 +36,27 @@ func makeTXTDataSource() *plugin.DataSource { }, }, }, - Doc: ` - Loads TXT files with the names that match provided ` + "`glob`" + ` pattern or a single file from a provided path. + Doc: utils.Dedent(` + Loads TXT files with the names that match provided ` + "`glob`" + ` pattern or a single file from a provided path. - Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set. + Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set. - When ` + "`path`" + ` argument is specified, the data source returns only the content of a file. - When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example: - ` + "```json" + ` - [ - { - "file_path": "path/file-a.txt", - "file_name": "file-a.txt", - "content": "foobar" - }, - { - "file_path": "path/file-b.txt", - "file_name": "file-b.txt", - "content": "x\\ny\\nz" - } - ] - ` + "```", + When ` + "`path`" + ` argument is specified, the data source returns only the content of a file. + When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example: + ` + "```json" + ` + [ + { + "file_path": "path/file-a.txt", + "file_name": "file-a.txt", + "content": "foobar" + }, + { + "file_path": "path/file-b.txt", + "file_name": "file-b.txt", + "content": "x\\ny\\nz" + } + ] + ` + "```"), } } diff --git a/internal/crowdstrike/data_falcon_intel_indicators.go b/internal/crowdstrike/data_falcon_intel_indicators.go index a69c5654..ec60caf6 100644 --- a/internal/crowdstrike/data_falcon_intel_indicators.go +++ b/internal/crowdstrike/data_falcon_intel_indicators.go @@ -32,12 +32,12 @@ func makeFalconIntelIndicatorsDataSource(loader ClientLoaderFn) *plugin.DataSour { Name: "filter", Type: cty.String, - Doc: " Indicators filter expression using Falcon Query Language (FQL)", + Doc: "Indicators filter expression using Falcon Query Language (FQL)", }, { Name: "sort", Type: cty.String, - Doc: " Indicators sort expression using Falcon Query Language (FQL)", + Doc: "Indicators sort expression using Falcon Query Language (FQL)", }, }, }, diff --git a/internal/github/data_github_issues.go b/internal/github/data_github_issues.go index f2699058..d2f306ce 100644 --- a/internal/github/data_github_issues.go +++ b/internal/github/data_github_issues.go @@ -12,6 +12,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/pkg/diagnostics" + "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/dataspec/constraint" @@ -45,12 +46,13 @@ func makeGithubIssuesDataSchema(loader ClientLoaderFn) *plugin.DataSource { Name: "milestone", Type: cty.String, Constraints: constraint.TrimSpace | constraint.NonNull, - Doc: ` + Doc: utils.Dedent(` Filter issues by milestone. Possible values are: * a milestone number * "none" for issues with no milestone * "*" for issues with any milestone - * "" (empty string) performs no filtering`, + * "" (empty string) performs no filtering + `), DefaultVal: cty.StringVal(""), }, { @@ -69,32 +71,35 @@ func makeGithubIssuesDataSchema(loader ClientLoaderFn) *plugin.DataSource { Name: "assignee", Type: cty.String, Constraints: constraint.TrimSpace | constraint.NonNull, - Doc: ` - Filter issues based on their assignee. Possible values are: - * a user name - * "none" for issues that are not assigned - * "*" for issues with any assigned user - * "" (empty string) performs no filtering.`, + Doc: utils.Dedent(` + Filter issues based on their assignee. Possible values are: + * a user name + * "none" for issues that are not assigned + * "*" for issues with any assigned user + * "" (empty string) performs no filtering. + `), DefaultVal: cty.StringVal(""), }, { Name: "creator", Type: cty.String, Constraints: constraint.TrimSpace | constraint.NonNull, - Doc: ` - Filter issues based on their creator. Possible values are: - * a user name - * "" (empty string) performs no filtering.`, + Doc: utils.Dedent(` + Filter issues based on their creator. Possible values are: + * a user name + * "" (empty string) performs no filtering. + `), DefaultVal: cty.StringVal(""), }, { Name: "mentioned", Type: cty.String, Constraints: constraint.TrimSpace | constraint.NonNull, - Doc: ` - Filter issues to once where this username is mentioned. Possible values are: - * a user name - * "" (empty string) performs no filtering.`, + Doc: utils.Dedent(` + Filter issues to once where this username is mentioned. Possible values are: + * a user name + * "" (empty string) performs no filtering. + `), DefaultVal: cty.StringVal(""), }, { @@ -130,16 +135,17 @@ func makeGithubIssuesDataSchema(loader ClientLoaderFn) *plugin.DataSource { Name: "since", Type: cty.String, Constraints: constraint.Meaningful, - Doc: ` - Only show results that were last updated after the given time. - This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ.`, + Doc: utils.Dedent(` + Only show results that were last updated after the given time. + This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ. + `), }, { Name: "limit", Type: cty.Number, Constraints: constraint.Integer, MinInclusive: cty.NumberIntVal(-1), - Doc: `Limit the number of issues to return. -1 means no limit.`, + Doc: "Limit the number of issues to return. -1 means no limit.", DefaultVal: cty.NumberIntVal(-1), }, }, @@ -237,8 +243,11 @@ func parseIssuesArgs(args *dataspec.Block) (*parsedIssuesArgs, diagnostics.Diag) return nil, diagnostics.Diag{{ Severity: hcl.DiagError, Summary: "Invalid timestamp format", - Detail: fmt.Sprintf("Timestamp must be in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ, but %s", err.Error()), - Subject: &since.ValueRange, + Detail: fmt.Sprintf( + "Timestamp must be in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ, but %s", + err.Error(), + ), + Subject: &since.ValueRange, }} } parsed.opts.Since = ts diff --git a/plugin/dataspec/attr_spec.go b/plugin/dataspec/attr_spec.go index 092685a2..c2e7b0f3 100644 --- a/plugin/dataspec/attr_spec.go +++ b/plugin/dataspec/attr_spec.go @@ -1,9 +1,14 @@ package dataspec import ( + "bytes" "fmt" + "log/slog" "math/big" + "path" + "strconv" "strings" + "text/template" "unicode/utf8" "github.com/hashicorp/hcl/v2" @@ -14,6 +19,10 @@ import ( "github.com/blackstork-io/fabric/plugin/dataspec/constraint" ) +const ( + docTmplPath = "plugin/dataspec/attr_spec_doc_comment.gotmpl" +) + type AttrSpec struct { Name string Type cty.Type @@ -63,81 +72,86 @@ func formatType(buf *strings.Builder, t cty.Type) { } func (a *AttrSpec) DocComment() hclwrite.Tokens { - tokens := comment(nil, a.Doc) - if len(tokens) != 0 { - tokens = appendCommentNewLine(tokens) - } - var buf strings.Builder - if a.Constraints.Is(constraint.Required) { - buf.WriteString("Required ") - } else { - buf.WriteString("Optional ") + docTmplFilename := path.Base(docTmplPath) + tmpl, err := template.New(docTmplFilename).ParseFiles(docTmplPath) + if err != nil { + slog.Error("Error while reading an attribute doc template file", "template_file", docTmplPath, "err", err) + panic("Error while reading an attribute doc template file") } + + isRequired := a.Constraints.Is(constraint.Required) + + var attrType string if a.Constraints.Is(constraint.Integer) { - buf.WriteString("integer") + attrType = "integer" } else { + var buf strings.Builder formatType(&buf, a.Type) + attrType = buf.String() } - buf.WriteString(".\n") + var oneOf string if !a.OneOf.IsEmpty() { - buf.WriteString("Must be one of: ") - buf.WriteString(a.OneOf.String()) - buf.WriteString("\n") + oneOf = a.OneOf.String() } - min := a.computeMinInclusive() max := a.MaxInclusive - if !min.IsNull() && !max.IsNull() { - if a.Type.IsPrimitiveType() && a.Type == cty.Number { - fmt.Fprintf(&buf, "Must be between %s and %s (inclusive)\n", min.AsBigFloat().String(), max.AsBigFloat().String()) - } else { - min, _ := min.AsBigFloat().Uint64() - max, _ := max.AsBigFloat().Uint64() - if min == max { - fmt.Fprintf(&buf, "Must have a length of %d\n", min) - } else { - fmt.Fprintf(&buf, "Must have a length between %d and %d (inclusive)\n", min, max) - } + var minVal string + var maxVal string + + var minLenVal string + var maxLenVal string + + if a.Type.IsPrimitiveType() && a.Type == cty.Number { + if !min.IsNull() { + minVal = min.AsBigFloat().String() } - } else if !min.IsNull() { - if a.Type.IsPrimitiveType() && a.Type == cty.Number { - fmt.Fprintf(&buf, "Must be >= %s\n", min.AsBigFloat().String()) - } else { - min, _ := min.AsBigFloat().Uint64() - if min == 1 { - buf.WriteString("Must be non-empty\n") - } else { - fmt.Fprintf(&buf, "Must have a length of at least %d\n", min) - } + if !max.IsNull() { + maxVal = max.AsBigFloat().String() } - } else if !max.IsNull() { - if a.Type.IsPrimitiveType() && a.Type == cty.Number { - fmt.Fprintf(&buf, "Must be <= %s\n", max.AsBigFloat().String()) - } else { - max, _ := max.AsBigFloat().Uint64() - fmt.Fprintf(&buf, "Must have a length of at most %d\n", max) + } else { + if !min.IsNull() { + minLen, _ := min.AsBigFloat().Uint64() + minLenVal = strconv.FormatUint(minLen, 10) + } + if !max.IsNull() { + maxLen, _ := max.AsBigFloat().Uint64() + maxLenVal = strconv.FormatUint(maxLen, 10) } } - if a.Constraints.Is(constraint.Required) { - buf.WriteString("For example:") - } else { + + var example string + if !isRequired { if a.ExampleVal != cty.NilVal { - buf.WriteString("For example:\n") f := hclwrite.NewEmptyFile() f.Body().SetAttributeValue(a.Name, a.ExampleVal) - buf.Write(hclwrite.Format(f.Bytes())) - buf.WriteString("\n") + example = string(hclwrite.Format(f.Bytes())) } - buf.WriteString("Default value:") } - tokens = comment( - tokens, - buf.String(), - ) - return tokens + + trimmedDoc := strings.Trim(a.Doc, "\n ") + + details := map[string]interface{}{ + "Doc": trimmedDoc, + "IsRequired": isRequired, + "Type": attrType, + "OneOf": oneOf, + "MinVal": minVal, + "MaxVal": maxVal, + "MinLenVal": minLenVal, + "MaxLenVal": maxLenVal, + "Example": example, + } + + var docVal bytes.Buffer + if err := tmpl.Execute(&docVal, details); err != nil { + slog.Error("Error while rendering an attribute doc template", "err", err) + panic("Error while rendering an attribute doc template") + } + + return comment(nil, docVal.String()) } func (a *AttrSpec) WriteDoc(w *hclwrite.Body) { @@ -301,7 +315,14 @@ func (a *AttrSpec) ValidateSpec() (diags diagnostics.Diag) { diags.AddWarn(fmt.Sprintf("Missing example value for a required attribute %q", a.Name), "") } if a.DefaultVal != cty.NilVal { - diags.Add(fmt.Sprintf("Default value is specified for a required attribute %q = %s", a.Name, a.DefaultVal.GoString()), "") + diags.Add( + fmt.Sprintf( + "Default value is specified for a required attribute %q = %s", + a.Name, + a.DefaultVal.GoString(), + ), + "", + ) } } @@ -326,7 +347,15 @@ func (a *AttrSpec) ValidateSpec() (diags diagnostics.Diag) { continue } if !(v.val.Type().IsPrimitiveType() && v.val.Type() == cty.Number) { - diags.Add(fmt.Sprintf("%s specified for %q must be a number, not %s", v.name, a.Name, v.val.Type().FriendlyName()), "") + diags.Add( + fmt.Sprintf( + "%s specified for %q must be a number, not %s", + v.name, + a.Name, + v.val.Type().FriendlyName(), + ), + "", + ) skipMinMaxRelativeCheck = true continue } diff --git a/plugin/dataspec/attr_spec_doc_comment.gotmpl b/plugin/dataspec/attr_spec_doc_comment.gotmpl new file mode 100644 index 00000000..54508ca8 --- /dev/null +++ b/plugin/dataspec/attr_spec_doc_comment.gotmpl @@ -0,0 +1,41 @@ +{{- if .Doc -}} +{{ .Doc }} + +{{ end -}} +{{ if .IsRequired }}Required{{ else }}Optional{{ end }} {{ .Type }}. +{{- if .OneOf }} +Must be one of: {{ .OneOf }} +{{- end }} +{{- if and .MinVal .MaxVal }} +Must be between {{ .MinVal }} and {{ .MaxVal }} (inclusive) +{{- else if .MinVal }} +Must be >= {{ .MinVal }} +{{- else if .MaxVal }} +Must be <= {{ .MaxVal }} +{{- end }} +{{- if and .MinLenVal .MaxLenVal -}} +{{- if eq .MinLenVal .MaxLenVal }} +Must have a length of {{ .MinLenVal }} +{{- else }} +Must have a length between {{ .MinLenVal }} and {{ .MaxLenVal }} (inclusive) +{{- end -}} +{{- else if .MinLenVal -}} +{{- if eq .MinLenVal "1" }} +Must be non-empty +{{- else }} +Must contain at least {{ .MinLenVal }} elements. +{{- end -}} +{{- else if .MaxLenVal }} +Must contain no more than {{ .MaxLenVal }} elements. +{{- end -}} +{{- if .IsRequired }} + +For example: +{{- else -}} +{{- if .Example }} + +For example: +{{ .Example }} +{{- end }} +Default value: +{{- end -}} diff --git a/plugin/dataspec/block_spec.go b/plugin/dataspec/block_spec.go index 98df89d0..146ab108 100644 --- a/plugin/dataspec/block_spec.go +++ b/plugin/dataspec/block_spec.go @@ -1,6 +1,8 @@ package dataspec import ( + "strings" + "github.com/hashicorp/hcl/v2/hclwrite" "github.com/blackstork-io/fabric/pkg/diagnostics" @@ -64,7 +66,8 @@ type BlockSpec struct { } func (b *BlockSpec) WriteBlockDoc(w *hclwrite.Body) { - tokens := comment(nil, b.Doc) + trimmedDoc := strings.Trim(b.Doc, "\n ") + tokens := comment(nil, trimmedDoc) if len(tokens) != 0 { tokens = appendCommentNewLine(tokens) } diff --git a/plugin/dataspec/genutil.go b/plugin/dataspec/genutil.go index e63ec4c7..3811a365 100644 --- a/plugin/dataspec/genutil.go +++ b/plugin/dataspec/genutil.go @@ -6,8 +6,6 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/zclconf/go-cty/cty" - - "github.com/blackstork-io/fabric/pkg/utils" ) func exampleValueForType(ty cty.Type) cty.Value { @@ -57,9 +55,14 @@ func exampleValueForType(ty cty.Type) cty.Value { func comment(tokens hclwrite.Tokens, text string) hclwrite.Tokens { var sb strings.Builder - for _, line := range utils.TrimDedent(text) { - sb.WriteString("# ") - sb.WriteString(line) + lines := strings.Split(text, "\n") + for _, line := range lines { + if line != "" { + sb.WriteString("# ") + sb.WriteString(line) + } else { + sb.WriteString("#") + } sb.WriteByte('\n') tokens = append(tokens, &hclwrite.Token{ Type: hclsyntax.TokenComment, diff --git a/tools/docgen/main.go b/tools/docgen/main.go index b304414a..12aa0269 100644 --- a/tools/docgen/main.go +++ b/tools/docgen/main.go @@ -390,7 +390,7 @@ func renderDataSourceDoc(pluginSchema *plugin.Schema, dataSourceName string, dat } func description(doc string) string { - return strings.Join(utils.TrimDedent(doc), "\n") + return utils.Dedent(doc) } func shortDescription(doc string) string { From f5a0f9ecfafd3ef28aff01942b8ebc08c73d4e95 Mon Sep 17 00:00:00 2001 From: traut Date: Wed, 8 Jan 2025 19:59:06 +0100 Subject: [PATCH 3/5] Refactor dedenting func --- pkg/utils/string.go | 92 +++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 58 deletions(-) diff --git a/pkg/utils/string.go b/pkg/utils/string.go index 9b4d7a47..32614c6f 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -1,7 +1,6 @@ package utils import ( - "bufio" "slices" "strings" "sync" @@ -68,71 +67,48 @@ func MemoizedKeys[M ~map[string]V, V any](m *M) func() string { }) } -const defaultTabSize = 4 - // Strip common whitespace from the beginnings of the lines, trim the end whitespace // Tab is assumed to be equal to 4 spaces -func TrimDedent(text string) (lines []string) { - return TrimDedentTabsize(text, defaultTabSize) -} +func Dedent(text string) string { + lines := strings.Split(text, "\n") + commonIndent := findCommonIndent(lines) -// Strip common whitespace from the beginnings of the lines, trim the end whitespace -func TrimDedentTabsize(text string, tabSize int) (lines []string) { - s := bufio.NewScanner(strings.NewReader(strings.ReplaceAll(text, "\t", " "))) - var commonWhitespace string - lenNonempty := 0 - - for s.Scan() { - line := s.Text() - if lenNonempty != 0 && commonWhitespace == "" { - lines = append(lines, line) - lenNonempty = len(lines) - continue - } - numTabs := 0 - numSpaces := 0 - firstNonSpace := strings.IndexFunc(line, func(r rune) bool { - switch r { - case '\t': - numTabs += 1 - return false - case ' ': - numSpaces += 1 - return false - default: - return true - } - }) - if firstNonSpace == -1 { - if lenNonempty != 0 { - lines = append(lines, commonWhitespace) - } - continue + newlines := make([]string, 0) + + for _, line := range lines { + trimmed := strings.TrimLeft(line, "\t") + // If the line is empty, trim it fully + var newline string + if trimmed == "" { + newline = trimmed + } else { + newline = strings.TrimPrefix(line, commonIndent) } + newlines = append(newlines, newline) + } + + return strings.Join(newlines, "\n") +} + +func findCommonIndent(lines []string) string { + minCommonIndent := "" - whitespace := numSpaces + numTabs*tabSize - if numTabs > 0 { - // replace tabs for better slice-ability - var sb strings.Builder + for _, line := range lines { + trimmed := strings.TrimLeft(line, "\t") - for i := 0; i < whitespace; i++ { - sb.WriteByte(' ') - } - sb.WriteString(line[firstNonSpace:]) - line = sb.String() + // Ignore empty lines + if trimmed == "" { + continue } + nonIndentCharIndex := strings.IndexFunc(line, func(c rune) bool { + return c != '\t' + }) - if lenNonempty == 0 { - commonWhitespace = strings.Repeat(" ", whitespace) - } else if whitespace < len(commonWhitespace) { - commonWhitespace = commonWhitespace[:whitespace] + lineIndent := line[:nonIndentCharIndex] + if minCommonIndent == "" || len(lineIndent) < len(minCommonIndent) { + minCommonIndent = lineIndent } - lines = append(lines, line) - lenNonempty = len(lines) } - lines = lines[:lenNonempty:lenNonempty] - for i := range lines { - lines[i] = strings.TrimRightFunc(lines[i][len(commonWhitespace):], unicode.IsSpace) - } - return + + return minCommonIndent } From 257ee797a4746b58bf601ea74dbcb03d1d4c7c68 Mon Sep 17 00:00:00 2001 From: traut Date: Wed, 8 Jan 2025 20:01:32 +0100 Subject: [PATCH 4/5] Comment adjustment --- pkg/utils/string.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/utils/string.go b/pkg/utils/string.go index 32614c6f..f2a8b9b3 100644 --- a/pkg/utils/string.go +++ b/pkg/utils/string.go @@ -67,8 +67,7 @@ func MemoizedKeys[M ~map[string]V, V any](m *M) func() string { }) } -// Strip common whitespace from the beginnings of the lines, trim the end whitespace -// Tab is assumed to be equal to 4 spaces +// Strip common margin from the beginnings of the lines func Dedent(text string) string { lines := strings.Split(text, "\n") commonIndent := findCommonIndent(lines) From a86f4b3b9d0e2a21add5311ec29b076461010fdb Mon Sep 17 00:00:00 2001 From: traut Date: Wed, 8 Jan 2025 20:10:36 +0100 Subject: [PATCH 5/5] Formatting --- internal/builtin/data_http.go | 2 +- internal/builtin/data_rss.go | 5 ++--- plugin/dataspec/attr_spec.go | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/builtin/data_http.go b/internal/builtin/data_http.go index 3d7e8ea1..1cd0b089 100644 --- a/internal/builtin/data_http.go +++ b/internal/builtin/data_http.go @@ -19,8 +19,8 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/blackstork-io/fabric/internal/builtin/utils" - u "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/pkg/diagnostics" + u "github.com/blackstork-io/fabric/pkg/utils" "github.com/blackstork-io/fabric/plugin" "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/dataspec/constraint" diff --git a/internal/builtin/data_rss.go b/internal/builtin/data_rss.go index 1b141c23..3495912d 100644 --- a/internal/builtin/data_rss.go +++ b/internal/builtin/data_rss.go @@ -11,7 +11,9 @@ import ( "sync" "time" + readability "github.com/go-shiori/go-readability" "github.com/hashicorp/hcl/v2" + "github.com/microcosm-cc/bluemonday" "github.com/mmcdole/gofeed" "github.com/zclconf/go-cty/cty" @@ -21,9 +23,6 @@ import ( "github.com/blackstork-io/fabric/plugin/dataspec" "github.com/blackstork-io/fabric/plugin/dataspec/constraint" "github.com/blackstork-io/fabric/plugin/plugindata" - - readability "github.com/go-shiori/go-readability" - "github.com/microcosm-cc/bluemonday" ) const ( diff --git a/plugin/dataspec/attr_spec.go b/plugin/dataspec/attr_spec.go index c2e7b0f3..996da0dd 100644 --- a/plugin/dataspec/attr_spec.go +++ b/plugin/dataspec/attr_spec.go @@ -72,7 +72,6 @@ func formatType(buf *strings.Builder, t cty.Type) { } func (a *AttrSpec) DocComment() hclwrite.Tokens { - docTmplFilename := path.Base(docTmplPath) tmpl, err := template.New(docTmplFilename).ParseFiles(docTmplPath) if err != nil {