Simple includes implementation

README.md

@@ -380,6 +380,30 @@ Use this command to install the package in Kibana.
 
 The command uses Kibana API to install the package in Kibana. The package must be exposed via the Package Registry or built locally in zip format so they can be installed using --zip parameter. Zip packages can be installed directly in Kibana >= 8.7.0. More details in this [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/install_package.md).
 
+### `elastic-package links`
+
+_Context: global_
+
+Use this command to manage linked files in the repository.
+
+### `elastic-package links check`
+
+_Context: global_
+
+Use this command to check if linked files references inside the current directory are up to date.
+
+### `elastic-package links list`
+
+_Context: global_
+
+Use this command to list all packages that have linked file references that include the current directory.
+
+### `elastic-package links update`
+
+_Context: global_
+
+Use this command to update all linked files references inside the current directory.
+
 ### `elastic-package lint`
 
 _Context: package_

cmd/links.go

@@ -0,0 +1,144 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/files"
+)
+
+const (
+	linksLongDescription       = `Use this command to manage linked files in the repository.`
+	linksCheckLongDescription  = `Use this command to check if linked files references inside the current directory are up to date.`
+	linksUpdateLongDescription = `Use this command to update all linked files references inside the current directory.`
+	linksListLongDescription   = `Use this command to list all packages that have linked file references that include the current directory.`
+)
+
+func setupLinksCommand() *cobraext.Command {
+	cmd := &cobra.Command{
+		Use:   "links",
+		Short: "Manage linked files",
+		Long:  linksLongDescription,
+		RunE: func(parent *cobra.Command, args []string) error {
+			return cobraext.ComposeCommandsParentContext(parent, args, parent.Commands()...)
+		},
+	}
+
+	cmd.AddCommand(getLinksCheckCommand())
+	cmd.AddCommand(getLinksUpdateCommand())
+	cmd.AddCommand(getLinksListCommand())
+
+	return cobraext.NewCommand(cmd, cobraext.ContextGlobal)
+}
+
+func getLinksCheckCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "check",
+		Short: "Check for linked files changes",
+		Long:  linksCheckLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  linksCheckCommandAction,
+	}
+	return cmd
+}
+
+func linksCheckCommandAction(cmd *cobra.Command, args []string) error {
+	cmd.Printf("Check for linked files changes\n")
+	pwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("reading current working directory failed: %w", err)
+	}
+
+	linkedFiles, err := files.CheckLinkedFiles(pwd)
+	if err != nil {
+		return fmt.Errorf("checking linked files are up-to-date failed: %w", err)
+	}
+	for _, f := range linkedFiles {
+		if !f.UpToDate {
+			cmd.Printf("%s is outdated.\n", filepath.Join(f.WorkDir, f.LinkFilePath))
+		}
+	}
+	if len(linkedFiles) > 0 {
+		return fmt.Errorf("linked files are outdated")
+	}
+	return nil
+}
+
+func getLinksUpdateCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "update",
+		Short: "Update linked files checksums if needed.",
+		Long:  linksUpdateLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  linksUpdateCommandAction,
+	}
+	return cmd
+}
+
+func linksUpdateCommandAction(cmd *cobra.Command, args []string) error {
+	cmd.Printf("Update linked files checksums if needed.\n")
+	pwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("reading current working directory failed: %w", err)
+	}
+
+	updatedLinks, err := files.UpdateLinkedFiles(pwd)
+	if err != nil {
+		return fmt.Errorf("updating linked files checksums failed: %w", err)
+	}
+
+	for _, l := range updatedLinks {
+		cmd.Printf("%s was updated.\n", l.LinkFilePath)
+	}
+
+	return nil
+}
+
+func getLinksListCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "list",
+		Short: "List packages linking files from this path",
+		Long:  linksListLongDescription,
+		Args:  cobra.NoArgs,
+		RunE:  linksListCommandAction,
+	}
+	cmd.Flags().BoolP(cobraext.PackagesFlagName, "", false, cobraext.PackagesFlagDescription)
+	return cmd
+}
+
+func linksListCommandAction(cmd *cobra.Command, args []string) error {
+	onlyPackages, err := cmd.Flags().GetBool(cobraext.PackagesFlagName)
+	if err != nil {
+		return cobraext.FlagParsingError(err, cobraext.PackagesFlagName)
+	}
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("reading current working directory failed: %w", err)
+	}
+
+	byPackage, err := files.ListLinkedFilesByPackage(pwd)
+	if err != nil {
+		return fmt.Errorf("listing linked packages failed: %w", err)
+	}
+
+	for _, pkg := range byPackage {
+		if onlyPackages {
+			cmd.Println(pkg.PackageName)
+			continue
+		}
+		for _, link := range pkg.Links {
+			cmd.Println(link)
+		}
+	}
+
+	return nil
+}

cmd/lint.go

@@ -45,7 +45,6 @@ func setupLintCommand() *cobraext.Command {
 
 func lintCommandAction(cmd *cobra.Command, args []string) error {
 	cmd.Println("Lint the package")
-
 	readmeFiles, err := docs.AreReadmesUpToDate()
 	if err != nil {
 		for _, f := range readmeFiles {

cmd/root.go

@@ -28,6 +28,7 @@ var commands = []*cobraext.Command{
 	setupExportCommand(),
 	setupFormatCommand(),
 	setupInstallCommand(),
+	setupLinksCommand(),
 	setupLintCommand(),
 	setupProfilesCommand(),
 	setupReportsCommand(),

docs/howto/links.md

@@ -0,0 +1,110 @@
+# HOWTO: Use links to reuse common files.
+
+## Introduction
+
+Many packages have files that are equal between them. This is more common in pipelines, 
+input configurations, and field definitions.
+
+In order to help developers, there is the ability to define links, so a file that might be reused needs to only be defined once, and can be reused from any other packages.
+
+
+# Links
+
+Currently, there are some specific places where links can be defined:
+
+- `elasticsearch/ingest_pipeline`
+- `data_stream/**/elasticsearch/ingest_pipeline`
+- `agent/input`
+- `data_stream/**/agent/stream`
+- `data_stream/**/fields`
+
+A link consists of a file with a `.link` extension that contains a path, relative to its location, to the file that it will be replaced with. It also consists of a checksum to validate the linked file is up to date with the package expectations.
+
+`data_stream/foo/elasticsearch/ingest_pipeline/default.yml.link`
+
+```
+../../../../../testpackage/data_stream/test/elasticsearch/ingest_pipeline/default.yml f7c5f0c03aca8ef68c379a62447bdafbf0dcf32b1ff2de143fd6878ee01a91ad
+```
+
+This will use the contents of the linked file during validation, tests, and building of the package, so functionally nothing changes from the package point of view.
+
+## The `_dev/shared` folder
+
+As a convenience, shared files can be placed under `_dev/shared` if they are going to be
+reused from several places. They can even be added outside of any package, in any place in the repository.
+
+## Managing Links with elastic-package
+
+The `elastic-package` tool provides several subcommands to help manage linked files:
+
+### `elastic-package links check`
+
+Check if all linked files in the current directory and its subdirectories are up to date. This command verifies that the checksums in link files match the actual content of the included files.
+
+```bash
+elastic-package links check
+```
+
+This command will:
+- Scan for all `.link` files in the current directory tree
+- Validate that each linked file's checksum matches the included file's current content
+- Report any outdated link files that need updating
+- Exit with an error if any link files are outdated
+
+### `elastic-package links update`
+
+Update the checksums of all outdated linked files in the current directory and its subdirectories.
+
+```bash
+elastic-package links update
+```
+
+This command will:
+- Find all `.link` files that have outdated checksums
+- Calculate new checksums for the included files
+- Update the `.link` files with the new checksums
+- Report which link files were updated
+
+### `elastic-package links list`
+
+List all packages that have linked files referencing content from the current directory.
+
+```bash
+# List all linked file paths
+elastic-package links list
+
+# List only package names (without individual file paths)
+elastic-package links list --packages
+```
+
+This command will:
+- Find all `.link` files in the repository that reference files in the current directory
+- Group the results by package name
+- Display either the full file paths or just the package names (with `--packages` flag)
+
+## Workflow
+
+A typical workflow for managing linked files:
+
+1. **Create a shared file** in a central location (e.g., `_dev/shared/` or in a reference package)
+
+2. **Create link files** in packages that need to reference the shared file:
+   ```bash
+   echo "../../_dev/shared/common-pipeline.yml" > data_stream/logs/elasticsearch/ingest_pipeline/default.yml.link
+   ```
+
+3. **Update checksums** to make the link valid:
+   ```bash
+   elastic-package links update
+   ```
+
+4. **Check links regularly** to ensure they stay up to date:
+   ```bash
+   elastic-package links check
+   ```
+
+5. **When modifying shared files**, update all dependent links:
+   ```bash
+   # After editing a shared file, update all links that reference it
+   elastic-package links update
+   ```

internal/builder/packages.go

@@ -185,6 +185,15 @@ func BuildPackage(ctx context.Context, options BuildOptions) (string, error) {
 		return "", fmt.Errorf("adding dynamic mappings: %w", err)
 	}
 
+	logger.Debug("Include linked files")
+	links, err := files.IncludeLinkedFilesFromPath(options.PackageRoot, destinationDir)
+	if err != nil {
+		return "", fmt.Errorf("including linked files failed: %w", err)
+	}
+	for _, l := range links {
+		logger.Debugf("Linked file included (path: %s)", l.TargetFilePath(destinationDir))
+	}
+
 	if options.CreateZip {
 		return buildZippedPackage(ctx, options, destinationDir)
 	}

internal/cobraext/flags.go

@@ -138,6 +138,9 @@ const (
 	GenerateTestResultFlagName        = "generate"
 	GenerateTestResultFlagDescription = "generate test result file"
 
+	PackagesFlagName        = "packages"
+	PackagesFlagDescription = "whether to return packages names or complete paths for the linked files found"
+
 	IngestPipelineIDsFlagName        = "id"
 	IngestPipelineIDsFlagDescription = "Elasticsearch ingest pipeline IDs (comma-separated values)"
 

internal/elasticsearch/ingest/datastream.go

@@ -19,13 +19,16 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/packages"
 )
 
 var (
-	ingestPipelineTag   = regexp.MustCompile(`{{\s*IngestPipeline.+}}`)
-	defaultPipelineJSON = "default.json"
-	defaultPipelineYML  = "default.yml"
+	ingestPipelineTag       = regexp.MustCompile(`{{\s*IngestPipeline.+}}`)
+	defaultPipelineJSON     = "default.json"
+	defaultPipelineJSONLink = "default.json"
+	defaultPipelineYML      = "default.yml.link"
+	defaultPipelineYMLLink  = "default.yml.link"
 )
 
 type Rule struct {
@@ -71,17 +74,21 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er
 	elasticsearchPath := filepath.Join(dataStreamPath, "elasticsearch", "ingest_pipeline")
 
 	var pipelineFiles []string
-	for _, pattern := range []string{"*.json", "*.yml"} {
+	for _, pattern := range []string{"*.json", "*.yml", "*.link"} {
 		files, err := filepath.Glob(filepath.Join(elasticsearchPath, pattern))
 		if err != nil {
 			return nil, fmt.Errorf("listing '%s' in '%s': %w", pattern, elasticsearchPath, err)
 		}
 		pipelineFiles = append(pipelineFiles, files...)
 	}
 
+	linksFS, err := files.CreateLinksFSFromPath(elasticsearchPath)
+	if err != nil {
+		return nil, fmt.Errorf("creating links filesystem failed: %w", err)
+	}
 	var pipelines []Pipeline
 	for _, path := range pipelineFiles {
-		c, err := os.ReadFile(path)
+		c, err := linksFS.ReadFile(path)
 		if err != nil {
 			return nil, fmt.Errorf("reading ingest pipeline failed (path: %s): %w", path, err)
 		}
@@ -108,7 +115,7 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er
 		pipelines = append(pipelines, Pipeline{
 			Path:            path,
 			Name:            getPipelineNameWithNonce(name[:strings.Index(name, ".")], nonce),
-			Format:          filepath.Ext(path)[1:],
+			Format:          filepath.Ext(strings.TrimSuffix(path, ".link"))[1:],
 			Content:         cWithRerouteProcessors,
 			ContentOriginal: c,
 		})
@@ -119,7 +126,8 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er
 func addRerouteProcessors(pipeline []byte, dataStreamPath, path string) ([]byte, error) {
 	// Only attach routing_rules.yml reroute processors after the default pipeline
 	filename := filepath.Base(path)
-	if filename != defaultPipelineJSON && filename != defaultPipelineYML {
+	if filename != defaultPipelineJSON && filename != defaultPipelineYML &&
+		filename != defaultPipelineJSONLink && filename != defaultPipelineYMLLink {
 		return pipeline, nil
 	}