diff --git a/.schema/devbox-plugin.schema.json b/.schema/devbox-plugin.schema.json index 3a0a9cb5cfe..d78be0b6ca9 100644 --- a/.schema/devbox-plugin.schema.json +++ b/.schema/devbox-plugin.schema.json @@ -114,7 +114,10 @@ "description": "Shell specific options and hooks for the plugin.", "items": { "init_hook": { - "type": ["array", "string"], + "type": [ + "array", + "string" + ], "description": "Shell command to run right before initializing the user's shell, running a script, or starting a service" }, "scripts": { @@ -123,7 +126,10 @@ "patternProperties": { ".*": { "description": "Alias name for the script.", - "type": ["array", "string"], + "type": [ + "array", + "string" + ], "items": { "type": "string", "description": "The script's shell commands." @@ -137,10 +143,115 @@ "description": "List of additional plugins to activate within your devbox shell", "type": "array", "items": { - "description": "Name of the plugin to activate.", - "type": "string" + "type": "object", + "properties": { + "owner": { + "description": "Username or organization name", + "type": "string" + }, + "repo": { + "description": "Repository name", + "type": "string" + }, + "type": { + "description": "Type of connection (https, ssh, file, builtin)", + "type": "string", + "enum": [ + "https", + "ssh", + "file", + "builtin" + ], + "default": "https" + }, + "host": { + "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", + "type": "string" + }, + "port": { + "description": "Port to use for the connection", + "type": "integer", + "minimum": 1, + "maximum": 65535, + "default": 0 + }, + "dir": { + "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", + "type": "string" + }, + "ref": { + "description": "Git reference", + "type": "string" + }, + "rev": { + "description": "Git revision (only relevant for https or ssh types)", + "type": "string" + } + }, + "required": [ + "owner", + "repo" + ], + "dependencies": { + "host": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } + } + }, + "owner": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } + } + }, + "rev": { + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "https", + "ssh" + ] + } + } + }, + { + "properties": { + "type": { + "enum": [ + "file", + "builtin" + ] + } + }, + "not": { + "required": [ + "rev" + ] + } + } + ] + } + }, + "additionalProperties": false } } }, - "required": ["name", "version", "description"] + "required": [ + "name", + "version", + "description" + ] } diff --git a/.schema/devbox.schema.json b/.schema/devbox.schema.json index f540406976e..0819d97e018 100644 --- a/.schema/devbox.schema.json +++ b/.schema/devbox.schema.json @@ -1,154 +1,253 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "$id": "https://github.com/jetify-com/devbox", - "title": "Devbox json definition", - "description": "Defines fields and acceptable values of devbox.json", - "type": "object", - "properties": { - "$schema": { - "description": "The schema version of this devbox.json file.", - "type": "string" - }, - "name": { - "description": "The name of the Devbox development environment.", - "type": "string" - }, - "description": { - "description": "A description of the Devbox development environment.", + "$schema": "http://json-schema.org/draft-05/schema#", + "$id": "https://github.com/jetify-com/devbox", + "title": "Devbox json definition", + "description": "Defines fields and acceptable values of devbox.json", + "type": "object", + "properties": { + "$schema": { + "description": "The schema version of this devbox.json file.", + "type": "string" + }, + "name": { + "description": "The name of the Devbox development environment.", + "type": "string" + }, + "description": { + "description": "A description of the Devbox development environment.", + "type": "string" + }, + "packages": { + "description": "Collection of packages to install", + "oneOf": [ + { + "type": "array", + "items": { + "description": "Name and version of each package in name@version format.", "type": "string" + } }, - "packages": { - "description": "Collection of packages to install", - "oneOf": [ + { + "type": "object", + "description": "Name of each package in {\"name\": {\"version\": \"1.2.3\"}} format.", + "patternProperties": { + ".*": { + "oneOf": [ { - "type": "array", - "items": { - "description": "Name and version of each package in name@version format.", - "type": "string" + "type": "object", + "description": "Version number of the specified package in {\"version\": \"1.2.3\"} format.", + "properties": { + "version": { + "type": "string", + "description": "Version of the package" + }, + "platforms": { + "type": "array", + "description": "Names of platforms to install the package on. This package will be skipped for any platforms not on this list", + "items": { + "enum": [ + "i686-linux", + "aarch64-linux", + "aarch64-darwin", + "x86_64-darwin", + "x86_64-linux", + "armv7l-linux" + ] + } + }, + "excluded_platforms": { + "type": "array", + "description": "Names of platforms to exclude the package on", + "items": { + "enum": [ + "i686-linux", + "aarch64-linux", + "aarch64-darwin", + "x86_64-darwin", + "x86_64-linux", + "armv7l-linux" + ] + } + }, + "glibc_patch": { + "type": "boolean", + "description": "Whether to patch glibc to the latest available version for this package" } + } }, { - "type": "object", - "description": "Name of each package in {\"name\": {\"version\": \"1.2.3\"}} format.", - "patternProperties": { - ".*": { - "oneOf": [ - { - "type": "object", - "description": "Version number of the specified package in {\"version\": \"1.2.3\"} format.", - "properties": { - "version": { - "type": "string", - "description": "Version of the package" - }, - "platforms": { - "type": "array", - "description": "Names of platforms to install the package on. This package will be skipped for any platforms not on this list", - "items": { - "enum": [ - "i686-linux", - "aarch64-linux", - "aarch64-darwin", - "x86_64-darwin", - "x86_64-linux", - "armv7l-linux" - ] - } - }, - "excluded_platforms": { - "type": "array", - "description": "Names of platforms to exclude the package on", - "items": { - "enum": [ - "i686-linux", - "aarch64-linux", - "aarch64-darwin", - "x86_64-darwin", - "x86_64-linux", - "armv7l-linux" - ] - } - }, - "glibc_patch": { - "type": "boolean", - "description": "Whether to patch glibc to the latest available version for this package" - } - } - }, - { - "type": "string", - "description": "Version of the package to install." - } - ] - } - } - } - ] - }, - "env": { - "description": "List of additional environment variables to be set in the Devbox environment. Values containing $PATH or $PWD will be expanded. No other variable expansion or command substitution will occur.", - "type": "object", - "patternProperties": { - ".*": { - "type": "string", - "description": "Value of the environment variable." + "type": "string", + "description": "Version of the package to install." } + ] } + } + } + ] + }, + "env": { + "description": "List of additional environment variables to be set in the Devbox environment. Values containing $PATH or $PWD will be expanded. No other variable expansion or command substitution will occur.", + "type": "object", + "patternProperties": { + ".*": { + "type": "string", + "description": "Value of the environment variable." + } + } + }, + "shell": { + "description": "Definitions of scripts and actions to take when in devbox shell.", + "type": "object", + "properties": { + "init_hook": { + "type": [ + "array", + "string" + ], + "items": { + "description": "List of shell commands/scripts to run right after devbox shell starts.", + "type": "string" + } }, - "shell": { - "description": "Definitions of scripts and actions to take when in devbox shell.", - "type": "object", - "properties": { - "init_hook": { - "type": [ - "array", - "string" - ], - "items": { - "description": "List of shell commands/scripts to run right after devbox shell starts.", - "type": "string" - } - }, - "scripts": { - "description": "List of command/script definitions to run with `devbox run `.", - "type": "object", - "patternProperties": { - ".*": { - "description": "Alias name for the script.", - "type": [ - "array", - "string" - ], - "items": { - "type": "string", - "description": "The script's shell commands." - } - } - } - } - }, - "additionalProperties": false - }, - "include": { - "description": "List of additional plugins to activate within your devbox shell", - "type": "array", - "items": { - "description": "Name of the plugin to activate.", - "type": "string" + "scripts": { + "description": "List of command/script definitions to run with `devbox run `.", + "type": "object", + "patternProperties": { + ".*": { + "description": "Alias name for the script.", + "type": [ + "array", + "string" + ], + "items": { + "type": "string", + "description": "The script's shell commands." + } } - }, - "env_from": { + } + } + }, + "additionalProperties": false + }, + "include": { + "description": "List of additional plugins to activate within your devbox shell", + "type": "array", + "items": { + "type": "object", + "properties": { + "owner": { + "description": "Username or organization name", + "type": "string" + }, + "repo": { + "description": "Repository name", + "type": "string" + }, + "type": { + "description": "Type of connection (https, ssh, file, builtin)", + "type": "string", + "enum": [ + "https", + "ssh", + "file", + "builtin" + ], + "default": "https" + }, + "host": { + "description": "Host of the repository (e.g., gitlab, github, bitbucket, localhost)", + "type": "string" + }, + "port": { + "description": "Port to use for the connection", + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "dir": { + "description": "Subdirectory that should be accessed in the repo. Defaults to the path key if not provided", + "type": "string" + }, + "ref": { + "description": "Git reference", + "type": "string" + }, + "rev": { + "description": "Git revision (only relevant for https or ssh types)", "type": "string" + } }, - "nixpkgs": { - "type": "object", - "properties": { - "commit": { - "type": "string", - "description": "The commit hash of the nixpkgs repository to use" + "required": [ + "owner", + "repo" + ], + "dependencies": { + "host": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] } + } } + }, + "owner": { + "not": { + "properties": { + "type": { + "enum": [ + "builtin" + ] + } + } + } + }, + "rev": { + "oneOf": [ + { + "properties": { + "type": { + "enum": [ + "https", + "ssh" + ] + } + } + }, + { + "properties": { + "type": { + "enum": [ + "file", + "builtin" + ] + } + }, + "not": { + "required": [ + "rev" + ] + } + } + ] + } + }, + "additionalProperties": false + } + }, + "env_from": { + "type": "string" + }, + "nixpkgs": { + "type": "object", + "properties": { + "commit": { + "type": "string", + "description": "The commit hash of the nixpkgs repository to use" } + } }, "additionalProperties": false -} + } diff --git a/devbox.d/jetify-com.devbox-plugins.mongodb/mongod.conf b/devbox.d/jetify-com.devbox-plugins.mongodb/mongod.conf new file mode 100644 index 00000000000..dfbb8ae9102 --- /dev/null +++ b/devbox.d/jetify-com.devbox-plugins.mongodb/mongod.conf @@ -0,0 +1,3 @@ +storage: + journal: + enabled: true \ No newline at end of file diff --git a/devbox.d/nginx/fastcgi.conf b/devbox.d/nginx/fastcgi.conf new file mode 100644 index 00000000000..97f4358a4ba --- /dev/null +++ b/devbox.d/nginx/fastcgi.conf @@ -0,0 +1,17 @@ +fastcgi_param GATEWAY_INTERFACE CGI/1.1; +fastcgi_param SERVER_SOFTWARE nginx; +fastcgi_param QUERY_STRING $query_string; +fastcgi_param REQUEST_METHOD $request_method; +fastcgi_param CONTENT_TYPE $content_type; +fastcgi_param CONTENT_LENGTH $content_length; +fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name; +fastcgi_param SCRIPT_NAME $fastcgi_script_name; +fastcgi_param REQUEST_URI $request_uri; +fastcgi_param DOCUMENT_URI $document_uri; +fastcgi_param DOCUMENT_ROOT $document_root; +fastcgi_param SERVER_PROTOCOL $server_protocol; +fastcgi_param REMOTE_ADDR $remote_addr; +fastcgi_param REMOTE_PORT $remote_port; +fastcgi_param SERVER_ADDR $server_addr; +fastcgi_param SERVER_PORT $server_port; +fastcgi_param SERVER_NAME $server_name; diff --git a/devbox.d/nginx/nginx.conf b/devbox.d/nginx/nginx.conf new file mode 100644 index 00000000000..3b0edd2f534 --- /dev/null +++ b/devbox.d/nginx/nginx.conf @@ -0,0 +1,23 @@ +# The nginx.conf in this folder is automatically generated from nginx.template +# To modify your NGINX config, edit the nginx.template file + +events {} +http{ +server { + listen 8081; + listen [::]:8081; + server_name localhost; + root ../../../devbox.d/web; + + error_log error.log error; + access_log access.log; + client_body_temp_path temp/client_body; + proxy_temp_path temp/proxy; + fastcgi_temp_path temp/fastcgi; + uwsgi_temp_path temp/uwsgi; + scgi_temp_path temp/scgi; + + index index.html; + server_tokens off; + } +} diff --git a/devbox.d/nginx/nginx.template b/devbox.d/nginx/nginx.template new file mode 100644 index 00000000000..48da6a0e924 --- /dev/null +++ b/devbox.d/nginx/nginx.template @@ -0,0 +1,23 @@ +# The nginx.conf in this folder is automatically generated from nginx.template +# To modify your NGINX config, edit the nginx.template file + +events {} +http{ +server { + listen $NGINX_WEB_PORT; + listen [::]:$NGINX_WEB_PORT; + server_name $NGINX_WEB_SERVER_NAME; + root $NGINX_WEB_ROOT; + + error_log error.log error; + access_log access.log; + client_body_temp_path temp/client_body; + proxy_temp_path temp/proxy; + fastcgi_temp_path temp/fastcgi; + uwsgi_temp_path temp/uwsgi; + scgi_temp_path temp/scgi; + + index index.html; + server_tokens off; + } +} diff --git a/devbox.d/web/index.html b/devbox.d/web/index.html new file mode 100644 index 00000000000..21250f01b2a --- /dev/null +++ b/devbox.d/web/index.html @@ -0,0 +1,10 @@ + + + + + Hello World! + + + Hello World! + + diff --git a/devbox.json b/devbox.json index f346011420f..75d75e90d51 100644 --- a/devbox.json +++ b/devbox.json @@ -1,14 +1,38 @@ { - "name": "devbox", + "name": "devbox", "description": "Instant, easy, and predictable development environments", "packages": { - "fd": "latest", + "fd": "latest", "git": "latest", - "go": "latest" + "go": "latest" }, + "include": [ + /* examples */ + //{ + // "type": "github", + // "owner": "jetify-com", + // "repo": "devbox-plugins", + // "dir": "tmux" + //} + //, + //{ + // "type": "path", + // "path": "../testing-devbox-plugin", + //}, + //{ + // "type": "github", + // "owner": "jetify-com", + // "repo": "devbox-plugins", + // "dir": "mongodb" + //}, + //{ + // "type": "builtin", + // "path": "nginx", + //}, + ], "env": { "GOENV": "off", - "PATH": "$PWD/dist/tools:$PATH:$PWD/dist" + "PATH": "$PWD/dist/tools:$PATH:$PWD/dist" }, "shell": { "init_hook": [ @@ -21,11 +45,11 @@ ], "scripts": { // Build devbox for the current platform - "build": "go build -o dist/devbox ./cmd/devbox", + "build": "go build -o dist/devbox ./cmd/devbox", "build-darwin-amd64": "GOOS=darwin GOARCH=amd64 go build -o dist/devbox-darwin-amd64 ./cmd/devbox", "build-darwin-arm64": "GOOS=darwin GOARCH=arm64 go build -o dist/devbox-darwin-arm64 ./cmd/devbox", - "build-linux-amd64": "GOOS=linux GOARCH=amd64 go build -o dist/devbox-linux-amd64 ./cmd/devbox", - "build-linux-arm64": "GOOS=linux GOARCH=arm64 go build -o dist/devbox-linux-arm64 ./cmd/devbox", + "build-linux-amd64": "GOOS=linux GOARCH=amd64 go build -o dist/devbox-linux-amd64 ./cmd/devbox", + "build-linux-arm64": "GOOS=linux GOARCH=arm64 go build -o dist/devbox-linux-arm64 ./cmd/devbox", "build-all": [ "devbox run build-darwin-amd64", "devbox run build-darwin-arm64", @@ -33,12 +57,12 @@ "devbox run build-linux-arm64" ], // Open VSCode - "code": "code .", - "lint": "go tool golangci-lint run --timeout 5m && scripts/gofumpt.sh", - "fmt": "scripts/gofumpt.sh", - "test": "go test -race -cover ./...", + "code": "code .", + "lint": "go tool golangci-lint run --timeout 5m && scripts/gofumpt.sh", + "fmt": "scripts/gofumpt.sh", + "test": "go test -race -cover ./...", "test-projects-only": "DEVBOX_RUN_PROJECT_TESTS=1 go test -v -timeout ${DEVBOX_GOLANG_TEST_TIMEOUT:-30m} ./... -run \"TestExamples|TestScriptsWithProjects\"", - "update-examples": "devbox run build && go run testscripts/testrunner/updater/main.go", + "update-examples": "devbox run build && go run testscripts/testrunner/updater/main.go", // Updates the Flake's vendorHash: First run `go mod vendor` to vendor // the dependencies, then hash the vendor directory with Nix. // The hash is saved to the `vendor-hash` file, which is then @@ -50,7 +74,10 @@ "go mod vendor -o $vendor", "nix hash path $vendor >vendor-hash" ], - "tidy": ["go mod tidy", "devbox run update-hash"], + "tidy": [ + "go mod tidy", + "devbox run update-hash" + ], // docker-testscripts runs the testscripts with Docker to exercise // Linux-specific tests. It invokes the test binary directly, so any extra // test runner flags must have their "-test." prefix. @@ -60,7 +87,6 @@ // devbox run docker-testscripts -test.run ^TestScripts$/python "docker-testscripts": [ "cd testscripts", - // The Dockerfile looks for a testscripts-$TARGETOS-$TARGETARCH binary // to run the tests. Pre-compiling a static test binary lets us avoid // polluting the container with a Go toolchain or shared libraries that diff --git a/devbox.lock b/devbox.lock index 3e972c85304..c60dda93bf6 100644 --- a/devbox.lock +++ b/devbox.lock @@ -121,10 +121,6 @@ } } }, - "github:NixOS/nixpkgs/nixpkgs-unstable": { - "last_modified": "2025-04-07T13:23:10Z", - "resolved": "github:NixOS/nixpkgs/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b?lastModified=1744032190&narHash=sha256-KSlfrncSkcu1YE%2BuuJ%2FPTURsSlThoGkRqiGDVdbiE%2Fk%3D" - }, "go@latest": { "last_modified": "2025-03-11T17:52:14Z", "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#go", diff --git a/examples/plugins/builtin/devbox.json b/examples/plugins/builtin/devbox.json index 5718936fe66..232881c81c1 100644 --- a/examples/plugins/builtin/devbox.json +++ b/examples/plugins/builtin/devbox.json @@ -12,6 +12,9 @@ }, "include": [ // Installs the php plugin using php82 as trigger package - "plugin:php82" + { + "type": "builtin", + "path": "php82" + } ] } diff --git a/examples/plugins/github-with-revision/devbox.json b/examples/plugins/github-with-revision/devbox.json index 201fcaa9e33..59dc5cc793d 100644 --- a/examples/plugins/github-with-revision/devbox.json +++ b/examples/plugins/github-with-revision/devbox.json @@ -11,6 +11,11 @@ } }, "include": [ - "github:jetify-com/devbox-plugin-example/d9c00334353c9b1294c7bd5dbea128c149b2eb3a" + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example", + "rev": "d9c00334353c9b1294c7bd5dbea128c149b2eb3a" + } ] } diff --git a/examples/plugins/github/devbox.json b/examples/plugins/github/devbox.json index 4711986c7b9..e896f907f3b 100644 --- a/examples/plugins/github/devbox.json +++ b/examples/plugins/github/devbox.json @@ -11,8 +11,22 @@ } }, "include": [ - "github:jetify-com/devbox-plugin-example", - "github:jetify-com/devbox-plugin-example?dir=custom-dir", - "github:jetify-com/devbox-plugin-example/test/branch", + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example" + }, + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example", + "dir": "custom-dir" + }, + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example/test", + "ref": "branch" + } ] } diff --git a/examples/plugins/local/devbox.json b/examples/plugins/local/devbox.json index b537e30e930..d17db910c29 100644 --- a/examples/plugins/local/devbox.json +++ b/examples/plugins/local/devbox.json @@ -11,6 +11,9 @@ } }, "include": [ - "path:my-plugin/plugin.json" + { + "type": "path", + "path": "my-plugin/plugin.json" + } ] } diff --git a/examples/plugins/v2-github/devbox.json b/examples/plugins/v2-github/devbox.json index dbcea18f16a..e4466cd5fae 100644 --- a/examples/plugins/v2-github/devbox.json +++ b/examples/plugins/v2-github/devbox.json @@ -12,7 +12,16 @@ }, "include": [ // TODO, make these more interesting by adding v2 capabilities - "github:jetify-com/devbox-plugin-example", - "github:jetify-com/devbox-plugin-example?dir=custom-dir" + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example" + }, + { + "type": "github", + "owner": "jetify-com", + "repo": "devbox-plugin-example", + "dir": "custom-dir" + } ] } diff --git a/examples/plugins/v2-local/devbox.json b/examples/plugins/v2-local/devbox.json index 9ebd1ae46c1..83d373024df 100644 --- a/examples/plugins/v2-local/devbox.json +++ b/examples/plugins/v2-local/devbox.json @@ -1,5 +1,4 @@ { - "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/main/.schema/devbox.schema.json", "packages": [], "env": { "PLUGIN1_ENV2": "override" @@ -28,8 +27,20 @@ } }, "include": [ - "./plugin1", - "path:plugin2", - "./plugin3/plugin.json", + { + "type": "path", + "path": "./plugin1" + }, + { + "type": "path", + "path": "plugin2" + }, + { + "type": "path", + "path": "./plugin3/plugin.json" + } + //"./plugin1", + //"path:plugin2", + //"./plugin3/plugin.json", ] } diff --git a/examples/plugins/v2-local/plugin1/plugin.json b/examples/plugins/v2-local/plugin1/plugin.json index 54dda78bffc..1ba583e39b7 100644 --- a/examples/plugins/v2-local/plugin1/plugin.json +++ b/examples/plugins/v2-local/plugin1/plugin.json @@ -1,6 +1,8 @@ { "name": "plugin1", - "packages": ["hello@latest"], + "packages": [ + "hello@latest" + ], "env": { "PLUGIN1_ENV": "success", "PLUGIN1_ENV2": "success" @@ -20,6 +22,9 @@ "{{ .Virtenv }}/process-compose.yaml": "process-compose.yaml" }, "include": [ - "./plugin1a" + { + "type": "path", + "path": "./plugin1a" + } ] } diff --git a/examples/plugins/v2-local/plugin1/plugin1a/plugin.json b/examples/plugins/v2-local/plugin1/plugin1a/plugin.json index bbe1fa2bda0..7b951201a17 100644 --- a/examples/plugins/v2-local/plugin1/plugin1a/plugin.json +++ b/examples/plugins/v2-local/plugin1/plugin1a/plugin.json @@ -1,6 +1,8 @@ { "name": "plugin1a", - "packages": ["cowsay@latest"], + "packages": [ + "cowsay@latest" + ], "env": { "PLUGIN1A_ENV": "success" }, @@ -15,6 +17,9 @@ } }, "include": [ - "../../plugin2" + { + "type": "path", + "path": "../../plugin2" + } ] } diff --git a/internal/devconfig/config.go b/internal/devconfig/config.go index 445585dd04a..d395be52845 100644 --- a/internal/devconfig/config.go +++ b/internal/devconfig/config.go @@ -15,12 +15,14 @@ import ( "github.com/pkg/errors" "github.com/samber/lo" "github.com/samber/lo/mutable" + "go.jetify.com/devbox/internal/build" "go.jetify.com/devbox/internal/cachehash" "go.jetify.com/devbox/internal/devbox/shellcmd" "go.jetify.com/devbox/internal/devconfig/configfile" "go.jetify.com/devbox/internal/lock" "go.jetify.com/devbox/internal/plugin" + "go.jetify.com/devbox/nix/flake" ) // ErrNotFound occurs when [Open] or [Find] cannot find a devbox config file @@ -237,14 +239,20 @@ func (c *Config) loadRecursive( ) error { included := make([]*Config, 0, len(c.Root.Include)) - for _, includeRef := range c.Root.Include { - pluginConfig, err := plugin.LoadConfigFromInclude( - includeRef, lockfile, filepath.Dir(c.Root.AbsRootPath)) + for _, ref := range c.Root.Include { + switch ref.Type { + case flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: + ref.Host = fmt.Sprintf("%s.com", ref.Type) + } + + pluginConfig, err := plugin.LoadConfigFromInclude(ref, lockfile, filepath.Dir(c.Root.AbsRootPath)) + if err != nil { return errors.WithStack(err) } - newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, includeRef) + newCyclePath := fmt.Sprintf("%s -> %s", cyclePath, ref) + if seen[pluginConfig.Source.Hash()] { // Note that duplicate includes are allowed if they are in different paths // e.g. 2 different plugins can include the same plugin. @@ -252,8 +260,8 @@ func (c *Config) loadRecursive( return errors.Errorf( "circular or duplicate include detected:\n%s", newCyclePath) } - seen[pluginConfig.Source.Hash()] = true + seen[pluginConfig.Source.Hash()] = true includable := createIncludableFromPluginConfig(pluginConfig) if err := includable.loadRecursive( @@ -268,6 +276,7 @@ func (c *Config) loadRecursive( c.Root.TopLevelPackages(), lockfile, ) + if err != nil { return errors.WithStack(err) } diff --git a/internal/devconfig/configfile/file.go b/internal/devconfig/configfile/file.go index de88ed952a5..03d7f736269 100644 --- a/internal/devconfig/configfile/file.go +++ b/internal/devconfig/configfile/file.go @@ -14,9 +14,11 @@ import ( "github.com/pkg/errors" "github.com/tailscale/hujson" + "go.jetify.com/devbox/internal/boxcli/usererr" "go.jetify.com/devbox/internal/cachehash" "go.jetify.com/devbox/internal/devbox/shellcmd" + "go.jetify.com/devbox/nix/flake" ) const ( @@ -53,7 +55,7 @@ type ConfigFile struct { // https:// for remote files // plugin: for built-in plugins // This is a similar format to nix inputs - Include []string `json:"include,omitempty"` + Include []flake.Ref `json:"include,omitempty"` ast *configAST } diff --git a/internal/plugin/files.go b/internal/plugin/files.go index 7bbef21e1d8..8f53c2d702a 100644 --- a/internal/plugin/files.go +++ b/internal/plugin/files.go @@ -18,7 +18,7 @@ func getConfigIfAny(inc Includable, projectDir string) (*Config, error) { switch includable := inc.(type) { case *devpkg.Package: return getBuiltinPluginConfigIfExists(includable, projectDir) - case *githubPlugin: + case *gitPlugin: content, err := includable.Fetch() if err != nil { return nil, errors.WithStack(err) diff --git a/internal/plugin/git.go b/internal/plugin/git.go new file mode 100644 index 00000000000..fca998dcb18 --- /dev/null +++ b/internal/plugin/git.go @@ -0,0 +1,364 @@ +package plugin + +import ( + "cmp" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/samber/lo" + + "go.jetify.com/devbox/internal/boxcli/usererr" + "go.jetify.com/devbox/internal/cachehash" + "go.jetify.com/devbox/internal/fileutil" + "go.jetify.com/devbox/nix/flake" + "go.jetify.com/pkg/filecache" +) + +var gitCache = filecache.New[[]byte]("devbox/plugin/git") + +type gitPlugin struct { + ref flake.Ref + name string +} + +// Github only allows alphanumeric, hyphen, underscore, and period in repo names. +// but we clean up just in case. +var githubNameRegexp = regexp.MustCompile("[^a-zA-Z0-9-_.]+") + +func newGitPlugin(ref flake.Ref) (*gitPlugin, error) { + plugin := &gitPlugin{ref: ref} + // For backward compatibility, we don't strictly require name to be present + // in github plugins. If it's missing, we just use the directory as the name. + name, err := getPluginNameFromContent(plugin) + + if err != nil && !errors.Is(err, errNameMissing) { + return nil, err + } + + if name == "" { + name = strings.ReplaceAll(ref.Dir, "/", "-") + } + + // gitlab repos can have up to 20 subgroups. We need to capture the subgroups in the plugin name + repoDotted := strings.ReplaceAll(ref.Repo, "/", ".") + + plugin.name = githubNameRegexp.ReplaceAllString( + strings.Join(lo.Compact([]string{ref.Owner, repoDotted, name}), "."), + " ", + ) + + return plugin, nil +} + +func (p *gitPlugin) Fetch() ([]byte, error) { + content, err := p.FileContent(pluginConfigName) + if err != nil { + return nil, err + } + return jsonPurifyPluginContent(content) +} + +func (p *gitPlugin) CanonicalName() string { + return p.name +} + +func (p *gitPlugin) Hash() string { + return cachehash.Bytes([]byte(p.ref.String())) +} + +func (p *gitPlugin) fetchSSHArchive(location string) ([]byte, error) { + archiveDir, _ := os.MkdirTemp("", p.ref.Repo) + archive := filepath.Join(archiveDir, p.ref.Owner+".tar.gz") + args := strings.Fields(location + archive) // this is really just the base git archive command + file + + defer os.RemoveAll(archiveDir) + + cmd := exec.Command(args[0], args[1:]...) + _, err := cmd.Output() + + if err != nil { + slog.Error("Error executing git archive: " + err.Error()) + return nil, err + } + + reader, err := os.Open(archive) + err = fileutil.Untar(reader, archiveDir) + + if err != nil { + slog.Error("Encountered error while trying to extract " + archive + ": " + err.Error()) + return nil, err + } + + pluginJson := filepath.Join(archiveDir, p.ref.Dir, "plugin.json") + file, err := os.Open(pluginJson) + + defer file.Close() + info, err := file.Stat() + + if err != nil { + slog.Error("Error extracting file " + file.Name() + ". Cannot process plugin.") + return nil, err + } + + if info.Size() == 0 { + slog.Error("Extracted file " + file.Name() + " is empty. Cannot process plugin.") + return nil, err + } + + return io.ReadAll(file) +} + +func (p *gitPlugin) fetchHttp(location string) ([]byte, error) { + req, err := p.request(location) + + if err != nil { + return nil, err + } + + client := &http.Client{} + res, err := client.Do(req) + + if err != nil { + return nil, err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, usererr.New( + "failed to get plugin %s @ %s (Status code %d). \nPlease make "+ + "sure a plugin.json file exists in plugin directory.", + p.LockfileKey(), + req.URL.String(), + res.StatusCode, + ) + } + + return io.ReadAll(res.Body) +} + +func (p *gitPlugin) FileContent(subpath string) ([]byte, error) { + location, err := p.url(subpath) + + if err != nil { + return nil, err + } + + process := func() ([]byte, time.Duration, error) { + var bytes []byte + + if p.ref.Type == flake.TypeSSH { + bytes, err = p.fetchSSHArchive(location) + } else { + bytes, err = p.fetchHttp(location) + } + + if err != nil { + return nil, 0, err + } + + // Cache for 24 hours. Once we store the plugin in the lockfile, we + // should cache this indefinitely and only invalidate if the plugin + // is updated. + return bytes, 24 * time.Hour, nil + } + + switch p.ref.Type { + case flake.TypeSSH, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket, flake.TypeGit: + return gitCache.GetOrSet(location, process) + default: + slog.Error("Unable to handle flake ref type: " + p.ref.Type) + return nil, err + } +} + +func (p *gitPlugin) url(subpath string) (string, error) { + switch p.ref.Type { + case flake.TypeSSH: + return p.sshBaseGitCommand() + case flake.TypeGit, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: + return p.repoUrl(subpath) + default: + return "", errors.New("Unsupported plugin type: " + p.ref.Type) + } +} + +func (p *gitPlugin) sshBaseGitCommand() (string, error) { + defaultBranch := "main" + + if p.ref.Host == flake.TypeGitHub+".com" { + // using master for GitHub repos for the same reasoning established in `githubUrl` + defaultBranch = "master" + } + + prefix := "git archive --format=tar.gz --remote=ssh://git@" + path, _ := url.JoinPath(p.ref.Owner, p.ref.Repo) + branch := cmp.Or(p.ref.Rev, p.ref.Ref, defaultBranch) + host := p.ref.Host + + // the Ref struct defaults the field to 0. This technically a valid port for UDP, but we aren't using UDP + if p.ref.Port > 0 { + host += ":" + fmt.Sprintf("%d", p.ref.Port) + } + + command := fmt.Sprintf("%s%s/%s %s", prefix, host, path, branch) + if p.ref.Dir != "" { + command += fmt.Sprintf(" %s", p.ref.Dir) + } + command += " -o" + + slog.Debug("Generated base git archive command: " + command) + return command, nil +} + +func (p *gitPlugin) githubUrl(subpath string) (string, error) { + // Github redirects "master" to "main" in new repos. They don't do the reverse + // so setting master here is better. + return url.JoinPath( + "https://raw.githubusercontent.com/", + p.ref.Owner, + p.ref.Repo, + cmp.Or(p.ref.Rev, p.ref.Ref, "master"), + p.ref.Dir, + subpath, + ) +} + +func (p *gitPlugin) bitbucketUrl(subpath string) (string, error) { + // bitbucket doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case + + return url.JoinPath( + "https://api.bitbucket.org/2.0/repositories", + p.ref.Owner, + p.ref.Repo, + "src", + cmp.Or(p.ref.Rev, p.ref.Ref, "main"), + p.ref.Dir, + subpath, + ) +} + +func (p *gitPlugin) genericGitUrl(subpath string) (string, error) { + address, err := url.JoinPath( + p.ref.Host, + p.ref.Repo, + cmp.Or(p.ref.Rev, p.ref.Ref, "main"), + p.ref.Dir, + subpath, + ) + + if err != nil { + return "", err + } + + parsed, err := url.Parse(address) + + if err != nil { + return "", err + } + + query := parsed.Query() + + if p.ref.Dir != "" { + query.Add("dir", p.ref.Dir) + } + + if p.ref.Port != 0 { + query.Add("port", fmt.Sprintf("%d", p.ref.Port)) + } + + // gitlab doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case + query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) + parsed.RawQuery = query.Encode() + + return parsed.String(), nil +} + +func (p *gitPlugin) repoUrl(subpath string) (string, error) { + if p.ref.Type == flake.TypeGitHub { + return p.githubUrl(subpath) + } else if p.ref.Type == flake.TypeGitLab { + return p.gitlabUrl(subpath) + } else if p.ref.Type == flake.TypeBitBucket { + return p.bitbucketUrl(subpath) + } else if p.ref.Type == flake.TypeGit { + return p.genericGitUrl(subpath) + } + + return "", errors.New("Unknown hostname provided in plugin: " + p.ref.Host) +} + +func (p *gitPlugin) gitlabUrl(subpath string) (string, error) { + file, err := url.JoinPath(p.ref.Dir, subpath) + + if err != nil { + return "", err + } + + repoPath, err := url.JoinPath(p.ref.Owner, p.ref.Repo) + + if err != nil { + return "", err + } + + path, err := url.JoinPath( + "https://gitlab.com/api/v4/projects", + url.PathEscape(repoPath), + "repository", + "files", + url.PathEscape(file), + "raw", + ) + + if err != nil { + return "", err + } + + parsed, err := url.Parse(path) + + if err != nil { + return "", err + } + + // gitlab doesn't redirect master -> main or main -> master, so using "main" + // as the default in this case + query := parsed.Query() + query.Add("ref", cmp.Or(p.ref.Rev, p.ref.Ref, "main")) + parsed.RawQuery = query.Encode() + + return parsed.String(), nil +} + +func (p *gitPlugin) request(contentURL string) (*http.Request, error) { + req, err := http.NewRequest(http.MethodGet, contentURL, nil) + + if err != nil { + return nil, err + } + + // Add github token to request if available + ghToken := os.Getenv("GITHUB_TOKEN") + + if ghToken != "" { + authValue := fmt.Sprintf("token %s", ghToken) + req.Header.Add("Authorization", authValue) + } + + return req, nil +} + +func (p *gitPlugin) LockfileKey() string { + return p.ref.String() +} diff --git a/internal/plugin/git_test.go b/internal/plugin/git_test.go new file mode 100644 index 00000000000..0439978a6b1 --- /dev/null +++ b/internal/plugin/git_test.go @@ -0,0 +1,593 @@ +package plugin + +import ( + "fmt" + "strings" + "testing" + + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "go.jetify.com/devbox/nix/flake" +) + +func TestNewGitPlugin(t *testing.T) { + testCases := []struct { + name string + Include []flake.Ref + expected gitPlugin + expectedURL string + }{ + { + name: "parse basic github plugin", + Include: []flake.Ref{ + { + Type: "github", + Owner: "jetify-com", + Repo: "devbox-plugins", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "github", + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + }, + name: "jetify-com.devbox-plugins", + }, + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/master", + }, + { + name: "parse github plugin with dir param", + Include: []flake.Ref{ + { + Type: "github", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "github", + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + }, + name: "jetify-com.devbox-plugins.mongodb", + }, + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/master/mongodb", + }, + { + name: "parse github plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "github", + Owner: "jetify-com", + Repo: "devbox-plugins", + Ref: "my-branch", + Dir: "mongodb", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "github", + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Ref: "my-branch", + Dir: "mongodb", + }, + name: "jetify-com.devbox-plugins.mongodb", + }, + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/my-branch/mongodb", + }, + { + name: "parse github plugin with dir param and ref", + Include: []flake.Ref{ + { + Type: "github", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + Ref: "initials/my-branch", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "github", + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + Ref: "initials/my-branch", + }, + name: "jetify-com.devbox-plugins.mongodb", + }, + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/my-branch/mongodb", + }, + { + name: "parse github plugin with dir param, rev, and ref", + Include: []flake.Ref{ + { + Type: "github", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + Ref: "initials/my-branch", + Rev: "initials", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "github", + Host: "github.com", + Owner: "jetify-com", + Repo: "devbox-plugins", + Dir: "mongodb", + Ref: "initials/my-branch", // Rev takes precendence over Ref; we exclude the Ref in the URL based on original useage of cmp.Or + Rev: "initials", + }, + name: "jetify-com.devbox-plugins.mongodb", + }, + expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/mongodb", + }, + { + name: "parse basic gitlab plugin", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/raw?ref=main", + }, + { + name: "parse gitlab plugin with dir param", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=main", + }, + { + name: "parse gitlab plugin with dir param and ref", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=some%2Fbranch", + }, + { + name: "parse gitlab plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Rev: "1234567", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Rev: "1234567", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=1234567", + }, + { + name: "parse gitlab plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "gitlab", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + Rev: "1234567", + }, + }, + expected: gitPlugin{ + ref: flake.Ref{ + Type: "gitlab", + Host: "gitlab.com", + Owner: "username", + Repo: "my-plugin", + Dir: "mongodb", + Ref: "some/branch", + Rev: "1234567", + }, + name: "username.my-plugin.mongodb", + }, + expectedURL: "https://gitlab.com/api/v4/projects/username%2Fmy-plugin/repository/files/mongodb/raw?ref=1234567", + }, + { + name: "parse basic bitbucket plugin", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/main", + }, + { + name: "parse bitbucket plugin with dir param", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/main/subdir", + }, + { + name: "parse bitbucket plugin with dir param and ref", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/some/branch/subdir", + }, + { + name: "parse bitbucket plugin with dir param and rev", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/1234567/subdir", + }, + { + name: "parse bitbucket plugin with dir param, ref and rev", + Include: []flake.Ref{ + { + Type: "bitbucket", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "bitbucket", + Host: "bitbucket.com", + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "https://api.bitbucket.org/2.0/repositories/username/my-plugin/src/1234567/subdir", + }, + { + name: "parse basic ssh plugin", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost/username/my-plugin main -o", + }, + { + name: "parse ssh plugin with port", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + }, + name: "username.my-plugin", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin main -o", + }, + { + name: "parse ssh plugin with port and dir", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin main subdir -o", + }, + { + name: "parse ssh plugin with port, dir and rev", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin 1234567 subdir -o", + }, + { + name: "parse ssh plugin with port, dir and ref", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin some/branch subdir -o", + }, + { + name: "parse ssh plugin with port, dir, ref and ref", + Include: []flake.Ref{ + { + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + }, + + expected: gitPlugin{ + ref: flake.Ref{ + Type: "ssh", + Host: "localhost", + Port: 9999, + Owner: "username", + Repo: "my-plugin", + Dir: "subdir", + Ref: "some/branch", + Rev: "1234567", + }, + name: "username.my-plugin.subdir", + }, + expectedURL: "git archive --format=tar.gz --remote=ssh://git@localhost:9999/username/my-plugin 1234567 subdir -o", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + actual, err := newGitPluginForTest(testCase.Include[0]) // FIXME: need to evaluate URL + assert.NoError(t, err) + assert.Equal(t, &testCase.expected, actual) + u, err := testCase.expected.url("") + assert.Nil(t, err) + assert.Equal(t, testCase.expectedURL, u) + }) + } +} + +func newGitPluginForTest(ref flake.Ref) (*gitPlugin, error) { + // added because this occurs much earlier in processing within `internal/devconfig/config.go` + switch ref.Type { + case flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket: + ref.Host = fmt.Sprintf("%s.com", ref.Type) + } + + plugin := &gitPlugin{ref: ref} + name := strings.ReplaceAll(ref.Dir, "/", "-") + repoDotted := strings.ReplaceAll(ref.Repo, "/", ".") + plugin.name = githubNameRegexp.ReplaceAllString( + strings.Join(lo.Compact([]string{ref.Owner, repoDotted, name}), "."), + " ", + ) + return plugin, nil +} + +func TestGitPluginAuth(t *testing.T) { + gitPlugin := gitPlugin{ + ref: flake.Ref{ + Type: "github", + Owner: "jetpack-io", + Repo: "devbox-plugins", + }, + name: "jetpack-io.devbox-plugins", + } + + expectedURL := "https://raw.githubusercontent.com/jetpack-io/devbox-plugins/master/test" + + t.Run("generate request for public Github repository", func(t *testing.T) { + url, err := gitPlugin.url("test") + assert.NoError(t, err) + actual, err := gitPlugin.request(url) + assert.NoError(t, err) + assert.Equal(t, expectedURL, actual.URL.String()) + assert.Equal(t, "", actual.Header.Get("Authorization")) + }) + + t.Run("generate request for private Github repository", func(t *testing.T) { + t.Setenv("GITHUB_TOKEN", "gh_abcd") + url, err := gitPlugin.url("test") + assert.NoError(t, err) + actual, err := gitPlugin.request(url) + assert.NoError(t, err) + assert.Equal(t, expectedURL, actual.URL.String()) + assert.Equal(t, "token gh_abcd", actual.Header.Get("Authorization")) + }) +} diff --git a/internal/plugin/github.go b/internal/plugin/github.go deleted file mode 100644 index b3c14f9f2c3..00000000000 --- a/internal/plugin/github.go +++ /dev/null @@ -1,190 +0,0 @@ -package plugin - -import ( - "cmp" - "fmt" - "io" - "log/slog" - "net/http" - "net/url" - "os" - "regexp" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/samber/lo" - "go.jetify.com/devbox/internal/boxcli/usererr" - "go.jetify.com/devbox/internal/cachehash" - "go.jetify.com/devbox/nix/flake" - "go.jetify.com/pkg/filecache" -) - -var githubCache = filecache.New[[]byte]("devbox/plugin/github") - -type githubPlugin struct { - ref flake.Ref - name string -} - -// Github only allows alphanumeric, hyphen, underscore, and period in repo names. -// but we clean up just in case. -var githubNameRegexp = regexp.MustCompile("[^a-zA-Z0-9-_.]+") - -func newGithubPlugin(ref flake.Ref) (*githubPlugin, error) { - plugin := &githubPlugin{ref: ref} - // For backward compatibility, we don't strictly require name to be present - // in github plugins. If it's missing, we just use the directory as the name. - name, err := getPluginNameFromContent(plugin) - if err != nil && !errors.Is(err, errNameMissing) { - return nil, err - } - if name == "" { - name = strings.ReplaceAll(ref.Dir, "/", "-") - } - plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), - " ", - ) - return plugin, nil -} - -func (p *githubPlugin) Fetch() ([]byte, error) { - content, err := p.FileContent(pluginConfigName) - if err != nil { - return nil, err - } - return jsonPurifyPluginContent(content) -} - -func (p *githubPlugin) CanonicalName() string { - return p.name -} - -func (p *githubPlugin) Hash() string { - return cachehash.Bytes([]byte(p.ref.String())) -} - -func (p *githubPlugin) FileContent(subpath string) ([]byte, error) { - contentURL, err := p.url(subpath) - if err != nil { - return nil, err - } - - // Cache for 24 hours. Once we store the plugin in the lockfile, we - // should cache this indefinitely and only invalidate if the plugin - // is updated. - ttl := 24 * time.Hour - - // This is a stopgap until plugin is stored in lockfile. - // DEVBOX_X indicates this is an experimental env var. - // Use DEVBOX_X_GITHUB_PLUGIN_CACHE_TTL to override the default TTL. - // e.g. DEVBOX_X_GITHUB_PLUGIN_CACHE_TTL=1h will cache the plugin for 1 hour. - // Note: If you want to disable cache, we recommend using a low second value instead of zero to - // ensure only one network request is made. - ttlStr := os.Getenv("DEVBOX_X_GITHUB_PLUGIN_CACHE_TTL") - if ttlStr != "" { - ttl, err = time.ParseDuration(ttlStr) - if err != nil { - return nil, err - } - } - - return githubCache.GetOrSet( - contentURL+ttlStr, - func() ([]byte, time.Duration, error) { - req, err := p.request(contentURL) - if err != nil { - return nil, 0, err - } - - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - return nil, 0, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - authInfo := "No auth header was sent with this request." - if req.Header.Get("Authorization") != "" { - authInfo = fmt.Sprintf( - "The auth header `%s` was sent with this request.", - getRedactedAuthHeader(req), - ) - } - return nil, 0, usererr.New( - "failed to get plugin %s @ %s (Status code %d).\n%s\nPlease make "+ - "sure a plugin.json file exists in plugin directory.", - p.LockfileKey(), - req.URL.String(), - res.StatusCode, - authInfo, - ) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, 0, err - } - - return body, ttl, nil - }, - ) -} - -func (p *githubPlugin) url(subpath string) (string, error) { - // Github redirects "master" to "main" in new repos. They don't do the reverse - // so setting master here is better. - return url.JoinPath( - "https://raw.githubusercontent.com/", - p.ref.Owner, - p.ref.Repo, - cmp.Or(p.ref.Rev, p.ref.Ref, "master"), - p.ref.Dir, - subpath, - ) -} - -func (p *githubPlugin) request(contentURL string) (*http.Request, error) { - req, err := http.NewRequest(http.MethodGet, contentURL, nil) - if err != nil { - return nil, err - } - - // Add github token to request if available - ghToken := os.Getenv("GITHUB_TOKEN") - - if ghToken != "" { - authValue := fmt.Sprintf("token %s", ghToken) - req.Header.Add("Authorization", authValue) - slog.Debug( - "GITHUB_TOKEN env var found, adding to request's auth header", - "headerValue", - getRedactedAuthHeader(req), - ) - } - - return req, nil -} - -func (p *githubPlugin) LockfileKey() string { - return p.ref.String() -} - -func getRedactedAuthHeader(req *http.Request) string { - authHeader := req.Header.Get("Authorization") - parts := strings.SplitN(authHeader, " ", 2) - - if len(authHeader) < 10 || len(parts) < 2 { - // too short to safely reveal any part - return strings.Repeat("*", len(authHeader)) - } - - authType, token := parts[0], parts[1] - if len(token) < 10 { - // second word too short to reveal any, but show first word - return authType + " " + strings.Repeat("*", len(token)) - } - - // show first 4 chars of token to help with debugging (will often be "ghp_") - return authType + " " + token[:4] + strings.Repeat("*", len(token)-4) -} diff --git a/internal/plugin/github_test.go b/internal/plugin/github_test.go deleted file mode 100644 index d51e698b0ef..00000000000 --- a/internal/plugin/github_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package plugin - -import ( - "net/http" - "strings" - "testing" - - "github.com/samber/lo" - "github.com/stretchr/testify/assert" - "go.jetify.com/devbox/nix/flake" -) - -func TestNewGithubPlugin(t *testing.T) { - testCases := []struct { - name string - Include string - expected githubPlugin - expectedURL string - }{ - { - name: "parse basic github plugin", - Include: "github:jetify-com/devbox-plugins", - expected: githubPlugin{ - ref: flake.Ref{ - Type: "github", - Owner: "jetify-com", - Repo: "devbox-plugins", - }, - name: "jetify-com.devbox-plugins", - }, - expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/master", - }, - { - name: "parse github plugin with dir param", - Include: "github:jetify-com/devbox-plugins?dir=mongodb", - expected: githubPlugin{ - ref: flake.Ref{ - Type: "github", - Owner: "jetify-com", - Repo: "devbox-plugins", - Dir: "mongodb", - }, - name: "jetify-com.devbox-plugins.mongodb", - }, - expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/master/mongodb", - }, - { - name: "parse github plugin with dir param and rev", - Include: "github:jetify-com/devbox-plugins/my-branch?dir=mongodb", - expected: githubPlugin{ - ref: flake.Ref{ - Type: "github", - Owner: "jetify-com", - Repo: "devbox-plugins", - Ref: "my-branch", - Dir: "mongodb", - }, - name: "jetify-com.devbox-plugins.mongodb", - }, - expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/my-branch/mongodb", - }, - { - name: "parse github plugin with dir param and rev", - Include: "github:jetify-com/devbox-plugins/initials/my-branch?dir=mongodb", - expected: githubPlugin{ - ref: flake.Ref{ - Type: "github", - Owner: "jetify-com", - Repo: "devbox-plugins", - Ref: "initials/my-branch", - Dir: "mongodb", - }, - name: "jetify-com.devbox-plugins.mongodb", - }, - expectedURL: "https://raw.githubusercontent.com/jetify-com/devbox-plugins/initials/my-branch/mongodb", - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - actual, err := newGithubPluginForTest(testCase.Include) - assert.NoError(t, err) - assert.Equal(t, &testCase.expected, actual) - u, err := testCase.expected.url("") - assert.Nil(t, err) - assert.Equal(t, testCase.expectedURL, u) - }) - } -} - -// keep in sync with newGithubPlugin -func newGithubPluginForTest(include string) (*githubPlugin, error) { - ref, err := flake.ParseRef(include) - if err != nil { - return nil, err - } - - plugin := &githubPlugin{ref: ref} - name := strings.ReplaceAll(ref.Dir, "/", "-") - plugin.name = githubNameRegexp.ReplaceAllString( - strings.Join(lo.Compact([]string{ref.Owner, ref.Repo, name}), "."), - " ", - ) - return plugin, nil -} - -func TestGithubPluginAuth(t *testing.T) { - githubPlugin := githubPlugin{ - ref: flake.Ref{ - Type: "github", - Owner: "jetpack-io", - Repo: "devbox-plugins", - }, - name: "jetpack-io.devbox-plugins", - } - - expectedURL := "https://raw.githubusercontent.com/jetpack-io/devbox-plugins/master/test" - - t.Run("generate request for public Github repository", func(t *testing.T) { - t.Setenv("GITHUB_TOKEN", "") - url, err := githubPlugin.url("test") - assert.NoError(t, err) - actual, err := githubPlugin.request(url) - assert.NoError(t, err) - assert.Equal(t, expectedURL, actual.URL.String()) - assert.Equal(t, "", actual.Header.Get("Authorization")) - }) - - t.Run("generate request for private Github repository", func(t *testing.T) { - t.Setenv("GITHUB_TOKEN", "gh_abcd") - url, err := githubPlugin.url("test") - assert.NoError(t, err) - actual, err := githubPlugin.request(url) - assert.NoError(t, err) - assert.Equal(t, expectedURL, actual.URL.String()) - assert.Equal(t, "token gh_abcd", actual.Header.Get("Authorization")) - }) -} - -func TestGetRedactedAuthHeader(t *testing.T) { - testCases := []struct { - name string - authHeader string - expected string - }{ - { - "normal length token partially readable for debugging", - "token ghp_61b296fb898349778e20532cb65ce38e", - "token ghp_********************************", - }, - { - "short token redacted", - "token ghp_61b29", - "token *********", - }, - { - "short header fully redacted", - "token xyz", - "*********", - }, - { - "no token returns empty string", - "", - "", - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) - assert.NoError(t, err) - req.Header.Add("Authorization", testCase.authHeader) - assert.Equal(t, testCase.expected, getRedactedAuthHeader(req)) - }) - } -} diff --git a/internal/plugin/includable.go b/internal/plugin/includable.go index 84984e23551..f4770f77ac7 100644 --- a/internal/plugin/includable.go +++ b/internal/plugin/includable.go @@ -16,16 +16,12 @@ type Includable interface { LockfileKey() string } -func parseIncludable(includableRef, workingDir string) (Includable, error) { - ref, err := flake.ParseRef(includableRef) - if err != nil { - return nil, err - } +func parseIncludable(ref flake.Ref, workingDir string) (Includable, error) { switch ref.Type { case flake.TypePath: return newLocalPlugin(ref, workingDir) - case flake.TypeGitHub: - return newGithubPlugin(ref) + case flake.TypeSSH, flake.TypeBuiltin, flake.TypeGitHub, flake.TypeGitLab, flake.TypeBitBucket, flake.TypeGit: + return newGitPlugin(ref) default: return nil, fmt.Errorf("unsupported ref type %q", ref.Type) } diff --git a/internal/plugin/includes.go b/internal/plugin/includes.go index 28572b8df53..5b55522198f 100644 --- a/internal/plugin/includes.go +++ b/internal/plugin/includes.go @@ -1,22 +1,19 @@ package plugin import ( - "strings" - "go.jetify.com/devbox/internal/devpkg" "go.jetify.com/devbox/internal/lock" + "go.jetify.com/devbox/nix/flake" ) -func LoadConfigFromInclude(include string, lockfile *lock.File, workingDir string) (*Config, error) { +func LoadConfigFromInclude(ref flake.Ref, lockfile *lock.File, workingDir string) (*Config, error) { var includable Includable var err error - if t, name, _ := strings.Cut(include, ":"); t == "plugin" { - includable = devpkg.PackageFromStringWithDefaults( - name, - lockfile, - ) + + if ref.Type == flake.TypeBuiltin { + includable = devpkg.PackageFromStringWithDefaults(ref.Path, lockfile) } else { - includable, err = parseIncludable(include, workingDir) + includable, err = parseIncludable(ref, workingDir) if err != nil { return nil, err } diff --git a/internal/plugin/local.go b/internal/plugin/local.go index 56eeea96bca..d45739a528a 100644 --- a/internal/plugin/local.go +++ b/internal/plugin/local.go @@ -19,9 +19,11 @@ type LocalPlugin struct { func newLocalPlugin(ref flake.Ref, pluginDir string) (*LocalPlugin, error) { plugin := &LocalPlugin{ref: ref, pluginDir: pluginDir} name, err := getPluginNameFromContent(plugin) + if err != nil { return nil, err } + plugin.name = name return plugin, nil } diff --git a/internal/plugin/update.go b/internal/plugin/update.go index 7d7e521ee2e..a3e824db385 100644 --- a/internal/plugin/update.go +++ b/internal/plugin/update.go @@ -1,5 +1,5 @@ package plugin func Update() error { - return githubCache.Clear() + return gitCache.Clear() } diff --git a/nix/flake/flakeref.go b/nix/flake/flakeref.go index 6e9f545959b..270cc5880b1 100644 --- a/nix/flake/flakeref.go +++ b/nix/flake/flakeref.go @@ -14,12 +14,17 @@ import ( // Flake reference types supported by this package. const ( - TypeIndirect = "indirect" - TypePath = "path" - TypeFile = "file" - TypeGit = "git" - TypeGitHub = "github" - TypeTarball = "tarball" + TypeIndirect = "indirect" + TypePath = "path" + TypeHttps = "https" + TypeFile = "file" + TypeSSH = "ssh" + TypeGitHub = "github" + TypeGitLab = "gitlab" + TypeGit = "git" + TypeBitBucket = "bitbucket" + TypeTarball = "tarball" + TypeBuiltin = "builtin" ) // Ref is a parsed Nix flake reference. A flake reference is a subset of the @@ -77,6 +82,9 @@ type Ref struct { // LastModified is the last modification time of the flake. LastModified int64 `json:"lastModified,omitempty"` + + // Port of the server git server, to support privately hosted git servers or tunnels + Port int32 `json:port,omitempty` } // ParseRef parses a raw flake reference. Nix supports a variety of flake ref @@ -512,6 +520,8 @@ func isArchive(path string) bool { // ensuring that path elements with an encoded '/' (%2F) are not split. // For example, "/dir/file%2Fname" becomes the elements "dir" and "file/name". // The count limits the number of substrings per [strings.SplitN] + +// TODO git rid of this func splitPathOrOpaque(u *url.URL, n int) ([]string, error) { upath := u.EscapedPath() if upath == "" { @@ -539,6 +549,31 @@ func splitPathOrOpaque(u *url.URL, n int) ([]string, error) { return split, nil } +// TODO maybe use this? +func splitRepoString(repo string, n int) ([]string, error) { + repo = strings.TrimSpace(repo) + + if repo == "" { + return nil, nil + } + + // We don't want an empty element if the path is rooted. + if repo[0] == '/' { + repo = repo[1:] + } + repo = path.Clean(repo) + + var err error + split := strings.SplitN(repo, "/", n) + for i := range split { + split[i], err = url.PathUnescape(split[i]) + if err != nil { + return nil, err + } + } + return split, nil +} + // buildEscapedPath escapes and joins path elements for a URL flake ref. The // resulting path is cleaned according to url.JoinPath. func buildEscapedPath(elem ...string) string { @@ -657,7 +692,13 @@ func ParseInstallable(raw string) (Installable, error) { // Interpret installables with path-style flake refs as URLs to extract // the attribute path (fragment). This means that path-style flake refs - // cannot point to files with a '#' or '?' in their name, since those + // + // + // + // + // + // + //// cannot point to files with a '#' or '?' in their name, since those // would be parsed as the URL fragment or query string. This mimic's // Nix's CLI behavior. if raw[0] == '.' || raw[0] == '/' { diff --git a/testscripts/plugin/plugin.cycle.test.txt b/testscripts/plugin/plugin.cycle.test.txt index 5ddff0f7e4a..422c54a7618 100644 --- a/testscripts/plugin/plugin.cycle.test.txt +++ b/testscripts/plugin/plugin.cycle.test.txt @@ -10,25 +10,25 @@ stderr 'Finished installing packages.' -- devbox.json -- { "name": "test-with-cycle", - "include": ["./plugin1"] + "include": [{"type": "path", "path": "./plugin1"}] } -- plugin1/plugin.json -- { "name": "plugin1", - "include": ["../plugin2"] + "include": [{"type": "path", "path": "../plugin2"}] } -- plugin2/plugin.json -- { "name": "plugin2", - "include": ["../plugin1"] + "include": [{"type": "path", "path": "../plugin1"}] } -- no-cycle/devbox.json -- { "name": "test-without-cycle", - "include": ["./plugin3"] + "include": [{"type": "path", "path": "./plugin3"}] } -- no-cycle/plugin3/plugin.json -- @@ -40,8 +40,8 @@ stderr 'Finished installing packages.' { "name": "test-with-duplicate", "include": [ - "./plugin4", - "./plugin4" + {"type": "path", "path": "./plugin4/plugin.json"}, + {"type": "path", "path": "./plugin4"}, ] }