From 979690dd19a866a6c0f113830688ddd76f78eff8 Mon Sep 17 00:00:00 2001 From: Joonas Rautiola Date: Mon, 9 Dec 2024 13:12:39 +0200 Subject: [PATCH] Revert "Use rclone from nixpkgs now that it's >1.68.0" This reverts commit 8a42b2b74b9622fce8c9cfd3eaf672c581d69833. --- .../jenkins-controller/configuration.nix | 7 +- pkgs/rclone/default.nix | 13 ++ pkgs/rclone/http-socket-activation.patch | 192 ++++++++++++++++++ .../webdav-introduce-unix_socket_path.patch | 139 +++++++++++++ services/rclone-http/default.nix | 3 +- 5 files changed, 350 insertions(+), 4 deletions(-) create mode 100644 pkgs/rclone/default.nix create mode 100644 pkgs/rclone/http-socket-activation.patch create mode 100644 pkgs/rclone/webdav-introduce-unix_socket_path.patch diff --git a/hosts/azure/jenkins-controller/configuration.nix b/hosts/azure/jenkins-controller/configuration.nix index 78d64f61..9512fd0c 100644 --- a/hosts/azure/jenkins-controller/configuration.nix +++ b/hosts/azure/jenkins-controller/configuration.nix @@ -69,6 +69,7 @@ let s = client.get_secret(secret_name) print(s.value) ''; + rclone = pkgs.callPackage ../../../pkgs/rclone { }; in { imports = [ @@ -127,9 +128,9 @@ in csvkit curl nix-eval-jobs - rclone # used to copy artifacts ] ++ [ + rclone # used to copy artifacts inputs.sbomnix.packages.${pkgs.system}.sbomnix # sbomnix, provenance, vulnxscan inputs.ci-yubi.packages.${pkgs.system}.sigver # signing scripts ]; @@ -341,7 +342,7 @@ in RuntimeDirectory = "rclone-http"; EnvironmentFile = "/var/lib/rclone-jenkins-artifacts/env"; ExecStart = lib.concatStringsSep " " [ - "${pkgs.rclone}/bin/rclone" + "${rclone}/bin/rclone" "serve" "webdav" "--dir-cache-time" @@ -374,7 +375,7 @@ in RuntimeDirectory = "rclone-http"; EnvironmentFile = "/var/lib/rclone-jenkins-artifacts/env"; ExecStart = lib.concatStringsSep " " [ - "${pkgs.rclone}/bin/rclone" + "${rclone}/bin/rclone" "serve" "http" "--read-only" diff --git a/pkgs/rclone/default.nix b/pkgs/rclone/default.nix new file mode 100644 index 00000000..a65bd13d --- /dev/null +++ b/pkgs/rclone/default.nix @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors +# SPDX-License-Identifier: Apache-2.0 +{ pkgs, ... }: +pkgs.rclone.overrideAttrs (oldAttrs: { + # These will be included in rclone v1.68.0 + patches = (oldAttrs.patches or [ ]) ++ [ + # https://github.com/rclone/rclone/pull/7801 + ./http-socket-activation.patch + + # https://github.com/rclone/rclone/pull/7865 + ./webdav-introduce-unix_socket_path.patch + ]; +}) diff --git a/pkgs/rclone/http-socket-activation.patch b/pkgs/rclone/http-socket-activation.patch new file mode 100644 index 00000000..8d43eae7 --- /dev/null +++ b/pkgs/rclone/http-socket-activation.patch @@ -0,0 +1,192 @@ +From 283c3e00a7feb5bb302280e77504ca6a3452ef88 Mon Sep 17 00:00:00 2001 +From: Florian Klink +Date: Wed, 24 Apr 2024 15:37:07 +0300 +Subject: [PATCH 1/3] http: support listening on passed FDs +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Instead of the listening addresses specified above, rclone will listen to all +FDs passed by the service manager, if any (and ignore any arguments passed by +`--{{ .Prefix }}addr`. + +This allows rclone to be a socket-activated service. It can be configured as described in +https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html + +It's possible to test this interactively through `systemd-socket-activate`, +firing of a request in a second terminal: + +``` +❯ systemd-socket-activate -l 8088 -l 8089 --fdname=foo:bar -- ./rclone serve webdav :local:test/ +Listening on [::]:8088 as 3. +Listening on [::]:8089 as 4. +Communication attempt on fd 3. +Execing ./rclone (./rclone serve webdav :local:test/) +2024/04/24 18:14:42 NOTICE: Local file system at /home/flokli/dev/flokli/rclone/test: WebDav Server started on [sd-listen:bar-0/ sd-listen:foo-0/] +``` +--- + lib/http/server.go | 115 ++++++++++++++++++++++++++++++--------------- + 1 file changed, 78 insertions(+), 37 deletions(-) + +diff --git a/lib/http/server.go b/lib/http/server.go +index 68de85e50..608b5adef 100644 +--- a/lib/http/server.go ++++ b/lib/http/server.go +@@ -18,6 +18,7 @@ import ( + "sync" + "time" + ++ sdActivation "github.com/coreos/go-systemd/v22/activation" + "github.com/go-chi/chi/v5" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/lib/atexit" +@@ -74,6 +75,15 @@ certificate authority certificate. + --{{ .Prefix }}min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). ++ ++### Socket activation ++ ++Instead of the listening addresses specified above, rclone will listen to all ++FDs passed by the service manager, if any (and ignore any arguments passed by ` + ++ "--{{ .Prefix }}addr`" + `). ++ ++This allows rclone to be a socket-activated service. It can be configured as described in ++https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html + ` + tmpl, err := template.New("server help").Parse(help) + if err != nil { +@@ -194,6 +204,32 @@ func WithTemplate(cfg TemplateConfig) Option { + } + } + ++// For a given listener, and optional tlsConfig, construct a instance. ++// The url string ends up in the `url` field of the `instance`. ++// This unconditionally wraps the listener with the provided TLS config if one ++// is specified, so all decision logic on whether to use TLS needs to live at ++// the callsite. ++func newInstance(ctx context.Context, s *Server, listener net.Listener, tlsCfg *tls.Config, url string) *instance { ++ if tlsCfg != nil { ++ listener = tls.NewListener(listener, tlsCfg) ++ } ++ ++ return &instance{ ++ url: url, ++ listener: listener, ++ httpServer: &http.Server{ ++ Handler: s.mux, ++ ReadTimeout: s.cfg.ServerReadTimeout, ++ WriteTimeout: s.cfg.ServerWriteTimeout, ++ MaxHeaderBytes: s.cfg.MaxHeaderBytes, ++ ReadHeaderTimeout: 10 * time.Second, // time to send the headers ++ IdleTimeout: 60 * time.Second, // time to keep idle connections open ++ TLSConfig: tlsCfg, ++ BaseContext: NewBaseContext(ctx, url), ++ }, ++ } ++} ++ + // NewServer instantiates a new http server using provided listeners and options + // This function is provided if the default http server does not meet a services requirements and should not generally be used + // A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443. +@@ -242,55 +278,60 @@ func NewServer(ctx context.Context, options ...Option) (*Server, error) { + + s.initAuth() + ++ // (Only) listen on FDs provided by the service manager, if any. ++ sdListeners, err := sdActivation.ListenersWithNames() ++ if err != nil { ++ return nil, fmt.Errorf("unable to acquire listeners: %w", err) ++ } ++ ++ if len(sdListeners) != 0 { ++ for listenerName, listeners := range sdListeners { ++ for i, listener := range listeners { ++ url := fmt.Sprintf("sd-listen:%s-%d/%s", listenerName, i, s.cfg.BaseURL) ++ if s.tlsConfig != nil { ++ url = fmt.Sprintf("sd-listen+tls:%s-%d/%s", listenerName, i, s.cfg.BaseURL) ++ } ++ ++ instance := newInstance(ctx, s, listener, s.tlsConfig, url) ++ ++ s.instances = append(s.instances, *instance) ++ } ++ } ++ ++ return s, nil ++ } ++ ++ // Process all listeners specified in the CLI Args. + for _, addr := range s.cfg.ListenAddr { +- var url string +- var network = "tcp" +- var tlsCfg *tls.Config ++ var instance *instance + + if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) { +- network = "unix" + addr = strings.TrimPrefix(addr, "unix://") +- url = addr + ++ listener, err := net.Listen("unix", addr) ++ if err != nil { ++ return nil, err ++ } ++ instance = newInstance(ctx, s, listener, s.tlsConfig, addr) + } else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) { +- tlsCfg = s.tlsConfig + addr = strings.TrimPrefix(addr, "tls://") +- } +- +- var listener net.Listener +- if tlsCfg == nil { +- listener, err = net.Listen(network, addr) ++ listener, err := net.Listen("tcp", addr) ++ if err != nil { ++ return nil, err ++ } ++ instance = newInstance(ctx, s, listener, s.tlsConfig, fmt.Sprintf("https://%s%s/", listener.Addr().String(), s.cfg.BaseURL)) + } else { +- listener, err = tls.Listen(network, addr, tlsCfg) +- } +- if err != nil { +- return nil, err +- } +- +- if network == "tcp" { +- var secure string +- if tlsCfg != nil { +- secure = "s" ++ // HTTP case ++ addr = strings.TrimPrefix(addr, "http://") ++ listener, err := net.Listen("tcp", addr) ++ if err != nil { ++ return nil, err + } +- url = fmt.Sprintf("http%s://%s%s/", secure, listener.Addr().String(), s.cfg.BaseURL) +- } ++ instance = newInstance(ctx, s, listener, nil, fmt.Sprintf("http://%s%s/", listener.Addr().String(), s.cfg.BaseURL)) + +- ii := instance{ +- url: url, +- listener: listener, +- httpServer: &http.Server{ +- Handler: s.mux, +- ReadTimeout: s.cfg.ServerReadTimeout, +- WriteTimeout: s.cfg.ServerWriteTimeout, +- MaxHeaderBytes: s.cfg.MaxHeaderBytes, +- ReadHeaderTimeout: 10 * time.Second, // time to send the headers +- IdleTimeout: 60 * time.Second, // time to keep idle connections open +- TLSConfig: tlsCfg, +- BaseContext: NewBaseContext(ctx, url), +- }, + } + +- s.instances = append(s.instances, ii) ++ s.instances = append(s.instances, *instance) + } + + return s, nil +-- +2.44.0 + diff --git a/pkgs/rclone/webdav-introduce-unix_socket_path.patch b/pkgs/rclone/webdav-introduce-unix_socket_path.patch new file mode 100644 index 00000000..16ff5f2c --- /dev/null +++ b/pkgs/rclone/webdav-introduce-unix_socket_path.patch @@ -0,0 +1,139 @@ +From 5e34ec7912a629e1d6561a6cc60c1663be4f4423 Mon Sep 17 00:00:00 2001 +From: Florian Klink +Date: Fri, 24 May 2024 11:45:10 +0200 +Subject: [PATCH] webdav: introduce unix_socket_path + +This adds a new optional parameter to the backend, allowing to specify a +path to a unix domain socket to connect to, instead the specified URL. + +If the parameter is set, we use `fshttp.NewClientCustom` to modify the +HTTP transport, to use a dialer connecting to the unix domain socket +path specified for that backend. + +The URL itself is still used for the rest of the HTTP client, allowing +host and subpath to stay intact. + +This allows using rclone with the webdav backend to connect to a WebDAV +server provided at a Unix Domain socket: + +``` +RCLONE_WEBDAV_UNIX_SOCKET_PATH=/path/to/my.sock \ +RCLONE_WEBDAV_URL=http://localhost \ +rclone sync mydir :webdav:/somewhere +``` +--- + backend/webdav/webdav.go | 19 ++++++++++++++++++- + docs/content/webdav.md | 11 +++++++++++ + fs/config.go | 1 + + fs/fshttp/http.go | 8 +++++++- + 4 files changed, 37 insertions(+), 2 deletions(-) + +diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go +index f1c16f35a..c72875db8 100644 +--- a/backend/webdav/webdav.go ++++ b/backend/webdav/webdav.go +@@ -15,6 +15,7 @@ import ( + "errors" + "fmt" + "io" ++ "net" + "net/http" + "net/url" + "os/exec" +@@ -154,6 +155,11 @@ Set to 0 to disable chunked uploading. + Help: "Exclude ownCloud shares", + Advanced: true, + Default: false, ++ }, { ++ Name: "unix_socket_path", ++ Help: "Path to a unix domain socket to dial to, instead of opening a TCP connection directly", ++ Advanced: true, ++ Default: "", + }}, + }) + } +@@ -171,6 +177,7 @@ type Options struct { + PacerMinSleep fs.Duration `config:"pacer_min_sleep"` + ChunkSize fs.SizeSuffix `config:"nextcloud_chunk_size"` + ExcludeShares bool `config:"owncloud_exclude_shares"` ++ UnixSocketPath string `config:"unix_socket_path"` + } + + // Fs represents a remote webdav +@@ -452,7 +459,17 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e + precision: fs.ModTimeNotSupported, + } + +- client := fshttp.NewClient(ctx) ++ var client *http.Client ++ if opt.UnixSocketPath == "" { ++ client = fshttp.NewClient(ctx) ++ } else { ++ fs.Debugf(f, "custom unix_socket_path configured (%v), updating dialer…", opt.UnixSocketPath) ++ client = fshttp.NewClientCustom(ctx, func(t *http.Transport) { ++ t.DialContext = func(reqCtx context.Context, network, addr string) (net.Conn, error) { ++ return fshttp.NewDialer(ctx).DialContext(reqCtx, "unix", opt.UnixSocketPath) ++ } ++ }) ++ } + if opt.Vendor == "sharepoint-ntlm" { + // Disable transparent HTTP/2 support as per https://golang.org/pkg/net/http/ , + // otherwise any connection to IIS 10.0 fails with 'stream error: stream ID 39; HTTP_1_1_REQUIRED' +diff --git a/docs/content/webdav.md b/docs/content/webdav.md +index 7da008014..855b0e2a0 100644 +--- a/docs/content/webdav.md ++++ b/docs/content/webdav.md +@@ -283,6 +283,17 @@ Properties: + - Type: bool + - Default: false + ++#### --webdav-unix-socket-path ++ ++Path to a unix domain socket to dial to, instead of opening a TCP connection directly ++ ++Properties: ++ ++- Config: unix_socket_path ++- Env Var: RCLONE_WEBDAV_UNIX_SOCKET_PATH ++- Type: string ++- Required: false ++ + #### --webdav-description + + Description of the remote +diff --git a/fs/config.go b/fs/config.go +index 435fbf8e7..5122aae18 100644 +--- a/fs/config.go ++++ b/fs/config.go +@@ -154,6 +154,7 @@ type ConfigInfo struct { + Inplace bool // Download directly to destination file instead of atomic download to temp/rename + PartialSuffix string + MetadataMapper SpaceSepList ++ UnixSocketPath string // Optional unix socket to connect to + } + + // NewConfig creates a new config with everything set to the default +diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go +index 498714dfd..ff756d511 100644 +--- a/fs/fshttp/http.go ++++ b/fs/fshttp/http.go +@@ -127,9 +127,15 @@ func NewTransport(ctx context.Context) http.RoundTripper { + + // NewClient returns an http.Client with the correct timeouts + func NewClient(ctx context.Context) *http.Client { ++ return NewClientCustom(ctx, nil) ++} ++ ++// NewClientCustom returns an http.Client with the correct timeouts. ++// It allows customizing the transport, using NewTransportCustom. ++func NewClientCustom(ctx context.Context, customize func(*http.Transport)) *http.Client { + ci := fs.GetConfig(ctx) + client := &http.Client{ +- Transport: NewTransport(ctx), ++ Transport: NewTransportCustom(ctx, customize), + } + if ci.Cookie { + client.Jar = cookieJar +-- +2.44.0 + diff --git a/services/rclone-http/default.nix b/services/rclone-http/default.nix index b6ac41ab..440dbd88 100644 --- a/services/rclone-http/default.nix +++ b/services/rclone-http/default.nix @@ -9,6 +9,7 @@ with lib; let cfg = config.services.rclone-http; + rclone = pkgs.callPackage ../../pkgs/rclone { }; in { options.services.rclone-http = { @@ -55,7 +56,7 @@ in EnvironmentFile = "/var/lib/rclone-http/env"; ExecStart = concatStringsSep " " ( [ - "${pkgs.rclone}/bin/rclone" + "${rclone}/bin/rclone" "serve" cfg.protocol ]