Revert "Use rclone from nixpkgs now that it's >1.68.0"

This reverts commit 8a42b2b.
tiiuae · Dec 9, 2024 · 979690d · 979690d
1 parent d837595
commit 979690d
Show file tree

Hide file tree

Showing 5 changed files with 350 additions and 4 deletions.
diff --git a/hosts/azure/jenkins-controller/configuration.nix b/hosts/azure/jenkins-controller/configuration.nix
@@ -69,6 +69,7 @@ let
         s = client.get_secret(secret_name)
         print(s.value)
       '';
+  rclone = pkgs.callPackage ../../../pkgs/rclone { };
 in
 {
   imports = [
@@ -127,9 +128,9 @@ in
         csvkit
         curl
         nix-eval-jobs
-        rclone # used to copy artifacts
       ]
       ++ [
+        rclone # used to copy artifacts
         inputs.sbomnix.packages.${pkgs.system}.sbomnix # sbomnix, provenance, vulnxscan
         inputs.ci-yubi.packages.${pkgs.system}.sigver # signing scripts
       ];
@@ -341,7 +342,7 @@ in
       RuntimeDirectory = "rclone-http";
       EnvironmentFile = "/var/lib/rclone-jenkins-artifacts/env";
       ExecStart = lib.concatStringsSep " " [
-        "${pkgs.rclone}/bin/rclone"
+        "${rclone}/bin/rclone"
         "serve"
         "webdav"
         "--dir-cache-time"
@@ -374,7 +375,7 @@ in
       RuntimeDirectory = "rclone-http";
       EnvironmentFile = "/var/lib/rclone-jenkins-artifacts/env";
       ExecStart = lib.concatStringsSep " " [
-        "${pkgs.rclone}/bin/rclone"
+        "${rclone}/bin/rclone"
         "serve"
         "http"
         "--read-only"

diff --git a/pkgs/rclone/default.nix b/pkgs/rclone/default.nix
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors
+# SPDX-License-Identifier: Apache-2.0
+{ pkgs, ... }:
+pkgs.rclone.overrideAttrs (oldAttrs: {
+  # These will be included in rclone v1.68.0
+  patches = (oldAttrs.patches or [ ]) ++ [
+    # https://github.com/rclone/rclone/pull/7801
+    ./http-socket-activation.patch
+
+    # https://github.com/rclone/rclone/pull/7865
+    ./webdav-introduce-unix_socket_path.patch
+  ];
+})
diff --git a/pkgs/rclone/http-socket-activation.patch b/pkgs/rclone/http-socket-activation.patch
@@ -0,0 +1,192 @@
+From 283c3e00a7feb5bb302280e77504ca6a3452ef88 Mon Sep 17 00:00:00 2001
+From: Florian Klink <[email protected]>
+Date: Wed, 24 Apr 2024 15:37:07 +0300
+Subject: [PATCH 1/3] http: support listening on passed FDs
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+Instead of the listening addresses specified above, rclone will listen to all
+FDs passed by the service manager, if any (and ignore any arguments passed by
+`--{{ .Prefix }}addr`.
+
+This allows rclone to be a socket-activated service. It can be configured as described in
+https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html
+
+It's possible to test this interactively through `systemd-socket-activate`,
+firing of a request in a second terminal:
+
+```
+❯ systemd-socket-activate -l 8088 -l 8089 --fdname=foo:bar -- ./rclone serve webdav :local:test/
+Listening on [::]:8088 as 3.
+Listening on [::]:8089 as 4.
+Communication attempt on fd 3.
+Execing ./rclone (./rclone serve webdav :local:test/)
+2024/04/24 18:14:42 NOTICE: Local file system at /home/flokli/dev/flokli/rclone/test: WebDav Server started on [sd-listen:bar-0/ sd-listen:foo-0/]
+```
+---
+ lib/http/server.go | 115 ++++++++++++++++++++++++++++++---------------
+ 1 file changed, 78 insertions(+), 37 deletions(-)
+
+diff --git a/lib/http/server.go b/lib/http/server.go
+index 68de85e50..608b5adef 100644
+--- a/lib/http/server.go
++++ b/lib/http/server.go
+@@ -18,6 +18,7 @@ import (
+ 	"sync"
+ 	"time"
+
++	sdActivation "github.com/coreos/go-systemd/v22/activation"
+ 	"github.com/go-chi/chi/v5"
+ 	"github.com/rclone/rclone/fs/config/flags"
+ 	"github.com/rclone/rclone/lib/atexit"
+@@ -74,6 +75,15 @@ certificate authority certificate.
+ --{{ .Prefix }}min-tls-version is minimum TLS version that is acceptable. Valid
+   values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+   "tls1.0").
++
++### Socket activation
++
++Instead of the listening addresses specified above, rclone will listen to all
++FDs passed by the service manager, if any (and ignore any arguments passed by ` +
++		"--{{ .Prefix }}addr`" + `).
++
++This allows rclone to be a socket-activated service. It can be configured as described in
++https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html
+ `
+ 	tmpl, err := template.New("server help").Parse(help)
+ 	if err != nil {
+@@ -194,6 +204,32 @@ func WithTemplate(cfg TemplateConfig) Option {
+ 	}
+ }
+
++// For a given listener, and optional tlsConfig, construct a instance.
++// The url string ends up in the `url` field of the `instance`.
++// This unconditionally wraps the listener with the provided TLS config if one
++// is specified, so all decision logic on whether to use TLS needs to live at
++// the callsite.
++func newInstance(ctx context.Context, s *Server, listener net.Listener, tlsCfg *tls.Config, url string) *instance {
++	if tlsCfg != nil {
++		listener = tls.NewListener(listener, tlsCfg)
++	}
++
++	return &instance{
++		url:      url,
++		listener: listener,
++		httpServer: &http.Server{
++			Handler:           s.mux,
++			ReadTimeout:       s.cfg.ServerReadTimeout,
++			WriteTimeout:      s.cfg.ServerWriteTimeout,
++			MaxHeaderBytes:    s.cfg.MaxHeaderBytes,
++			ReadHeaderTimeout: 10 * time.Second, // time to send the headers
++			IdleTimeout:       60 * time.Second, // time to keep idle connections open
++			TLSConfig:         tlsCfg,
++			BaseContext:       NewBaseContext(ctx, url),
++		},
++	}
++}
++
+ // NewServer instantiates a new http server using provided listeners and options
+ // This function is provided if the default http server does not meet a services requirements and should not generally be used
+ // A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443.
+@@ -242,55 +278,60 @@ func NewServer(ctx context.Context, options ...Option) (*Server, error) {
+
+ 	s.initAuth()
+
++	// (Only) listen on FDs provided by the service manager, if any.
++	sdListeners, err := sdActivation.ListenersWithNames()
++	if err != nil {
++		return nil, fmt.Errorf("unable to acquire listeners: %w", err)
++	}
++
++	if len(sdListeners) != 0 {
++		for listenerName, listeners := range sdListeners {
++			for i, listener := range listeners {
++				url := fmt.Sprintf("sd-listen:%s-%d/%s", listenerName, i, s.cfg.BaseURL)
++				if s.tlsConfig != nil {
++					url = fmt.Sprintf("sd-listen+tls:%s-%d/%s", listenerName, i, s.cfg.BaseURL)
++				}
++
++				instance := newInstance(ctx, s, listener, s.tlsConfig, url)
++
++				s.instances = append(s.instances, *instance)
++			}
++		}
++
++		return s, nil
++	}
++
++	// Process all listeners specified in the CLI Args.
+ 	for _, addr := range s.cfg.ListenAddr {
+-		var url string
+-		var network = "tcp"
+-		var tlsCfg *tls.Config
++		var instance *instance
+
+ 		if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) {
+-			network = "unix"
+ 			addr = strings.TrimPrefix(addr, "unix://")
+-			url = addr
+
++			listener, err := net.Listen("unix", addr)
++			if err != nil {
++				return nil, err
++			}
++			instance = newInstance(ctx, s, listener, s.tlsConfig, addr)
+ 		} else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) {
+-			tlsCfg = s.tlsConfig
+ 			addr = strings.TrimPrefix(addr, "tls://")
+-		}
+-
+-		var listener net.Listener
+-		if tlsCfg == nil {
+-			listener, err = net.Listen(network, addr)
++			listener, err := net.Listen("tcp", addr)
++			if err != nil {
++				return nil, err
++			}
++			instance = newInstance(ctx, s, listener, s.tlsConfig, fmt.Sprintf("https://%s%s/", listener.Addr().String(), s.cfg.BaseURL))
+ 		} else {
+-			listener, err = tls.Listen(network, addr, tlsCfg)
+-		}
+-		if err != nil {
+-			return nil, err
+-		}
+-
+-		if network == "tcp" {
+-			var secure string
+-			if tlsCfg != nil {
+-				secure = "s"
++			// HTTP case
++			addr = strings.TrimPrefix(addr, "http://")
++			listener, err := net.Listen("tcp", addr)
++			if err != nil {
++				return nil, err
+ 			}
+-			url = fmt.Sprintf("http%s://%s%s/", secure, listener.Addr().String(), s.cfg.BaseURL)
+-		}
++			instance = newInstance(ctx, s, listener, nil, fmt.Sprintf("http://%s%s/", listener.Addr().String(), s.cfg.BaseURL))
+
+-		ii := instance{
+-			url:      url,
+-			listener: listener,
+-			httpServer: &http.Server{
+-				Handler:           s.mux,
+-				ReadTimeout:       s.cfg.ServerReadTimeout,
+-				WriteTimeout:      s.cfg.ServerWriteTimeout,
+-				MaxHeaderBytes:    s.cfg.MaxHeaderBytes,
+-				ReadHeaderTimeout: 10 * time.Second, // time to send the headers
+-				IdleTimeout:       60 * time.Second, // time to keep idle connections open
+-				TLSConfig:         tlsCfg,
+-				BaseContext:       NewBaseContext(ctx, url),
+-			},
+ 		}
+
+-		s.instances = append(s.instances, ii)
++		s.instances = append(s.instances, *instance)
+ 	}
+
+ 	return s, nil
+-- 
+2.44.0
+
diff --git a/pkgs/rclone/webdav-introduce-unix_socket_path.patch b/pkgs/rclone/webdav-introduce-unix_socket_path.patch
@@ -0,0 +1,139 @@
+From 5e34ec7912a629e1d6561a6cc60c1663be4f4423 Mon Sep 17 00:00:00 2001
+From: Florian Klink <[email protected]>
+Date: Fri, 24 May 2024 11:45:10 +0200
+Subject: [PATCH] webdav: introduce unix_socket_path
+
+This adds a new optional parameter to the backend, allowing to specify a
+path to a unix domain socket to connect to, instead the specified URL.
+
+If the parameter is set, we use `fshttp.NewClientCustom` to modify the
+HTTP transport, to use a dialer connecting to the unix domain socket
+path specified for that backend.
+
+The URL itself is still used for the rest of the HTTP client, allowing
+host and subpath to stay intact.
+
+This allows using rclone with the webdav backend to connect to a WebDAV
+server provided at a Unix Domain socket:
+
+```
+RCLONE_WEBDAV_UNIX_SOCKET_PATH=/path/to/my.sock \
+RCLONE_WEBDAV_URL=http://localhost \
+rclone sync mydir :webdav:/somewhere
+```
+---
+ backend/webdav/webdav.go | 19 ++++++++++++++++++-
+ docs/content/webdav.md   | 11 +++++++++++
+ fs/config.go             |  1 +
+ fs/fshttp/http.go        |  8 +++++++-
+ 4 files changed, 37 insertions(+), 2 deletions(-)
+
+diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go
+index f1c16f35a..c72875db8 100644
+--- a/backend/webdav/webdav.go
++++ b/backend/webdav/webdav.go
+@@ -15,6 +15,7 @@ import (
+ 	"errors"
+ 	"fmt"
+ 	"io"
++	"net"
+ 	"net/http"
+ 	"net/url"
+ 	"os/exec"
+@@ -154,6 +155,11 @@ Set to 0 to disable chunked uploading.
+ 			Help:     "Exclude ownCloud shares",
+ 			Advanced: true,
+ 			Default:  false,
++		}, {
++			Name:     "unix_socket_path",
++			Help:     "Path to a unix domain socket to dial to, instead of opening a TCP connection directly",
++			Advanced: true,
++			Default:  "",
+ 		}},
+ 	})
+ }
+@@ -171,6 +177,7 @@ type Options struct {
+ 	PacerMinSleep      fs.Duration          `config:"pacer_min_sleep"`
+ 	ChunkSize          fs.SizeSuffix        `config:"nextcloud_chunk_size"`
+ 	ExcludeShares      bool                 `config:"owncloud_exclude_shares"`
++	UnixSocketPath     string               `config:"unix_socket_path"`
+ }
+
+ // Fs represents a remote webdav
+@@ -452,7 +459,17 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
+ 		precision:   fs.ModTimeNotSupported,
+ 	}
+
+-	client := fshttp.NewClient(ctx)
++	var client *http.Client
++	if opt.UnixSocketPath == "" {
++		client = fshttp.NewClient(ctx)
++	} else {
++		fs.Debugf(f, "custom unix_socket_path configured (%v), updating dialer…", opt.UnixSocketPath)
++		client = fshttp.NewClientCustom(ctx, func(t *http.Transport) {
++			t.DialContext = func(reqCtx context.Context, network, addr string) (net.Conn, error) {
++				return fshttp.NewDialer(ctx).DialContext(reqCtx, "unix", opt.UnixSocketPath)
++			}
++		})
++	}
+ 	if opt.Vendor == "sharepoint-ntlm" {
+ 		// Disable transparent HTTP/2 support as per https://golang.org/pkg/net/http/ ,
+ 		// otherwise any connection to IIS 10.0 fails with 'stream error: stream ID 39; HTTP_1_1_REQUIRED'
+diff --git a/docs/content/webdav.md b/docs/content/webdav.md
+index 7da008014..855b0e2a0 100644
+--- a/docs/content/webdav.md
++++ b/docs/content/webdav.md
+@@ -283,6 +283,17 @@ Properties:
+ - Type:        bool
+ - Default:     false
+
++#### --webdav-unix-socket-path
++
++Path to a unix domain socket to dial to, instead of opening a TCP connection directly
++
++Properties:
++
++- Config:      unix_socket_path
++- Env Var:     RCLONE_WEBDAV_UNIX_SOCKET_PATH
++- Type:        string
++- Required:    false
++
+ #### --webdav-description
+
+ Description of the remote
+diff --git a/fs/config.go b/fs/config.go
+index 435fbf8e7..5122aae18 100644
+--- a/fs/config.go
++++ b/fs/config.go
+@@ -154,6 +154,7 @@ type ConfigInfo struct {
+ 	Inplace                    bool // Download directly to destination file instead of atomic download to temp/rename
+ 	PartialSuffix              string
+ 	MetadataMapper             SpaceSepList
++	UnixSocketPath             string // Optional unix socket to connect to
+ }
+
+ // NewConfig creates a new config with everything set to the default
+diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
+index 498714dfd..ff756d511 100644
+--- a/fs/fshttp/http.go
++++ b/fs/fshttp/http.go
+@@ -127,9 +127,15 @@ func NewTransport(ctx context.Context) http.RoundTripper {
+
+ // NewClient returns an http.Client with the correct timeouts
+ func NewClient(ctx context.Context) *http.Client {
++	return NewClientCustom(ctx, nil)
++}
++
++// NewClientCustom returns an http.Client with the correct timeouts.
++// It allows customizing the transport, using NewTransportCustom.
++func NewClientCustom(ctx context.Context, customize func(*http.Transport)) *http.Client {
+ 	ci := fs.GetConfig(ctx)
+ 	client := &http.Client{
+-		Transport: NewTransport(ctx),
++		Transport: NewTransportCustom(ctx, customize),
+ 	}
+ 	if ci.Cookie {
+ 		client.Jar = cookieJar
+-- 
+2.44.0
+