Re-enable case for /guilds/!/channels, add BIND_IP and OUTBOUND_IP en…

…v vars, organize documentation

CONFIG.md

@@ -0,0 +1,29 @@
+# Config
+##### LOG_LEVEL
+Logrus log level. Passed directly to [ParseLevel](https://github.com/sirupsen/logrus/blob/master/logrus.go#L25-L45)
+
+##### PORT
+The port to listen for requests on
+
+##### METRICS_PORT 
+The port for to listen on for metrics
+
+##### ENABLE_METRICS
+Wether to enable and register metrics. Disabling may improve resource usage
+
+##### ENABLE_PPROF
+Enables the performance profiling handler. Read more [here](https://github.com/google/pprof/blob/master/doc/README.md)
+
+##### BUFFER_SIZE
+Size for the internal proxy go channels. Channels are used to synchronize and order requests. As each request comes in, it gets pushed to a channel. In go, channels can be buffered, this var defines the size of this buffer.
+Decreasing this will improve memory usage, but beware that once a channel buffer is full, requests will fight to be added to the channel on the next free spot. This means that during high usage periods, a part of the requests will be unordered if this value is set too low.
+
+##### OUTBOUND_IP
+The local address to use when firing requests to discord.
+
+Example: `"120.121.122.123"`
+
+##### BIND_IP
+The IP to bind the HTTP server on (both for requests and metrics). 127.0.0.1 will only allow requests coming from the loopback interface. Useful for preventing the proxy from being accessed from outside of LAN, for example.
+
+Example: `"10.0.0.42"` - Would only listen on LAN

README.md

@@ -22,9 +22,13 @@ Configuration options are
 | LOG_LEVEL   | panic, fatal, error, warn, info, debug, trace | info |
 | PORT        | number | 8080    |
 | METRICS_PORT| number | 9000    |
-| ENABLE_METRICS| boolean| true   |
+| ENABLE_METRICS| boolean| true  |
 | ENABLE_PPROF| boolean| false   |
-| BUFFER_SIZE [(?)](https://github.com/germanoeich/nirn-proxy/blob/main/lib/queue.go#L37-L43) | number | 50      |
+| BUFFER_SIZE | number | 50      |
+| OUTBOUND_IP | string | ""      |
+| BIND_IP     | string | 0.0.0.0 |
+
+Information on each config var can be found [here](https://github.com/germanoeich/nirn-proxy/blob/main/CONFIG.md)
 
 .env files are loaded if present
 
@@ -34,7 +38,7 @@ The proxy listens on all routes and relays them to Discord, while keeping track
 
 When using the proxy, it is safe to remove the ratelimiting logic from clients and fire requests instantly, however, the proxy does not handle retries. If for some reason (i.e shared ratelimits, internal discord ratelimits, etc) the proxy encounters a 429, it will return that to the client. It is safe to immediately retry requests that return 429 or even setup retry logic elsewhere (like in a load balancer or service mesh).
 
-The proxy also guards against known scenarios that might cause a cloudflare ban, like too webhook 404s or too many 401s.
+The proxy also guards against known scenarios that might cause a cloudflare ban, like too many webhook 404s or too many 401s.
 
 ### Limitations
 

lib/bucketpath.go

@@ -89,11 +89,10 @@ func GetOptimisticBucketPath(url string, method string) string {
 		bucket += MajorInvites + "/!"
 		currMajor = MajorInvites
 	case MajorGuilds:
-		/* TODO: Figure out why this makes the bot unresponsive
+		// guilds/:guildId/channels share the same bucket for all guilds
 		if numParts == 3 && parts[2] == "channels" {
-			return "/guilds/!/channels"
+			return "/" + MajorGuilds + "/!/channels"
 		}
-		*/
 		fallthrough
 	case MajorWebhooks:
 		fallthrough

lib/discord.go

@@ -1,12 +1,14 @@
 package lib
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"github.com/sirupsen/logrus"
 	"io"
 	"io/ioutil"
 	"math"
+	"net"
 	"net/http"
 	"strings"
 	"time"
@@ -18,6 +20,26 @@ type BotGatewayResponse struct {
 	SessionStartLimit map[string]int `json:"session_start_limit"`
 }
 
+func ConfigureDiscordHTTPClient(ip string) {
+	addr, err := net.ResolveTCPAddr("tcp", ip + ":0")
+
+	if err != nil {
+		panic(err)
+	}
+
+	dialer := &net.Dialer{LocalAddr: addr}
+
+	dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := dialer.Dial(network, addr)
+		return conn, err
+	}
+
+	transport := &http.Transport{DialContext: dialContext}
+	client = &http.Client{
+		Transport: transport,
+	}
+}
+
 func GetBotGlobalLimit(token string) (uint, error) {
 	if token == "" {
 		return math.MaxUint32, nil

lib/metrics.go

@@ -20,11 +20,11 @@ var (
 	}, []string{"method", "status", "route", "clientId"})
 )
 
-func StartMetrics(port string) {
+func StartMetrics(addr string) {
 	prometheus.MustRegister(RequestSummary)
 	http.Handle("/metrics", promhttp.Handler())
-	logger.Info("Starting metrics server on :" + port)
-	err := http.ListenAndServe(":" + port, nil)
+	logger.Info("Starting metrics server on " + addr)
+	err := http.ListenAndServe(addr, nil)
 	if err != nil {
 		logger.Error(err)
 		return

lib/queue.go

@@ -35,12 +35,6 @@ type RequestQueue struct {
 	processor func(item *QueueItem) *http.Response
 	globalBucket leakybucket.Bucket
 	// bufferSize Defines the size of the request channel buffer for each bucket
-	// Realistically, this should be as high as possible to prevent blocking sends
-	// While blocking sends aren't a problem in itself, they are unordered, meaning
-	// in a high load situation, if this number is too low, it would cause requests to
-	// fight to send, which messes up the ordering of requests. This variable can be tweaked
-	// using ENV vars, lower will improve memory usage, higher will provide higher ordering guarantees
-	// Defaults to 50
 	bufferSize int64
 }
 

main.go

@@ -70,6 +70,11 @@ func (_ *GenericHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
 }
 
 func main()  {
+	outboundIp := os.Getenv("OUTBOUND_IP")
+	if outboundIp != "" {
+		lib.ConfigureDiscordHTTPClient(outboundIp)
+	}
+
 	logLevel := os.Getenv("LOG_LEVEL")
 	if logLevel == "" {
 		logLevel = "info"
@@ -85,11 +90,16 @@ func main()  {
 		port = "8080"
 	}
 
+	bindIp := os.Getenv("BIND_IP")
+	if bindIp == "" {
+		bindIp = "0.0.0.0"
+	}
+
 	logger.SetLevel(lvl)
-	logger.Info("Starting proxy on :" + port)
+	logger.Info("Starting proxy on " + bindIp + ":" + port)
 	lib.SetLogger(logger)
 	s := &http.Server{
-		Addr:           ":" + port,
+		Addr:           bindIp + ":" + port,
 		Handler:        &GenericHandler{},
 		ReadTimeout:    10 * time.Second,
 		WriteTimeout:   1 * time.Hour,
@@ -105,7 +115,7 @@ func main()  {
 		if port == "" {
 			port = "9000"
 		}
-		go lib.StartMetrics(port)
+		go lib.StartMetrics(bindIp + ":" + port)
 	}
 
 	bufferEnv := os.Getenv("BUFFER_SIZE")

tests/Bucketpath_test.go

@@ -29,6 +29,8 @@ func TestPaths(t *testing.T) {
 		{"/api/v9/invalid/203039963636301824/route/203039963636301824", "GET", "/invalid/203039963636301824/route/!"},
 		//Special case for /guilds/:id/channels
 		{"/api/v9/guilds/203039963636301824/channels", "GET", "/guilds/!/channels"},
+		// Wierd routes
+		{"/api/v9/guilds/templates/203039963636301824", "GET", "/guilds/templates/!"},
 	}
 	for _, tt := range tests {
 		testname := fmt.Sprintf("%s-%s", tt.method, tt.path)