browser assets to be served via the cdn (#3683)

more stuff offloaded from byond's slowness the better

thank you mso tgstation/tgstation#52681

🆑
server: server's can now support using a cdn for web assets
/🆑

code/__HELPERS/files.dm

@@ -12,11 +12,6 @@
 
 	return text
 
-//Sends resource files to client cache
-/client/proc/getFiles()
-	for(var/file in args)
-		src << browse_rsc(file)
-
 /client/proc/browse_files(root="data/logs/", max_iterations=10, list/valid_extensions=list(".txt",".log",".htm"))
 	var/path = root
 

code/controllers/subsystem/assets.dm

@@ -7,6 +7,20 @@ SUBSYSTEM_DEF(assets)
 	var/list/preload = list()
 	var/datum/asset_transport/transport = new()
 
+/datum/controller/subsystem/assets/OnConfigLoad()
+	var/newtransporttype = /datum/asset_transport
+	switch (CONFIG_GET(string/asset_transport))
+		if ("webroot")
+			newtransporttype = /datum/asset_transport/webroot
+
+	if (newtransporttype == transport.type)
+		return
+
+	var/datum/asset_transport/newtransport = new newtransporttype ()
+	if (newtransport.validate_config())
+		transport = newtransport
+	transport.Load()
+
 /datum/controller/subsystem/assets/Initialize()
 	for(var/type in typesof(/datum/asset))
 		var/datum/asset/A = type

code/modules/admin/admin_verbs.dm

@@ -176,6 +176,7 @@ var/list/admin_verbs_server = list(
 	/client/proc/cmd_admin_delete, /*delete an instance/object/mob/etc*/
 	/client/proc/cmd_debug_del_all,
 	/datum/admins/proc/togglejoin,
+	/client/proc/toggle_cdn,
 )
 
 var/list/admin_verbs_debug = list(

code/modules/admin/tabs/round_tab.dm

@@ -181,3 +181,34 @@
 	else
 		to_chat(usr, "<font color='red'>Error: Start Now: Game has already started.</font>")
 		return FALSE
+
+/client/proc/toggle_cdn()
+	set name = "Toggle CDN"
+	set category = "Server"
+	var/static/admin_disabled_cdn_transport = null
+	if(alert(usr, "Are you sure you want to toggle CDN asset transport?", "Confirm", "Yes", "No") != "Yes")
+		return
+
+	var/current_transport = CONFIG_GET(string/asset_transport)
+	if(!current_transport || current_transport == "simple")
+		if(admin_disabled_cdn_transport)
+			CONFIG_SET(string/asset_transport, admin_disabled_cdn_transport)
+			admin_disabled_cdn_transport = null
+			SSassets.OnConfigLoad()
+			message_admins("[key_name_admin(usr)] re-enabled the CDN asset transport")
+			log_admin("[key_name(usr)] re-enabled the CDN asset transport")
+			return
+
+		to_chat(usr, SPAN_ADMINNOTICE("The CDN is not enabled!"))
+		if(alert(usr, "CDN asset transport is not enabled! If you're having issues with assets, you can also try disabling filename mutations.", "CDN asset transport is not enabled!", "Try disabling filename mutations", "Nevermind") == "Try disabling filename mutations")
+			SSassets.transport.dont_mutate_filenames = !SSassets.transport.dont_mutate_filenames
+			message_admins("[key_name_admin(usr)] [(SSassets.transport.dont_mutate_filenames ? "disabled" : "re-enabled")] asset filename transforms.")
+			log_admin("[key_name(usr)] [(SSassets.transport.dont_mutate_filenames ? "disabled" : "re-enabled")] asset filename transforms.")
+		return
+
+	admin_disabled_cdn_transport = current_transport
+	CONFIG_SET(string/asset_transport, "simple")
+	SSassets.OnConfigLoad()
+	SSassets.transport.dont_mutate_filenames = TRUE
+	message_admins("[key_name_admin(usr)] disabled CDN asset transport")
+	log_admin("[key_name(usr)] disabled CDN asset transport")

code/modules/asset_cache/asset_cache_item.dm

@@ -33,3 +33,9 @@
 	if (extstart)
 		ext = ".[copytext(name, extstart+1)]"
 	resource = file
+
+/datum/asset_cache_item/vv_edit_var(var_name, var_value)
+	return FALSE
+
+/datum/asset_cache_item/CanProcCall(procname)
+	return FALSE

code/modules/asset_cache/transports/webroot_transport.dm

@@ -0,0 +1,87 @@
+/// CDN Webroot asset transport.
+/datum/asset_transport/webroot
+	name = "CDN Webroot asset transport"
+
+/datum/asset_transport/webroot/Load()
+	if (validate_config(log = FALSE))
+		load_existing_assets()
+
+/// Processes thru any assets that were registered before we were loaded as a transport.
+/datum/asset_transport/webroot/proc/load_existing_assets()
+	for (var/asset_name in SSassets.cache)
+		var/datum/asset_cache_item/ACI = SSassets.cache[asset_name]
+		save_asset_to_webroot(ACI)
+
+/// Register a browser asset with the asset cache system
+/// We also save it to the CDN webroot at this step instead of waiting for send_assets()
+/// asset_name - the identifier of the asset
+/// asset - the actual asset file or an asset_cache_item datum.
+/datum/asset_transport/webroot/register_asset(asset_name, asset)
+	. = ..()
+	var/datum/asset_cache_item/ACI = .
+
+	if (istype(ACI) && ACI.hash)
+		save_asset_to_webroot(ACI)
+
+/// Saves the asset to the webroot taking into account namespaces and hashes.
+/datum/asset_transport/webroot/proc/save_asset_to_webroot(datum/asset_cache_item/ACI)
+	var/webroot = CONFIG_GET(string/asset_cdn_webroot)
+	var/newpath = "[webroot][get_asset_suffex(ACI)]"
+	if (fexists(newpath))
+		return
+	if (fexists("[newpath].gz")) //its a common pattern in webhosting to save gzip'ed versions of text files and let the webserver serve them up as gzip compressed normal files, sometimes without keeping the original version.
+		return
+	return fcopy(ACI.resource, newpath)
+
+/// Returns a url for a given asset.
+/// asset_name - Name of the asset.
+/// asset_cache_item - asset cache item datum for the asset, optional, overrides asset_name
+/datum/asset_transport/webroot/get_asset_url(asset_name, datum/asset_cache_item/asset_cache_item)
+	if (!istype(asset_cache_item))
+		asset_cache_item = SSassets.cache[asset_name]
+	var/url = CONFIG_GET(string/asset_cdn_url) //config loading will handle making sure this ends in a /
+	return "[url][get_asset_suffex(asset_cache_item)]"
+
+/datum/asset_transport/webroot/proc/get_asset_suffex(datum/asset_cache_item/asset_cache_item)
+	var/base = "[copytext(asset_cache_item.hash, 1, 3)]/"
+	var/filename = "asset.[asset_cache_item.hash][asset_cache_item.ext]"
+	if (length(asset_cache_item.namespace))
+		base = "namespaces/[copytext(asset_cache_item.namespace, 1, 3)]/[asset_cache_item.namespace]/"
+		if (!asset_cache_item.namespace_parent)
+			filename = "[asset_cache_item.name]"
+	return base + filename
+
+
+/// webroot asset sending - does nothing unless passed legacy assets
+/datum/asset_transport/webroot/send_assets(client/client, list/asset_list)
+	. = FALSE
+	var/list/legacy_assets = list()
+	if (!islist(asset_list))
+		asset_list = list(asset_list)
+	for (var/asset_name in asset_list)
+		var/datum/asset_cache_item/ACI = asset_list[asset_name]
+		if (!istype(ACI))
+			ACI = SSassets.cache[asset_name]
+		if (!ACI)
+			legacy_assets += asset_name //pass it on to base send_assets so it can output an error
+			continue
+		if (ACI.legacy)
+			legacy_assets[asset_name] = ACI
+	if (length(legacy_assets))
+		. = ..(client, legacy_assets)
+
+
+/// webroot slow asset sending - does nothing.
+/datum/asset_transport/webroot/send_assets_slow(client/client, list/files, filerate)
+	return FALSE
+
+/datum/asset_transport/webroot/validate_config(log = TRUE)
+	if (!CONFIG_GET(string/asset_cdn_url))
+		if (log)
+			log_asset("ERROR: [type]: Invalid Config: ASSET_CDN_URL")
+		return FALSE
+	if (!CONFIG_GET(string/asset_cdn_webroot))
+		if (log)
+			log_asset("ERROR: [type]: Invalid Config: ASSET_CDN_WEBROOT")
+		return FALSE
+	return TRUE

colonialmarines.dme

@@ -1382,6 +1382,7 @@
 #include "code\modules\asset_cache\assets\tgui.dm"
 #include "code\modules\asset_cache\assets\vending.dm"
 #include "code\modules\asset_cache\transports\asset_transport.dm"
+#include "code\modules\asset_cache\transports\webroot_transport.dm"
 #include "code\modules\buildmode\bm-mode.dm"
 #include "code\modules\buildmode\buildmode.dm"
 #include "code\modules\buildmode\buttons.dm"

config/example/resources.txt

@@ -0,0 +1,40 @@
+# External resources
+# Set this to the location of a .zip with the server's .rsc inside of it.
+# If you set this mutiple times, the server will rotate between the links.
+# To use this, the compile option PRELOAD_RSC must be set to 0 to keep byond from preloading resources
+# Resource urls can not be encrypted (https://), as they are downloaded by byond, not IE, and byond can't into encryption
+
+EXTERNAL_RSC_URLS http://rsc.cm-ss13.com/
+
+
+########################
+# Browser Asset Config #
+########################
+# Browser assets are any file included in interfaces. css, images, javascript, etc.
+# This handles configuring how we get these to the player so interfaces can access them.
+
+# Asset Transport
+# The normal way of getting assets to clients is to use the internal byond system. This can be slow and delay the opening of interface windows. It also doesn't allow the internal IE windows byond uses to cache anything.
+# You can instead have the server save them to a website via a folder within the game server that the web server can read. This could be a simple webserver or something backed by a CDN.
+# Valid values: simple, webroot. Simple is the default.
+#ASSET_TRANSPORT webroot
+
+
+# Simple asset transport configurable values.
+
+# Uncomment this to have the server passively send all browser assets to each client in the background. (instead of waiting for them to be needed)
+# This should be uncommented in production and commented in development
+#ASSET_SIMPLE_PRELOAD
+
+
+# Webroot asset transport configurable values.
+
+# Local folder to save assets to.
+# Assets will be saved in the format of asset.MD5HASH.EXT or in namespaces/hash/ as ASSET_FILE_NAME or asset.MD5HASH.EXT
+#ASSET_CDN_WEBROOT data/asset-store/
+
+# URL the folder from above can be accessed from.
+# for best results the webserver powering this should return a long cache validity time, as all assets sent via this transport use hash based urls
+# Encryption (https) is supported here, but linux clients will have issues if you require higher then tls 1.0. Windows clients down to windows 7 can handle tls 1.2 no issue.
+# if you want to test this locally, you simpily run the `localhost-asset-webroot-server.py` python3 script to host assets stored in `data/asset-store/` via http://localhost:58715/
+#ASSET_CDN_URL http://localhost:58715/

tools/localhost-asset-webroot-server.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+import os
+
+class CORSRequestHandler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_header('Access-Control-Allow-Origin', '*')
+        self.send_header('Access-Control-Allow-Methods', 'GET')
+        self.send_header('Cache-Control', 'no-store, no-cache, must-revalidate')
+        return super(CORSRequestHandler, self).end_headers()
+
+os.makedirs('../data/asset-store/', exist_ok=True)
+os.chdir('../data/asset-store/')
+httpd = HTTPServer(('localhost', 58715), CORSRequestHandler)
+httpd.serve_forever()