From 312760bb011412cd71f69a82879a9cf995192077 Mon Sep 17 00:00:00 2001 From: harryob Date: Thu, 22 Jun 2023 07:33:32 +0100 Subject: [PATCH] browser assets to be served via the cdn (#3683) more stuff offloaded from byond's slowness the better thank you mso https://github.com/tgstation/tgstation/pull/52681 :cl: server: server's can now support using a cdn for web assets /:cl: --- code/__HELPERS/files.dm | 5 -- code/controllers/subsystem/assets.dm | 14 +++ code/modules/admin/admin_verbs.dm | 1 + code/modules/admin/tabs/round_tab.dm | 31 +++++++ code/modules/asset_cache/asset_cache_item.dm | 6 ++ .../transports/webroot_transport.dm | 87 +++++++++++++++++++ colonialmarines.dme | 1 + config/example/resources.txt | 40 +++++++++ tools/localhost-asset-webroot-server.py | 15 ++++ 9 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 code/modules/asset_cache/transports/webroot_transport.dm create mode 100644 config/example/resources.txt create mode 100644 tools/localhost-asset-webroot-server.py diff --git a/code/__HELPERS/files.dm b/code/__HELPERS/files.dm index 717a2ed5347b..f88fe7168f8d 100644 --- a/code/__HELPERS/files.dm +++ b/code/__HELPERS/files.dm @@ -12,11 +12,6 @@ return text -//Sends resource files to client cache -/client/proc/getFiles() - for(var/file in args) - src << browse_rsc(file) - /client/proc/browse_files(root="data/logs/", max_iterations=10, list/valid_extensions=list(".txt",".log",".htm")) var/path = root diff --git a/code/controllers/subsystem/assets.dm b/code/controllers/subsystem/assets.dm index 283fe79cfead..38e57df93cef 100644 --- a/code/controllers/subsystem/assets.dm +++ b/code/controllers/subsystem/assets.dm @@ -7,6 +7,20 @@ SUBSYSTEM_DEF(assets) var/list/preload = list() var/datum/asset_transport/transport = new() +/datum/controller/subsystem/assets/OnConfigLoad() + var/newtransporttype = /datum/asset_transport + switch (CONFIG_GET(string/asset_transport)) + if ("webroot") + newtransporttype = /datum/asset_transport/webroot + + if (newtransporttype == transport.type) + return + + var/datum/asset_transport/newtransport = new newtransporttype () + if (newtransport.validate_config()) + transport = newtransport + transport.Load() + /datum/controller/subsystem/assets/Initialize() for(var/type in typesof(/datum/asset)) var/datum/asset/A = type diff --git a/code/modules/admin/admin_verbs.dm b/code/modules/admin/admin_verbs.dm index d21188615785..debffa9b25b0 100644 --- a/code/modules/admin/admin_verbs.dm +++ b/code/modules/admin/admin_verbs.dm @@ -176,6 +176,7 @@ var/list/admin_verbs_server = list( /client/proc/cmd_admin_delete, /*delete an instance/object/mob/etc*/ /client/proc/cmd_debug_del_all, /datum/admins/proc/togglejoin, + /client/proc/toggle_cdn, ) var/list/admin_verbs_debug = list( diff --git a/code/modules/admin/tabs/round_tab.dm b/code/modules/admin/tabs/round_tab.dm index 980de9eed855..deb27e73f7dc 100644 --- a/code/modules/admin/tabs/round_tab.dm +++ b/code/modules/admin/tabs/round_tab.dm @@ -181,3 +181,34 @@ else to_chat(usr, "Error: Start Now: Game has already started.") return FALSE + +/client/proc/toggle_cdn() + set name = "Toggle CDN" + set category = "Server" + var/static/admin_disabled_cdn_transport = null + if(alert(usr, "Are you sure you want to toggle CDN asset transport?", "Confirm", "Yes", "No") != "Yes") + return + + var/current_transport = CONFIG_GET(string/asset_transport) + if(!current_transport || current_transport == "simple") + if(admin_disabled_cdn_transport) + CONFIG_SET(string/asset_transport, admin_disabled_cdn_transport) + admin_disabled_cdn_transport = null + SSassets.OnConfigLoad() + message_admins("[key_name_admin(usr)] re-enabled the CDN asset transport") + log_admin("[key_name(usr)] re-enabled the CDN asset transport") + return + + to_chat(usr, SPAN_ADMINNOTICE("The CDN is not enabled!")) + if(alert(usr, "CDN asset transport is not enabled! If you're having issues with assets, you can also try disabling filename mutations.", "CDN asset transport is not enabled!", "Try disabling filename mutations", "Nevermind") == "Try disabling filename mutations") + SSassets.transport.dont_mutate_filenames = !SSassets.transport.dont_mutate_filenames + message_admins("[key_name_admin(usr)] [(SSassets.transport.dont_mutate_filenames ? "disabled" : "re-enabled")] asset filename transforms.") + log_admin("[key_name(usr)] [(SSassets.transport.dont_mutate_filenames ? "disabled" : "re-enabled")] asset filename transforms.") + return + + admin_disabled_cdn_transport = current_transport + CONFIG_SET(string/asset_transport, "simple") + SSassets.OnConfigLoad() + SSassets.transport.dont_mutate_filenames = TRUE + message_admins("[key_name_admin(usr)] disabled CDN asset transport") + log_admin("[key_name(usr)] disabled CDN asset transport") diff --git a/code/modules/asset_cache/asset_cache_item.dm b/code/modules/asset_cache/asset_cache_item.dm index 52ebc7190372..72d976bf11f1 100644 --- a/code/modules/asset_cache/asset_cache_item.dm +++ b/code/modules/asset_cache/asset_cache_item.dm @@ -33,3 +33,9 @@ if (extstart) ext = ".[copytext(name, extstart+1)]" resource = file + +/datum/asset_cache_item/vv_edit_var(var_name, var_value) + return FALSE + +/datum/asset_cache_item/CanProcCall(procname) + return FALSE diff --git a/code/modules/asset_cache/transports/webroot_transport.dm b/code/modules/asset_cache/transports/webroot_transport.dm new file mode 100644 index 000000000000..e3cb33b8fabf --- /dev/null +++ b/code/modules/asset_cache/transports/webroot_transport.dm @@ -0,0 +1,87 @@ +/// CDN Webroot asset transport. +/datum/asset_transport/webroot + name = "CDN Webroot asset transport" + +/datum/asset_transport/webroot/Load() + if (validate_config(log = FALSE)) + load_existing_assets() + +/// Processes thru any assets that were registered before we were loaded as a transport. +/datum/asset_transport/webroot/proc/load_existing_assets() + for (var/asset_name in SSassets.cache) + var/datum/asset_cache_item/ACI = SSassets.cache[asset_name] + save_asset_to_webroot(ACI) + +/// Register a browser asset with the asset cache system +/// We also save it to the CDN webroot at this step instead of waiting for send_assets() +/// asset_name - the identifier of the asset +/// asset - the actual asset file or an asset_cache_item datum. +/datum/asset_transport/webroot/register_asset(asset_name, asset) + . = ..() + var/datum/asset_cache_item/ACI = . + + if (istype(ACI) && ACI.hash) + save_asset_to_webroot(ACI) + +/// Saves the asset to the webroot taking into account namespaces and hashes. +/datum/asset_transport/webroot/proc/save_asset_to_webroot(datum/asset_cache_item/ACI) + var/webroot = CONFIG_GET(string/asset_cdn_webroot) + var/newpath = "[webroot][get_asset_suffex(ACI)]" + if (fexists(newpath)) + return + if (fexists("[newpath].gz")) //its a common pattern in webhosting to save gzip'ed versions of text files and let the webserver serve them up as gzip compressed normal files, sometimes without keeping the original version. + return + return fcopy(ACI.resource, newpath) + +/// Returns a url for a given asset. +/// asset_name - Name of the asset. +/// asset_cache_item - asset cache item datum for the asset, optional, overrides asset_name +/datum/asset_transport/webroot/get_asset_url(asset_name, datum/asset_cache_item/asset_cache_item) + if (!istype(asset_cache_item)) + asset_cache_item = SSassets.cache[asset_name] + var/url = CONFIG_GET(string/asset_cdn_url) //config loading will handle making sure this ends in a / + return "[url][get_asset_suffex(asset_cache_item)]" + +/datum/asset_transport/webroot/proc/get_asset_suffex(datum/asset_cache_item/asset_cache_item) + var/base = "[copytext(asset_cache_item.hash, 1, 3)]/" + var/filename = "asset.[asset_cache_item.hash][asset_cache_item.ext]" + if (length(asset_cache_item.namespace)) + base = "namespaces/[copytext(asset_cache_item.namespace, 1, 3)]/[asset_cache_item.namespace]/" + if (!asset_cache_item.namespace_parent) + filename = "[asset_cache_item.name]" + return base + filename + + +/// webroot asset sending - does nothing unless passed legacy assets +/datum/asset_transport/webroot/send_assets(client/client, list/asset_list) + . = FALSE + var/list/legacy_assets = list() + if (!islist(asset_list)) + asset_list = list(asset_list) + for (var/asset_name in asset_list) + var/datum/asset_cache_item/ACI = asset_list[asset_name] + if (!istype(ACI)) + ACI = SSassets.cache[asset_name] + if (!ACI) + legacy_assets += asset_name //pass it on to base send_assets so it can output an error + continue + if (ACI.legacy) + legacy_assets[asset_name] = ACI + if (length(legacy_assets)) + . = ..(client, legacy_assets) + + +/// webroot slow asset sending - does nothing. +/datum/asset_transport/webroot/send_assets_slow(client/client, list/files, filerate) + return FALSE + +/datum/asset_transport/webroot/validate_config(log = TRUE) + if (!CONFIG_GET(string/asset_cdn_url)) + if (log) + log_asset("ERROR: [type]: Invalid Config: ASSET_CDN_URL") + return FALSE + if (!CONFIG_GET(string/asset_cdn_webroot)) + if (log) + log_asset("ERROR: [type]: Invalid Config: ASSET_CDN_WEBROOT") + return FALSE + return TRUE diff --git a/colonialmarines.dme b/colonialmarines.dme index 6e5bda16e73d..991c23d53bbe 100644 --- a/colonialmarines.dme +++ b/colonialmarines.dme @@ -1382,6 +1382,7 @@ #include "code\modules\asset_cache\assets\tgui.dm" #include "code\modules\asset_cache\assets\vending.dm" #include "code\modules\asset_cache\transports\asset_transport.dm" +#include "code\modules\asset_cache\transports\webroot_transport.dm" #include "code\modules\buildmode\bm-mode.dm" #include "code\modules\buildmode\buildmode.dm" #include "code\modules\buildmode\buttons.dm" diff --git a/config/example/resources.txt b/config/example/resources.txt new file mode 100644 index 000000000000..0f9aaf021aa5 --- /dev/null +++ b/config/example/resources.txt @@ -0,0 +1,40 @@ +# External resources +# Set this to the location of a .zip with the server's .rsc inside of it. +# If you set this mutiple times, the server will rotate between the links. +# To use this, the compile option PRELOAD_RSC must be set to 0 to keep byond from preloading resources +# Resource urls can not be encrypted (https://), as they are downloaded by byond, not IE, and byond can't into encryption + +EXTERNAL_RSC_URLS http://rsc.cm-ss13.com/ + + +######################## +# Browser Asset Config # +######################## +# Browser assets are any file included in interfaces. css, images, javascript, etc. +# This handles configuring how we get these to the player so interfaces can access them. + +# Asset Transport +# The normal way of getting assets to clients is to use the internal byond system. This can be slow and delay the opening of interface windows. It also doesn't allow the internal IE windows byond uses to cache anything. +# You can instead have the server save them to a website via a folder within the game server that the web server can read. This could be a simple webserver or something backed by a CDN. +# Valid values: simple, webroot. Simple is the default. +#ASSET_TRANSPORT webroot + + +# Simple asset transport configurable values. + +# Uncomment this to have the server passively send all browser assets to each client in the background. (instead of waiting for them to be needed) +# This should be uncommented in production and commented in development +#ASSET_SIMPLE_PRELOAD + + +# Webroot asset transport configurable values. + +# Local folder to save assets to. +# Assets will be saved in the format of asset.MD5HASH.EXT or in namespaces/hash/ as ASSET_FILE_NAME or asset.MD5HASH.EXT +#ASSET_CDN_WEBROOT data/asset-store/ + +# URL the folder from above can be accessed from. +# for best results the webserver powering this should return a long cache validity time, as all assets sent via this transport use hash based urls +# Encryption (https) is supported here, but linux clients will have issues if you require higher then tls 1.0. Windows clients down to windows 7 can handle tls 1.2 no issue. +# if you want to test this locally, you simpily run the `localhost-asset-webroot-server.py` python3 script to host assets stored in `data/asset-store/` via http://localhost:58715/ +#ASSET_CDN_URL http://localhost:58715/ diff --git a/tools/localhost-asset-webroot-server.py b/tools/localhost-asset-webroot-server.py new file mode 100644 index 000000000000..31a85a4eb4b5 --- /dev/null +++ b/tools/localhost-asset-webroot-server.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +from http.server import HTTPServer, SimpleHTTPRequestHandler +import os + +class CORSRequestHandler(SimpleHTTPRequestHandler): + def end_headers(self): + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET') + self.send_header('Cache-Control', 'no-store, no-cache, must-revalidate') + return super(CORSRequestHandler, self).end_headers() + +os.makedirs('../data/asset-store/', exist_ok=True) +os.chdir('../data/asset-store/') +httpd = HTTPServer(('localhost', 58715), CORSRequestHandler) +httpd.serve_forever()