refactors asset subsystem (#6459)

refactors the asset subsystem to make it much more managed

adds support for 'deferred' assets for tgui, which will be later used
for things like tgui character setup and admin panels.
deferred means that it'll be 'sent in a timely manner after the ui
opens', as opposed to 'sent before the ui opens'

---------

Co-authored-by: silicons <[email protected]>

.tgs4.yml

@@ -2,6 +2,7 @@ static_files:
   - name: config
     populate: true
   - name: data
+  - name: tmp
 linux_scripts:
   PreCompile.sh: tools/tgs4_scripts/PreCompile.sh
 windows_scripts:

README.md

@@ -56,8 +56,12 @@ These are also the folders you are likely going to encounter while managing the
 - /config: server configuration
   - /legacy: legacy configuration data go in here
 - /data: server persistent data
+  - /asset-cache - default location for caching of generated assets
+  - /asset-root - default location for generated assets to be served in webroot mode
   - /logs: logs are dumped in here
   - /players: player data, like saves and characters get dumped in here
+- /tmp: server scratch space
+  - /assets - for asset generation
 
 You only need to make the top level folders (e.g. config, data) static folders in TGS4.
 

citadel.dme

@@ -146,6 +146,7 @@
 #include "code\__DEFINES\combat\shieldcall.dm"
 #include "code\__DEFINES\controllers\_repositories.dm"
 #include "code\__DEFINES\controllers\_subsystems.dm"
+#include "code\__DEFINES\controllers\assets.dm"
 #include "code\__DEFINES\controllers\dbcore.dm"
 #include "code\__DEFINES\controllers\persistence.dm"
 #include "code\__DEFINES\controllers\processing.dm"
@@ -505,7 +506,6 @@
 #include "code\controllers\subsystem\chat.dm"
 #include "code\controllers\subsystem\dcs.dm"
 #include "code\controllers\subsystem\DPC.dm"
-#include "code\controllers\subsystem\early_assets.dm"
 #include "code\controllers\subsystem\early_init.dm"
 #include "code\controllers\subsystem\emergency_shuttle.dm"
 #include "code\controllers\subsystem\events.dm"
@@ -2221,22 +2221,22 @@
 #include "code\modules\assembly\signaler.dm"
 #include "code\modules\assembly\timer.dm"
 #include "code\modules\assembly\voice.dm"
-#include "code\modules\asset_cache\asset_cache_client.dm"
-#include "code\modules\asset_cache\asset_cache_item.dm"
-#include "code\modules\asset_cache\asset_list.dm"
+#include "code\modules\asset_cache\asset_item.dm"
+#include "code\modules\asset_cache\asset_pack.dm"
+#include "code\modules\asset_cache\asset_transport.dm"
+#include "code\modules\asset_cache\client.dm"
 #include "code\modules\asset_cache\assets\arcade.dm"
 #include "code\modules\asset_cache\assets\chat.dm"
 #include "code\modules\asset_cache\assets\common.dm"
 #include "code\modules\asset_cache\assets\condiments.dm"
-#include "code\modules\asset_cache\assets\fish.dm"
+#include "code\modules\asset_cache\assets\fishing.dm"
 #include "code\modules\asset_cache\assets\fontawesome.dm"
 #include "code\modules\asset_cache\assets\genetics.dm"
 #include "code\modules\asset_cache\assets\headers.dm"
 #include "code\modules\asset_cache\assets\inventory.dm"
 #include "code\modules\asset_cache\assets\jquery.dm"
 #include "code\modules\asset_cache\assets\legacy_nanomaps.dm"
 #include "code\modules\asset_cache\assets\legacy_nanoui.dm"
-#include "code\modules\asset_cache\assets\legacy_vstation.dm"
 #include "code\modules\asset_cache\assets\loadout.dm"
 #include "code\modules\asset_cache\assets\materials.dm"
 #include "code\modules\asset_cache\assets\moods.dm"
@@ -2256,8 +2256,13 @@
 #include "code\modules\asset_cache\assets\chemistry\patches.dm"
 #include "code\modules\asset_cache\assets\chemistry\pills.dm"
 #include "code\modules\asset_cache\assets\debug\fucky_wucky.dm"
-#include "code\modules\asset_cache\transports\asset_transport.dm"
-#include "code\modules\asset_cache\transports\webroot_transport.dm"
+#include "code\modules\asset_cache\packs\changelog_item.dm"
+#include "code\modules\asset_cache\packs\json.dm"
+#include "code\modules\asset_cache\packs\simple.dm"
+#include "code\modules\asset_cache\packs\spritesheet.dm"
+#include "code\modules\asset_cache\packs\spritesheet\simple.dm"
+#include "code\modules\asset_cache\transports\browse_rsc.dm"
+#include "code\modules\asset_cache\transports\webroot.dm"
 #include "code\modules\atmospherics\atmosphere\atmosphere.dm"
 #include "code\modules\atmospherics\atmosphere\planet.dm"
 #include "code\modules\atmospherics\environmental\atom.dm"

code/__DEFINES/_protect.dm

@@ -1,8 +1,11 @@
-#define GENERAL_PROTECT_DATUM(Path)\
+/**
+ * Completely occludes a path from view variable interactions.
+ */
+#define VV_PROTECT(Path)\
 ##Path/can_vv_get(var_name){\
     return FALSE;\
 }\
-##Path/vv_edit_var(var_name, var_value){\
+##Path/vv_edit_var(var_name, var_value, mass_edit, raw_edit){\
     return FALSE;\
 }\
 ##Path/CanProcCall(procname){\
@@ -11,3 +14,16 @@
 ##Path/can_vv_mark(){\
 	return FALSE;\
 }
+
+/**
+ * Makes a path read-only to view variables.
+ *
+ * * Does not prevent the path from being marked!
+ */
+#define VV_PROTECT_READONLY(Path)\
+##Path/vv_edit_var(var_name, var_value, mass_edit, raw_edit){\
+    return FALSE;\
+}\
+##Path/CanProcCall(procname){\
+    return FALSE;\
+}

code/__DEFINES/controllers/_subsystems.dm

@@ -77,12 +77,28 @@ DEFINE_BITFIELD(runlevels, list(
 
 // todo: tg init brackets
 
+// core security system, used by client/New()
 #define INIT_ORDER_FAIL2TOPIC      200
+// core security system, used by client/New()
 #define INIT_ORDER_IPINTEL         197
+
+// core timing system, used by almost everything
 #define INIT_ORDER_TIMER           195
+// just about every feature on the server requires the database backend
+// for storage and durability of permeance.
 #define INIT_ORDER_DBCORE          190
+// repository is just struct storage. its things depend on database,
+// but should depend on nothing else.
+//
+// for the rare occasion when a prototype requires asset registration,
+// it should be able to recognize if SSassets is ready,
+// and only queue an udpate if its asset is already loaded.
+#define INIT_ORDER_REPOSITORY      187
+// early init initializes what is basically expensive global variables. it needs to go before assets.
 #define INIT_ORDER_EARLY_INIT      185
-#define INIT_ORDER_REPOSITORY      180
+// assets is loaded early because things hook into this to register *their* assets
+#define INIT_ORDER_ASSETS          180
+
 #define INIT_ORDER_STATPANELS      170
 #define INIT_ORDER_PREFERENCES     165
 #define INIT_ORDER_INPUT           160
@@ -93,7 +109,6 @@ DEFINE_BITFIELD(runlevels, list(
 #define INIT_ORDER_VIS             80
 #define INIT_ORDER_SERVER_MAINT    65
 #define INIT_ORDER_INSTRUMENTS     50
-#define INIT_ORDER_EARLY_ASSETS    48
 #define INIT_ORDER_MEDIA_TRACKS    38
 #define INIT_ORDER_CHEMISTRY       35
 #define INIT_ORDER_MATERIALS       34
@@ -112,7 +127,6 @@ DEFINE_BITFIELD(runlevels, list(
 #define INIT_ORDER_AIR            -1
 #define INIT_ORDER_PLANETS        -2
 #define INIT_ORDER_PERSISTENCE    -3
-#define INIT_ORDER_ASSETS         -4
 #define INIT_ORDER_MISC_LATE      -5
 #define INIT_ORDER_HOLOMAPS       -5
 #define INIT_ORDER_NIGHTSHIFT     -5

code/__DEFINES/controllers/assets.dm

@@ -0,0 +1,8 @@
+//* asset loaded status
+
+/// not loaded at all
+#define ASSET_NOT_LOADED 1
+/// loading started, wait
+#define ASSET_IS_LOADING 2
+/// loading finished, go
+#define ASSET_FULLY_LOADED 3

code/__DEFINES/tgs.config.dm

@@ -9,4 +9,4 @@
 #define TGS_ERROR_LOG(message) log_world("TGS Error: [##message]")
 #define TGS_NOTIFY_ADMINS(event) message_admins(##event)
 #define TGS_CLIENT_COUNT GLOB.clients.len
-#define TGS_PROTECT_DATUM(Path) GENERAL_PROTECT_DATUM(##Path)
+#define TGS_PROTECT_DATUM(Path) VV_PROTECT(##Path)

code/__HELPERS/icons.dm

@@ -803,12 +803,6 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0
 		obj_flags &= ~FROZEN
 */
 
-/// Generate a filename for this asset
-/// The same asset will always lead to the same asset name
-/// (Generated names do not include file extention.)
-/proc/generate_asset_name(file)
-	return "asset.[md5(fcopy_rsc(file))]"
-
 /**
  * Converts an icon to base64. Operates by putting the icon in the iconCache savefile,
  * exporting it as text, and then parsing the base64 from that.
@@ -831,8 +825,6 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0
 		return
 //	if(SSlag_switch.measures[DISABLE_USR_ICON2HTML] && usr && !HAS_TRAIT(usr, TRAIT_BYPASS_MEASURES))
 //		return
-
-	var/key
 	var/icon/I = thing
 
 	if (!target)
@@ -850,14 +842,10 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0
 
 	if (!isicon(I))
 		if (isfile(thing)) //special snowflake
-			var/name = SANITIZE_FILENAME("[generate_asset_name(thing)].png")
-			if (!SSassets.cache[name])
-				SSassets.transport.register_asset(name, thing)
-			for (var/thing2 in targets)
-				SSassets.transport.send_assets(thing2, name)
+			var/url = SSassets.send_anonymous_file(targets, I, "png")
 			if(sourceonly)
-				return SSassets.transport.get_asset_url(name)
-			return "<img class='[extra_classes] icon icon-misc' src='[SSassets.transport.get_asset_url(name)]'>"
+				return url
+			return "<img class='[extra_classes] icon icon-misc' src='[url]'>"
 		var/atom/A = thing
 
 		I = A.icon
@@ -885,14 +873,10 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0
 
 	I = icon(I, icon_state, dir, frame, moving)
 
-	key = "[generate_asset_name(I)].png"
-	if(!SSassets.cache[key])
-		SSassets.transport.register_asset(key, I)
-	for (var/thing2 in targets)
-		SSassets.transport.send_assets(thing2, key)
+	var/url = SSassets.send_anonymous_file(targets, I, "png")
 	if(sourceonly)
-		return SSassets.transport.get_asset_url(key)
-	return "<img class='[extra_classes] icon icon-[icon_state]' src='[SSassets.transport.get_asset_url(key)]'>"
+		return url
+	return "<img class='[extra_classes] icon icon-[icon_state]' src='[url]'>"
 
 /proc/icon2base64html(thing)
 	if (!thing)

code/__HELPERS/type2type/type2type.dm

@@ -311,11 +311,6 @@
 		else //regex everything else (works for /proc too)
 			return lowertext(replacetext("[the_type]", "[type2parent(the_type)]/", ""))
 
-/// Return html to load a url.
-/// for use inside of browse() calls to html assets that might be loaded on a cdn.
-/proc/url2htmlloader(url)
-	return {"<html><head><meta http-equiv="refresh" content="0;URL='[url]'"/></head><body onLoad="parent.location='[url]'"></body></html>"}
-
 // Converts a string into ascii then converts it into hexadecimal.
 /proc/strtohex(str)
 	if(!istext(str)||!str)

code/___compile_options.dm

@@ -71,13 +71,6 @@
  */
 // #define USE_BYOND_TRACY
 
-
-/**
- * If defined, we will NOT defer asset generation till later in the game, and will instead do it all at once, during initiialize.
- */
-//#define DO_NOT_DEFER_ASSETS
-
-
 /**
  * If this is uncommented, Autowiki will generate edits and shut down the server.
  * Prefer the autowiki build target instead.
@@ -185,7 +178,12 @@
 	#define ZAS_BREAKPOINT_HOOKS
 #endif
 
+// ## Assets
 
+/**
+ * If defined, we will NOT defer asset generation till later in the game, and will instead do it all at once, during initiialize.
+ */
+//#define DO_NOT_DEFER_ASSETS
 
 // ## Overlays
 /**

code/controllers/configuration/entries/resources.dm

@@ -14,8 +14,8 @@
 /datum/config_entry/string/asset_cdn_webroot/ValidateAndSet(str_var)
 	if (!str_var || trim(str_var) == "")
 		return FALSE
-	if (str_var && str_var[length(str_var)] != "/")
-		str_var += "/"
+	if (str_var && str_var[length_char(str_var)] == "/")
+		str_var = copytext_char(str_var, 1, length_char(str_var))
 	return ..(str_var)
 
 /datum/config_entry/string/asset_cdn_url
@@ -25,6 +25,6 @@
 /datum/config_entry/string/asset_cdn_url/ValidateAndSet(str_var)
 	if (!str_var || trim(str_var) == "")
 		return FALSE
-	if (str_var && str_var[length(str_var)] != "/")
-		str_var += "/"
+	if (str_var && str_var[length_char(str_var)] == "/")
+		str_var = copytext_char(str_var, 1, length_char(str_var))
 	return ..(str_var)

code/controllers/subsystem/asset_loading.dm

@@ -6,25 +6,22 @@
 SUBSYSTEM_DEF(asset_loading)
 	name = "Asset Loading"
 	priority = FIRE_PRIORITY_ASSET_LOADING
+	// todo: hibernation
 	subsystem_flags = SS_NO_INIT
 	runlevels = RUNLEVEL_LOBBY | RUNLEVELS_DEFAULT
-	var/list/datum/asset/generate_queue = list()
 
-/datum/controller/subsystem/asset_loading/fire(resumed)
-	while(length(generate_queue))
-		var/datum/asset/to_load = generate_queue[generate_queue.len]
+	/// things waiting to be loaded
+	var/list/datum/asset_pack/loading = list()
 
-		to_load.queued_generation()
+/datum/controller/subsystem/asset_loading/proc/queue_asset_pack(datum/asset_pack/asset)
+	if(asset.loaded != ASSET_NOT_LOADED)
+		return
+	loading += asset
 
+/datum/controller/subsystem/asset_loading/fire(resumed)
+	while(length(loading))
+		var/datum/asset_pack/instance = loading[loading.len]
+		loading.len--
+		instance.ensure_ready(TRUE)
 		if(MC_TICK_CHECK)
 			return
-		generate_queue.len--
-
-/datum/controller/subsystem/asset_loading/proc/queue_asset(datum/asset/queue)
-#ifdef DO_NOT_DEFER_ASSETS
-	stack_trace("We queued an instance of [queue.type] for lateloading despite not allowing it")
-#endif
-	generate_queue += queue
-
-/datum/controller/subsystem/asset_loading/proc/dequeue_asset(datum/asset/queue)
-	generate_queue -= queue