System asset refactor

.github/workflows/main.yml

@@ -110,7 +110,7 @@ jobs:
       # Runs a set of commands using the runners shell
       - name: Run prebuild steps
         run: |
-          python fips nebula set work ../fips-deploy
+          python fips nebula set work tests
           python fips nebula set toolkit .
           python fips ultralight
           python fips gen

code/addons/graphicsfeature/components/graphics.nidl

@@ -12,7 +12,7 @@
     "components": {
         "ModelResource": {
             "_type_": "resource",
-            "_default_": "mdl:system/placeholder.n3"
+            "_default_": "sysmdl:placeholder.n3"
         },
         "Camera": {
             "viewHandle": {

code/addons/physicsfeature/managers/physicsmanager.cc

@@ -75,8 +75,8 @@ void PhysicsManager::InitCreateActorProcessor()
                 Resources::CreateResource(res, "PHYS",
                     [world, trans, &opBuffer, entity, staticPID](Resources::ResourceId id)
                     {
-                        bool const dynamic = !Game::HasComponent(world, entity, staticPID);
-                        Physics::ActorId actorid = Physics::CreateActorInstance(id, trans, dynamic, Ids::Id32(entity));
+                        bool const isStatic = Game::HasComponent(world, entity, staticPID);
+                        Physics::ActorId actorid = Physics::CreateActorInstance(id, trans, isStatic ? Physics::ActorType::Static : Physics::ActorType::Dynamic, Ids::Id32(entity));
 
                         Game::Op::RegisterComponent regOp;
                         regOp.entity = entity;

code/addons/staticui/ultralight/ultralightrenderer.cc

@@ -169,10 +169,10 @@ UltralightRenderer::UpdateTexture(uint32_t texture_id, ultralight::Ref<ultraligh
     // Create buffer to upload to texture
     CoreGraphics::BufferCreateInfo bufInfo;
     bufInfo.usageFlags = CoreGraphics::BufferUsageFlag::TransferBufferSource;
-    bufInfo.byteSize = bitmap->size();
+    bufInfo.byteSize = (SizeT)bitmap->size();
     bufInfo.mode = CoreGraphics::HostLocal;
     bufInfo.data = bitmap->LockPixels();
-    bufInfo.dataSize = bitmap->size();
+    bufInfo.dataSize = (SizeT)bitmap->size();
     CoreGraphics::BufferId tempBuf = CoreGraphics::CreateBuffer(bufInfo);
     bitmap->UnlockPixels();
 

code/application/CMakeLists.txt

@@ -77,6 +77,4 @@ nebula_end_module()
 if(FIPS_WINDOWS)
 target_link_options(application PUBLIC "/WHOLEARCHIVE:application")
 endif()
-if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.16.0")
-    target_precompile_headers(application PRIVATE "application/stdneb.h")
-endif()
+target_precompile_headers(application PRIVATE "application/stdneb.h")

code/foundation/CMakeLists.txt

@@ -5,13 +5,24 @@
 if(FIPS_LINUX)
     FIND_PACKAGE(CURL)
 endif()
+
+if (EXISTS "${NROOT}/syswork/export.zip")
+    create_resource("${NROOT}/syswork/" "${NROOT}/syswork/export.zip" system_resources)
+    set_target_properties(system_resources PROPERTIES FOLDER Resources)
+    set_target_properties(system_resources-res PROPERTIES FOLDER Resources)
+endif()
+
 nebula_begin_module(foundation)
 
     if(FIPS_LINUX)
     fips_libs(uuid cjson stdc++fs)
     endif()
     target_include_directories(foundation PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
     fips_deps(nebula-libs zlib tinyxml)
+    if(TARGET system_resources)
+        fips_deps(system_resources)
+        target_compile_definitions(foundation PUBLIC HAS_EMBEDDED_EXPORT)
+    endif()
 
     if(FIPS_WINDOWS)
         fips_libs(curl)
@@ -182,6 +193,8 @@ nebula_begin_module(foundation)
             filewatcher.cc
             filewatcher.h
             mediatype.h
+            embeddedmemorystream.cc
+            embeddedmemorystream.h
             memorystream.cc
             memorystream.h
             schemeregistry.cc

code/foundation/io/archfs/archivebase.cc

@@ -36,7 +36,7 @@ ArchiveBase::~ArchiveBase()
     found, or wrong format).
 */
 bool
-ArchiveBase::Setup(const URI& archiveFileURI)
+ArchiveBase::Setup(const URI& archiveFileURI, const Util::String& rootPath)
 {
     n_assert(!this->IsValid());
     this->uri = archiveFileURI;

code/foundation/io/archfs/archivebase.h

@@ -26,7 +26,7 @@ class ArchiveBase : public Core::RefCounted
     virtual ~ArchiveBase();
 
     /// setup the archive from an URI (without file extension)
-    bool Setup(const URI& archiveURI);
+    bool Setup(const URI& archiveURI, const Util::String& rootPath);
     /// discard the archive
     void Discard();
     /// return true if archive is valid
@@ -70,4 +70,3 @@ ArchiveBase::GetURI() const
 //------------------------------------------------------------------------------
 
 
-

code/foundation/io/archfs/archivefilesystembase.cc

@@ -76,11 +76,22 @@ ArchiveFileSystemBase::Discard()
 */
 Ptr<Archive>
 ArchiveFileSystemBase::Mount(const URI& uri)
+{
+    return MountEmbedded(uri, "");
+}
+//------------------------------------------------------------------------------
+/**
+    This "mounts" an archive file by creating a new Archive object
+    and adding it to the archive dictionary. If mounting fails, an invalid
+    pointer will be returned!
+*/
+Ptr<Archive>
+ArchiveFileSystemBase::MountEmbedded(const URI& uri, const Util::String& rootPath)
 {
     n_assert(!this->IsMounted(uri));
     String path = AssignRegistry::Instance()->ResolveAssigns(uri).LocalPath();
     Ptr<Archive> newArchive = Archive::Create();
-    if (newArchive->Setup(uri))
+    if (newArchive->Setup(uri, rootPath))
     {
         this->critSect.Enter();
         this->archives.Add(path, newArchive);
@@ -93,6 +104,7 @@ ArchiveFileSystemBase::Mount(const URI& uri)
     return newArchive;
 }
 
+
 //------------------------------------------------------------------------------
 /**
     Unmount a zip archive, this will remove the archive from the internal

code/foundation/io/archfs/archivefilesystembase.h

@@ -38,6 +38,8 @@ class ArchiveFileSystemBase : public Core::RefCounted
 
     /// mount an archive
     virtual Ptr<Archive> Mount(const URI& uri);
+    /// mount an embedded archive
+    virtual Ptr<Archive> MountEmbedded(const URI& uri, const Util::String& rootPath);
     /// unmount an archive by URI
     virtual void Unmount(const URI& uri);
     /// unmount an archive by pointer

code/foundation/io/assignregistry.cc

@@ -138,44 +138,39 @@ AssignRegistry::SetupProjectAssigns()
     String rootDir = Core::CoreServer::Instance()->GetRootDirectory().AsString();
     this->SetAssign(Assign("root", rootDir));
     this->SetAssign(Assign("export", "root:export"));
-
+    
     // setup content assigns
-    this->SetAssign(Assign("msh", "export:meshes"));
-    this->SetAssign(Assign("ani", "export:anims"));
-    this->SetAssign(Assign("ske", "export:skeletons"));
-    this->SetAssign(Assign("data", "export:data"));        
+    this->SetAssign(Assign("msh",   "export:meshes"));
+    this->SetAssign(Assign("ani",   "export:anims"));
+    this->SetAssign(Assign("ske",   "export:skeletons"));
+    this->SetAssign(Assign("data",  "export:data"));        
     this->SetAssign(Assign("video", "export:video"));
-    this->SetAssign(Assign("db", "export:db"));
-    this->SetAssign(Assign("seq", "export:sequences"));
+    this->SetAssign(Assign("db",    "export:db"));
+    this->SetAssign(Assign("seq",   "export:sequences"));
     this->SetAssign(Assign("audio", "export:audio"));
-    this->SetAssign(Assign("stream", "export:audio"));
-    this->SetAssign(Assign("shd", "export:shaders"));
-    this->SetAssign(Assign("tex", "export:textures"));
+    this->SetAssign(Assign("shd",   "export:shaders"));
+    this->SetAssign(Assign("tex",   "export:textures"));
     this->SetAssign(Assign("frame", "export:frame"));
-    this->SetAssign(Assign("mdl", "export:models"));
-    this->SetAssign(Assign("phys", "export:physics"));
+    this->SetAssign(Assign("mdl",   "export:models"));
+    this->SetAssign(Assign("phys",  "export:physics"));
     this->SetAssign(Assign("audio", "export:audio"));    
-    this->SetAssign(Assign("sui", "export:sui"));       
-    this->SetAssign(Assign("mat", "export:materials"));
-    this->SetAssign(Assign("sur", "export:surfaces"));
-    this->SetAssign(Assign("scr", "root:data/scripts"));
-    this->SetAssign(Assign("gui", "root:data/gui"));
-
-    // setup special assigns which may use original meshes but still needs an explicit load
-    this->SetAssign(Assign("parmsh", "export:meshes"));
-
+    this->SetAssign(Assign("sui",   "export:sui"));       
+    this->SetAssign(Assign("mat",   "export:materials"));
+    this->SetAssign(Assign("sur",   "export:surfaces"));
+    this->SetAssign(Assign("scr",   "root:data/scripts"));
+    this->SetAssign(Assign("gui",   "root:data/gui"));
+
     // assign for nav meshes
     this->SetAssign(Assign("nav", "root:data/navigation"));
 
     // setup special system assigns (for placeholders, etc...)
-    this->SetAssign(Assign("sysmsh", "export:meshes"));
-    this->SetAssign(Assign("systex", "export:textures"));
-    this->SetAssign(Assign("sysanim", "export:anims"));
-
-    // Nebula2 backward compat assigns:
-    this->SetAssign(Assign("meshes", "export:meshes"));
-    this->SetAssign(Assign("anims", "export:anims"));
-    this->SetAssign(Assign("textures", "export:textures"));
+    this->SetAssign(Assign("sysmsh",    "msh:system"));
+    this->SetAssign(Assign("systex",    "tex:system"));
+    this->SetAssign(Assign("sysani",    "ani:system"));
+    this->SetAssign(Assign("sysmdl",    "mdl:system"));
+    this->SetAssign(Assign("syssur",    "sur:system"));
+    this->SetAssign(Assign("sysphys",   "phys:system"));
+
 
     // tools assigns
     this->SetAssign(Assign("tool", Core::CoreServer::Instance()->GetToolDirectory().AsString()));    

code/foundation/io/embeddedmemorystream.cc

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+//  embeddedmemorystream.cc
+//  (C) 2023 Individual contributors, see AUTHORS file
+//------------------------------------------------------------------------------
+#include "foundation/stdneb.h"
+#include "io/embeddedmemorystream.h"
+
+#ifdef HAS_EMBEDDED_EXPORT
+#include "system_resources.h"
+#endif
+
+namespace IO
+{
+__ImplementClass(IO::EmbeddedMemoryStream, 'EMSR', IO::Stream);
+
+//------------------------------------------------------------------------------
+/**
+*/
+EmbeddedMemoryStream::EmbeddedMemoryStream() 
+{
+    // empty
+}
+
+//------------------------------------------------------------------------------
+/**
+*/
+EmbeddedMemoryStream::~EmbeddedMemoryStream()
+{
+    // close the stream if still open
+    if (this->IsOpen())
+    {
+        this->Close();
+    }
+
+    // release memory buffer if allocated
+    if (0 != this->buffer)
+    {
+        Memory::Free(Memory::StreamDataHeap, this->buffer);
+        this->buffer = 0;
+    }
+}
+
+//------------------------------------------------------------------------------
+/**
+*/
+bool
+EmbeddedMemoryStream::CanWrite() const
+{
+    return false;
+}
+
+
+
+//------------------------------------------------------------------------------
+/**
+    Open the stream for reading 
+*/
+bool
+EmbeddedMemoryStream::Open()
+{
+    n_assert(!this->IsOpen());
+#ifdef HAS_EMBEDDED_EXPORT
+    if (Stream::Open())
+    {
+        Util::String localPath = this->GetURI().LocalPath();
+        auto const& resource = system_resources::get_resource(localPath.AsCharPtr());
+        n_assert(resource.size_ > 0);
+        this->size = resource.size_;
+        this->capacity = resource.size_;
+        this->buffer = (unsigned char*)resource.ptr_;
+        this->position = 0;
+        return true;
+    }
+#else
+    n_error("trying to use missing embedded export");
+#endif
+    return false;
+}
+
+} // namespace IO

code/foundation/io/embeddedmemorystream.h

@@ -0,0 +1,31 @@
+#pragma once
+//------------------------------------------------------------------------------
+/**
+    @class IO::EmbeddedMemoryStream
+
+    Implements a stream class which reads from an embedded resource 
+
+    @copyright
+    (C) 2023 Individual contributors, see AUTHORS file
+*/
+#include "io/memorystream.h"
+
+//------------------------------------------------------------------------------
+namespace IO
+{
+class EmbeddedMemoryStream : public MemoryStream
+{
+    __DeclareClass(EmbeddedMemoryStream);
+public:
+    /// constructor
+    EmbeddedMemoryStream();
+    /// destructor
+    virtual ~EmbeddedMemoryStream();
+    /// memory streams support writing
+    virtual bool CanWrite() const;
+    /// open the stream
+    virtual bool Open();
+};
+
+} // namespace IO
+//------------------------------------------------------------------------------

code/foundation/io/ioserver.cc

@@ -13,6 +13,7 @@
 #include "io/archfs/archive.h"
 #include "io/archfs/archivefilesystem.h"
 #include "io/filewatcher.h"
+#include "io/filestream.h"
 #include <filesystem>
 
 namespace IO
@@ -126,8 +127,21 @@ IoServer::CreateStream(const URI& uri) const
 {
     n_assert(!uri.IsEmpty());
     n_assert(SchemeRegistry::Instance()->IsUriSchemeRegistered(uri.Scheme()));
-    SchemeRegistry* schemeRegistry = SchemeRegistry::Instance();
+    const Core::Rtti & streamClass = SchemeRegistry::Instance()->GetStreamClassByUriScheme(uri.Scheme());
 
+#if !PUBLIC_BUILD
+    if (streamClass == FileStream::RTTI)
+    {
+        const String path = uri.GetHostAndLocalPath();
+        n_assert(path.IsValid());
+        if (FSWrapper::FileExists(path))
+        {
+            Ptr<Stream> stream = (Stream*)streamClass.Create();
+            stream->SetURI(uri);
+            return stream;
+        }
+    }
+#endif
     // check if the URI points into a mounted archive
     if (this->IsArchiveFileSystemEnabled())
     {
@@ -154,6 +168,17 @@ IoServer::MountArchive(const URI& uri)
     return archive.isvalid();
 }
 
+//------------------------------------------------------------------------------
+/**
+*/
+bool
+IoServer::MountEmbeddedArchive(const URI& uri)
+{
+    Ptr<Archive> archive = this->archiveFileSystem->MountEmbedded(uri, "root:");
+    return archive.isvalid();
+}
+
+
 //------------------------------------------------------------------------------
 /**
 */

code/foundation/io/ioserver.h

@@ -48,6 +48,8 @@ class IoServer : public Core::RefCounted
 
     /// mount a file archive (without archive file extension!)
     bool MountArchive(const URI& uri);
+    /// mount an embedded file archive
+    bool MountEmbeddedArchive(const URI& uri);
     /// unmount a file archive (without archive file extension!)
     void UnmountArchive(const URI& uri);
     /// return true if a archive is mounted (without archive file extension!)

code/foundation/io/memorystream.h

@@ -58,7 +58,7 @@ class MemoryStream : public Stream
     /// get a direct "raw" pointer to the data
     void* GetRawPointer() const;
 
-private:
+protected:
     /// re-allocate the memory buffer
     void Realloc(Size s);
     /// return true if there's enough space for n more bytes