64Bit Memory Management

README.md

@@ -18,12 +18,12 @@
 *** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
 *** https://www.markdownguide.org/basic-syntax/#reference-style-links
 -->
-[![Contributors][contributors-shield]][contributors-url]
 [![Forks][forks-shield]][forks-url]
 [![Stargazers][stars-shield]][stars-url]
-[![Issues][issues-shield]][issues-url]
 [![Build][built-shield]][built-url]
 [![Lines of Code][loc-shield]][loc-url]
+[![wakatime][wakatime-shield]][wakatime-url]
+
 
 <!-- PROJECT LOGO -->
 <br />
@@ -35,7 +35,7 @@
 <h3 align="center">Max OS</h3>
 
   <p align="center">
-    A 32bit hobby operating system written in C++
+    A 64bit hobby operating system written in C++
     <br />
     <a href="https://maxtyson123.github.io/MaxOS/html/"><strong>Explore the docs »</strong></a>
     <br />
@@ -79,7 +79,9 @@
 
 [![MaxOS][product-screenshot]](#)
 
-Max OS is a hobby operating system developed for the 32bit platform using C++ and Assembly. The project is currently in the early stages of development and is not yet ready for use. The project is being developed as a learning experience and is not intended to be used as a production operating system.
+!-- CURRENTLY IMPLEMENTING 64BIT SUPPORT, OS IS IN A BROKEN STATE --!
+
+Max OS is a hobby operating system developed for the 64bit platform using C++ and Assembly. The project is currently in the early stages of development and is not yet ready for use. The project is being developed as a learning experience and is not intended to be used as a production operating system.
 
 Max OS supports device drivers, memory management, multitasking, a GUI, and more. The project is being developed with the goal of being able to run on real hardware, however, it is currently only able to run on a virtual machine.
 
@@ -224,6 +226,7 @@ Distributed under the BSD 3-Clause License. See `LICENSE` for more information.
 * [OSDev Notes](https://github.com/dreamportdev/Osdev-Notes/)
 * [OSDev Subreddit](https://www.reddit.com/r/osdev/)
 * [Duck OS](https://github.com/byteduck/duckOS)
+* [Dream OS](https://github.com/dreamos82/Dreamos64)
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
@@ -241,5 +244,8 @@ Distributed under the BSD 3-Clause License. See `LICENSE` for more information.
 [issues-url]: https://github.com/maxtyson123/MaxOS/issues
 [built-shield]: https://img.shields.io/github/actions/workflow/status/maxtyson123/MaxOS/max-os.yml?style=for-the-badge
 [built-url]: https://github.com/maxtyson123/MaxOS/actions/workflows/max-os.yml
-[loc-shield]: https://img.shields.io/tokei/lines/github/maxtyson123/MaxOS?style=for-the-badge
-[loc-url]: https://github.com/maxtyson123/MaxOS
+[loc-shield]: https://tokei.rs/b1/github/maxtyson123/MaxOS?style=for-the-badge
+[loc-url]: https://github.com/maxtyson123/MaxOS
+[wakatime-shield]: https://wakatime.com/badge/github/maxtyson123/MaxOS.svg?style=for-the-badge
+[wakatime-url]: https://wakatime.com/badge/github/maxtyson123/MaxOS
+

docs/Styles/Pre Merge Checklist

@@ -0,0 +1,5 @@
+- Are to dos completed?
+- Are all the files in the correct location?
+- Kprintf used where necessary - no testing information?
+- Tested to ensure that the changes work?
+- Doxygen comments added?

kernel/include/common/kprint.h

@@ -6,7 +6,18 @@
 #define MAXOS_KPRINT_H
 
 #include <drivers/console/serial.h>
+#include <stdarg.h>
 
-void _kprintf(const char* format, ...);
+
+#define _kprintf(format, ...) \
+    _kprintf_internal(0, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__)
+
+#define ASSERT(condition, format, ...) \
+    if(!(condition)) { \
+        _kprintf_internal(3, __FILE__, __LINE__, __FUNCTION__, format, ##__VA_ARGS__); \
+        while(1); \
+    }
+
+void _kprintf_internal(uint8_t type, const char* file, int line, const char* func, const char* format, ...);
 
 #endif // MAXOS_KPRINT_H

kernel/include/hardwarecommunication/acpi.h

@@ -10,6 +10,7 @@
 #include <system/multiboot.h>
 #include <common/string.h>
 #include <memory/memorymanagement.h>
+#include <memory/physical.h>
 
 namespace MaxOS {
     namespace hardwarecommunication {

kernel/include/hardwarecommunication/apic.h

@@ -8,6 +8,7 @@
 #include <hardwarecommunication/port.h>
 #include <system/cpu.h>
 #include <hardwarecommunication/acpi.h>
+#include <memory/physical.h>
 
 namespace MaxOS {
     namespace hardwarecommunication {
@@ -17,6 +18,7 @@ namespace MaxOS {
 
         protected:
           uint64_t m_apic_base;
+          uint64_t m_apic_base_high;
           uint32_t m_id;
           bool m_x2apic;
 
@@ -80,6 +82,7 @@ namespace MaxOS {
             AdvancedConfigurationAndPowerInterface* m_acpi;
             MADT* m_madt;
             uint32_t m_address;
+            uint64_t m_address_high;
             uint32_t m_version;
             uint8_t m_max_redirect_entry;
 

kernel/include/hardwarecommunication/interrupts.h

@@ -61,7 +61,7 @@ namespace MaxOS {
          * @class InterruptManager
          * @brief Handles all interrupts and passes them to the correct handler
          */
-        class InterruptManager : public common::InputStream {
+        class InterruptManager {
             friend class InterruptHandler;
 
             protected:

kernel/include/memory/memorymanagement.h

@@ -8,6 +8,7 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <system/multiboot.h>
+#include <memory/virtual.h>
 
 namespace MaxOS{
 
@@ -33,21 +34,44 @@ namespace MaxOS{
          */
         class MemoryManager{
 
-          protected:
-              MemoryChunk* m_first_memory_chunk;
+          private:
+            MemoryChunk* m_first_memory_chunk;
+            MemoryChunk* m_last_memory_chunk;
+
+            VirtualMemoryManager* m_virtual_memory_manager;
+
+            MemoryChunk* expand_heap(size_t size);
 
           public:
               static MemoryManager* s_active_memory_manager;
-              static const uint64_t s_higher_half_offset { 0xFFFFFFFF80000000 };
 
-              MemoryManager(multiboot_tag_mmap* memory_map);
+              static const uint64_t s_higher_half_kernel_offset {  0xFFFFFFFF80000000 };
+              static const uint64_t s_higher_half_mem_offset    {  0xFFFF800000000000 };
+              static const uint64_t s_higher_half_mem_reserved  {  0x280000000 };
+              static const uint64_t s_higher_half_offset        { s_higher_half_mem_offset + s_higher_half_mem_reserved};
+              static const uint64_t s_hh_direct_map_offset      { s_higher_half_offset + PhysicalMemoryManager::s_page_size };
+
+              // Each chunk is aligned to 16 bytes
+              static const size_t s_chunk_alignment { 0x10 };
+
+              MemoryManager(VirtualMemoryManager* virtual_memory_manager);
               ~MemoryManager();
 
+              // Memory Management
               void* malloc(size_t size);
               void free(void* pointer);
+
+              // Utility Functions
               int memory_used();
+              size_t align(size_t size);
+
 
-              static uint64_t map_to_higher_half(uint64_t physical_address);
+              // Higher Half Memory Management
+              static void* to_higher_region(uintptr_t physical_address);
+              static void* to_lower_region(uintptr_t virtual_address);
+              static void* to_io_region(uintptr_t physical_address);
+              static void* to_dm_region(uintptr_t physical_address);
+              static bool in_higher_region(uintptr_t virtual_address);
         };
     }
 }

kernel/include/memory/physical.h

@@ -0,0 +1,143 @@
+//
+// Created by 98max on 1/30/2024.
+//
+
+#ifndef MAXOS_MEMORY_PHYSICAL_H
+#define MAXOS_MEMORY_PHYSICAL_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <system/multiboot.h>
+
+namespace MaxOS {
+
+  namespace memory {
+
+
+    #define PMLX_GET_INDEX(ADDR, LEVEL) (((uint64_t)ADDR & ((uint64_t)0x1ff << (12 + LEVEL * 9))) >> (12 + LEVEL * 9))
+
+    #define PML4_GET_INDEX(ADDR) PMLX_GET_INDEX(ADDR, 3)
+    #define PML3_GET_INDEX(ADDR) PMLX_GET_INDEX(ADDR, 2)
+    #define PML2_GET_INDEX(ADDR) PMLX_GET_INDEX(ADDR, 1)
+    #define PML1_GET_INDEX(ADDR) PMLX_GET_INDEX(ADDR, 0)
+
+      // Useful for readability
+      typedef void virtual_address_t;
+      typedef void physical_address_t;
+
+      typedef enum PageFlags {
+        None          = 0,
+        Present       = (1 << 0),
+        Write         = (1 << 1),
+        User          = (1 << 2),
+        WriteThrough  = (1 << 3),
+        CacheDisabled = (1 << 4),
+        Accessed      = (1 << 5),
+        Dirty         = (1 << 6),
+        HugePage      = (1 << 7),
+        Global        = (1 << 8)
+
+      } page_flags_t;
+
+
+      // Struct for a page table entry
+        typedef struct PageTableEntry {
+          uint64_t present : 1;
+          uint64_t write : 1;
+          uint64_t user : 1;
+          uint64_t write_through : 1;
+          uint64_t cache_disabled : 1;
+          uint64_t accessed : 1;
+          uint64_t dirty : 1;
+          uint64_t huge_page : 1;
+          uint64_t global : 1;
+          uint64_t available : 3;
+          uint64_t physical_address : 52;
+        } __attribute__((packed)) pte_t;
+
+      // Struct for a page map level
+      typedef struct PageMapLevel {
+        pte_t entries[512];
+      } __attribute__((packed)) pml_t;
+
+      class PhysicalMemoryManager{
+
+        private:
+          const uint8_t ROW_BITS = { 64 };
+
+          uint64_t* m_bit_map;
+          uint32_t m_total_entries;
+          uint32_t m_bitmap_size;
+          uint32_t m_used_frames;
+          uint64_t m_memory_size;
+
+          uint64_t m_anonymous_memory_physical_address;
+          uint64_t m_anonymous_memory_virtual_address;
+
+          multiboot_mmap_entry* m_mmap;
+          multiboot_tag_mmap*   m_mmap_tag;
+
+          uint64_t* m_pml4_root_address;
+          pte_t* m_pml4_root;
+
+          bool m_initialized;
+
+          // Table Management
+          void create_table(pml_t* table, pml_t* next_table, size_t index);
+          pte_t create_page_table_entry(uintptr_t address, size_t flags);
+          bool table_has_entry(pml_t* table, size_t index);
+          uint64_t* get_or_create_table(uint64_t* table, size_t index, size_t flags);
+          uint64_t* get_table_if_exists(uint64_t* table, size_t index);
+
+
+          uint64_t* get_bitmap_address();
+
+        public:
+
+          PhysicalMemoryManager(unsigned long reserved, system::Multiboot* multiboot, uint64_t pml4_root[512]);
+          ~PhysicalMemoryManager();
+
+
+          // Vars
+          static const uint32_t s_page_size = { 0x1000 };    // 4096 bytes
+          uint64_t get_memory_size();
+
+          // Pml4
+          uint64_t* get_pml4_root_address();
+
+          // Frame Management
+          void* allocate_frame();
+          void free_frame(void* address);
+
+          void* allocate_area(uint64_t start_address, size_t size);
+          void free_area(uint64_t start_address, size_t size);
+
+          // Map
+          virtual_address_t* map(virtual_address_t* virtual_address, size_t flags);
+          virtual_address_t* map(physical_address_t* physical, virtual_address_t* virtual_address, size_t flags);
+          virtual_address_t* map(physical_address_t* physical, virtual_address_t* virtual_address, size_t flags, uint64_t* pml4_root);
+          void map_area(virtual_address_t* virtual_address_start, size_t length, size_t flags);
+          void map_area(physical_address_t* physical_address_start, virtual_address_t* virtual_address_start, size_t length, size_t flags);
+          void identity_map(physical_address_t* physical_address, size_t flags);
+
+          void unmap(virtual_address_t* virtual_address);
+          void unmap(virtual_address_t* virtual_address, uint64_t* pml4_root);
+          void unmap_area(virtual_address_t* virtual_address_start, size_t length);
+
+          // Tools
+          static size_t size_to_frames(size_t size);
+          static size_t align_to_page(size_t size);
+          static size_t align_up_to_page(size_t size, size_t s_page_size);
+          static bool check_aligned(size_t size);
+          bool is_mapped(uintptr_t physical_address, uintptr_t virtual_address);
+          bool is_anonymous_available(size_t size);
+          bool is_multiboot_reserved(uint64_t address);
+
+          static PhysicalMemoryManager* s_current_manager;
+          void clean_page_table(uint64_t* table);
+      };
+  }
+
+}
+
+#endif // MAXOS_MEMORY_PHYSICAL_H

kernel/include/memory/virtual.h

@@ -0,0 +1,68 @@
+//
+// Created by 98max on 2/11/2024.
+//
+
+#ifndef MAXOS_VIRTUAL_H
+#define MAXOS_VIRTUAL_H
+
+#include <stdint.h>
+#include <stddef.h>
+#include <memory/physical.h>
+
+namespace MaxOS {
+  namespace memory {
+
+
+   typedef enum VirtualFlags {
+    // 0 - (1 << 8) are reserved for the page flags
+    Reserve = (1 << 9),
+
+  } virtual_flags_t;
+
+    typedef struct VirtualMemoryChunk{
+      uintptr_t start_address;
+      size_t size;
+      size_t flags;
+
+    } virtual_memory_chunk_t;
+
+    typedef struct VirtualMemoryRegion{
+      virtual_memory_chunk_t chunks[(PhysicalMemoryManager::s_page_size / sizeof(virtual_memory_chunk_t) - 1)];
+      struct VirtualMemoryRegion* next;
+
+    } __attribute__((packed)) virtual_memory_region_t;
+
+    class VirtualMemoryManager{
+
+      private:
+        uint64_t * m_pml4_root_address;
+        uint64_t * m_pml4_root_physical_address;
+        PhysicalMemoryManager* m_physical_memory_manager;
+        bool m_is_kernel;
+
+        virtual_memory_region_t* m_first_region;
+        virtual_memory_region_t* m_current_region;
+        size_t m_current_chunk;
+        size_t m_next_available_address;
+
+        static const size_t s_chunks_per_page = (PhysicalMemoryManager::s_page_size / sizeof(virtual_memory_chunk_t) - 1);
+        static const size_t s_reserved_space = 0x138000000;
+
+        void new_region();
+
+      public:
+        VirtualMemoryManager(bool is_kernel);
+        ~VirtualMemoryManager();
+
+        void* allocate(size_t size, size_t flags);
+        void* allocate(uint64_t address, size_t size, size_t flags);
+        void free(void* address);
+
+        size_t memory_used();
+
+    };
+  }
+}
+
+
+#endif // MAXOS_VIRTUAL_H