From 8d714c62f943a799c63ceb06e3573bede80ccb4c Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 31 Mar 2023 15:22:07 +0100 Subject: [PATCH 01/23] Ensure index is also an int --- spinn_front_end_common/utilities/database/database_writer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index 223e21fe2f..1a26e2cd63 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -264,7 +264,8 @@ def create_atom_to_event_id_mapping(self, machine_vertices): INSERT INTO event_to_atom_mapping( vertex_id, event_id, atom_id) VALUES (?, ?, ?) - """, ((m_vertex_id, int(key), i) for i, key in atom_keys) + """, ((m_vertex_id, int(key), int(i)) + for i, key in atom_keys) ) def add_lpg_mapping(self): From 6c2c94a09e35bf235c2e67c4c485ba5818f517ea Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Thu, 18 May 2023 16:20:11 +0100 Subject: [PATCH 02/23] Go back to normal keys even when multi-dimensional --- .../utilities/database/database_writer.py | 4 ++-- .../reverse_ip_tag_multicast_source_machine_vertex.py | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index 156ef06be0..e21d9ac6e0 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -21,7 +21,6 @@ AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) from spinnman.spalloc import SpallocJob from spinn_front_end_common.utility_models import LivePacketGather -from pacman.utilities.utility_calls import get_field_based_keys logger = FormatAdapter(logging.getLogger(__name__)) DB_NAME = "input_output_database.sqlite3" @@ -255,7 +254,8 @@ def create_atom_to_event_id_mapping(self, machine_vertices): # at which point there is nothing to do here anyway if r_info is not None: vertex_slice = m_vertex.vertex_slice - keys = get_field_based_keys(r_info.key, vertex_slice) + key = r_info.key + keys = range(key, key + vertex_slice.n_atoms) start = vertex_slice.lo_atom atom_keys = [(i, k) for i, k in enumerate(keys, start)] m_vertex_id = self.__vertex_to_id[m_vertex] diff --git a/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py b/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py index 8dad5a7545..c693700f91 100644 --- a/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py +++ b/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py @@ -25,7 +25,6 @@ from pacman.model.resources import ReverseIPtagResource, VariableSDRAM from pacman.model.graphs.common import Slice from pacman.model.graphs.machine import MachineVertex -from pacman.utilities.utility_calls import get_field_based_keys from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.helpful_functions import ( locate_memory_region_for_placement) @@ -494,7 +493,7 @@ def _fill_send_buffer_2d(self, key_base): end_time_step = FecDataView.get_current_run_timesteps() if first_time_step == end_time_step: return - keys = get_field_based_keys(key_base, self._vertex_slice) + keys = numpy.arange(key_base, key_base + self._vertex_slice.n_atoms) for atom in range(self._vertex_slice.n_atoms): for tick in sorted(self._send_buffer_times[atom]): if self._is_in_range(tick, first_time_step, end_time_step): @@ -512,11 +511,10 @@ def _fill_send_buffer_1d(self, key_base): end_time_step = FecDataView.get_current_run_timesteps() if first_time_step == end_time_step: return - keys = get_field_based_keys(key_base, self._vertex_slice) - key_list = [keys[atom] for atom in range(self._vertex_slice.n_atoms)] + keys = numpy.arange(key_base, key_base + self._vertex_slice.n_atoms) for tick in sorted(self._send_buffer_times): if self._is_in_range(tick, first_time_step, end_time_step): - self._send_buffer.add_keys(tick, key_list) + self._send_buffer.add_keys(tick, keys) @staticmethod def _generate_prefix(virtual_key, prefix_type): From ca4c44f031f9a0b4658fa2531da885c784d5131b Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Wed, 24 May 2023 08:22:14 +0100 Subject: [PATCH 03/23] Fix keys --- spinn_front_end_common/utilities/database/database_writer.py | 4 ++-- .../reverse_ip_tag_multicast_source_machine_vertex.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index e21d9ac6e0..5bfa48b4b5 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -21,6 +21,7 @@ AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) from spinnman.spalloc import SpallocJob from spinn_front_end_common.utility_models import LivePacketGather +from pacman.utilities.utility_calls import get_keys logger = FormatAdapter(logging.getLogger(__name__)) DB_NAME = "input_output_database.sqlite3" @@ -254,8 +255,7 @@ def create_atom_to_event_id_mapping(self, machine_vertices): # at which point there is nothing to do here anyway if r_info is not None: vertex_slice = m_vertex.vertex_slice - key = r_info.key - keys = range(key, key + vertex_slice.n_atoms) + keys = get_keys(r_info.key, vertex_slice) start = vertex_slice.lo_atom atom_keys = [(i, k) for i, k in enumerate(keys, start)] m_vertex_id = self.__vertex_to_id[m_vertex] diff --git a/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py b/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py index c693700f91..c0c3b8dcb3 100644 --- a/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py +++ b/spinn_front_end_common/utility_models/reverse_ip_tag_multicast_source_machine_vertex.py @@ -25,6 +25,7 @@ from pacman.model.resources import ReverseIPtagResource, VariableSDRAM from pacman.model.graphs.common import Slice from pacman.model.graphs.machine import MachineVertex +from pacman.utilities.utility_calls import get_keys from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.helpful_functions import ( locate_memory_region_for_placement) @@ -493,7 +494,7 @@ def _fill_send_buffer_2d(self, key_base): end_time_step = FecDataView.get_current_run_timesteps() if first_time_step == end_time_step: return - keys = numpy.arange(key_base, key_base + self._vertex_slice.n_atoms) + keys = get_keys(key_base, self._vertex_slice) for atom in range(self._vertex_slice.n_atoms): for tick in sorted(self._send_buffer_times[atom]): if self._is_in_range(tick, first_time_step, end_time_step): @@ -511,7 +512,7 @@ def _fill_send_buffer_1d(self, key_base): end_time_step = FecDataView.get_current_run_timesteps() if first_time_step == end_time_step: return - keys = numpy.arange(key_base, key_base + self._vertex_slice.n_atoms) + keys = get_keys(key_base, self._vertex_slice) for tick in sorted(self._send_buffer_times): if self._is_in_range(tick, first_time_step, end_time_step): self._send_buffer.add_keys(tick, keys) From 3eeb43262d268510817bf4c14565d044bce38000 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 26 May 2023 07:52:04 +0100 Subject: [PATCH 04/23] Remove bitfield compression and simplify --- .../include/malloc_extras.h | 70 +- .../front_end_common_lib/src/malloc_extras.c | 576 +------- c_common/models/compressors/Makefile | 5 +- .../bit_field_ordered_covering_compressor.mk | 21 - .../compressors/bit_field_pair_compressor.mk | 23 - c_common/models/compressors/sorter.mk | 25 - .../bit_field_table_generator.h | 278 ---- .../compressor_sorter_structs.h | 154 --- .../src/bit_field_common/routing_tables.h | 195 --- .../bit_field_common/routing_tables_utils.h | 161 --- .../compressors/src/bit_field_compressor.c | 405 ------ .../src/compressor_includes/pair_minimize.h | 15 +- .../models/compressors/src/simple/rt_single.h | 14 - .../src/simple/simple_compressor.c | 7 +- .../compressors/src/sorter/bit_field_reader.h | 330 ----- .../sorter/bit_field_sorter_and_searcher.c | 1163 ----------------- .../machine_bit_field_router_compressor.py | 799 ----------- 17 files changed, 31 insertions(+), 4210 deletions(-) delete mode 100644 c_common/models/compressors/bit_field_ordered_covering_compressor.mk delete mode 100644 c_common/models/compressors/bit_field_pair_compressor.mk delete mode 100644 c_common/models/compressors/sorter.mk delete mode 100644 c_common/models/compressors/src/bit_field_common/bit_field_table_generator.h delete mode 100644 c_common/models/compressors/src/bit_field_common/compressor_sorter_structs.h delete mode 100644 c_common/models/compressors/src/bit_field_common/routing_tables.h delete mode 100644 c_common/models/compressors/src/bit_field_common/routing_tables_utils.h delete mode 100644 c_common/models/compressors/src/bit_field_compressor.c delete mode 100644 c_common/models/compressors/src/sorter/bit_field_reader.h delete mode 100644 c_common/models/compressors/src/sorter/bit_field_sorter_and_searcher.c delete mode 100644 spinn_front_end_common/interface/interface_functions/machine_bit_field_router_compressor.py diff --git a/c_common/front_end_common_lib/include/malloc_extras.h b/c_common/front_end_common_lib/include/malloc_extras.h index ea52390036..8e7370751d 100644 --- a/c_common/front_end_common_lib/include/malloc_extras.h +++ b/c_common/front_end_common_lib/include/malloc_extras.h @@ -37,27 +37,8 @@ typedef enum exit_states_for_user_one { DETECTED_MALLOC_FAILURE = 4 } exit_states_for_user_one; -//! An SDRAM block outside the heap -typedef struct sdram_block { - //! Base address of where the SDRAM block starts - uchar *sdram_base_address; - //! Size of block in bytes - uint size; -} sdram_block; - -//! Holds host-allocated SDRAM blocks outside the heap -typedef struct available_sdram_blocks { - //! Number of blocks of SDRAM which can be utilised outside of alloc - int n_blocks; - //! VLA of SDRAM blocks - sdram_block blocks[]; -} available_sdram_blocks; - // =========================================================================== -//! \brief Turn off safety code if wanted -void malloc_extras_turn_off_safety(void); - //! \brief Turn on printing //! \note Printing of allocations can take a lot of IOBUF space. void malloc_extras_turn_on_print(void); @@ -65,59 +46,10 @@ void malloc_extras_turn_on_print(void); //! \brief Turn off printing void malloc_extras_turn_off_print(void); -//! \brief Get the pointer to the stolen heap -//! \return the heap pointer. -heap_t *malloc_extras_get_stolen_heap(void); - -#if 0 -static inline void terminate(uint result_code) __attribute__((noreturn)); -#endif - //! \brief Stops execution with a result code //! \param[in] result_code: code to put in user 1 void malloc_extras_terminate(uint result_code); -//! \brief Checks a pointer for safety stuff -//! \param[in] ptr: the malloc pointer to check for memory overwrites -//! \return true if nothing is broken, false if there was detected overwrites. -bool malloc_extras_check(void *ptr); - -//! \brief Checks all malloc()s with a given marker. -//! \param[in] marker: the numerical marker for this test, allowing easier -//! tracking of where this check was called in the user application code -//! \internal probably should be a string marker, but meh -void malloc_extras_check_all_marked(int marker); - -//! \brief Checks all malloc's for overwrites with no marker -//! \details Calls malloc_extras_check_all_marked(), but does not provide an -//! easy marker to track back to the application user code. -void malloc_extras_check_all(void); - -//! \brief Update heap to join in the extra space from another heap. -//! \param[in] heap_location: address where heap is located -//! \return true states the initialisation was successful (or not) -bool malloc_extras_initialise_with_fake_heap(heap_t *heap_location); - -//! \brief Builds a new heap based off stolen sdram blocks from cores -//! synaptic matrices. -//! \details Needs to merge in the true SDRAM free heap, as otherwise it's -//! impossible to free the block properly. -//! \param[in] sizes_region; the sdram address where the free regions exist -//! \return None -bool malloc_extras_initialise_and_build_fake_heap( - available_sdram_blocks *sizes_region); - -//! \brief Builds a new heap with no stolen SDRAM and sets up the malloc -//! tracker. -//! \return true is a successful initialisation and false otherwise. -bool malloc_extras_initialise_no_fake_heap_data(void); - -//! \brief Frees the sdram allocated from whatever heap it came from -//! \param[in] ptr: the address to free. could be DTCM or SDRAM -//! \param[in] marker: the numerical marker for this test, allowing easier -//! tracking of where this check was called in the user application code -void malloc_extras_free_marked(void *ptr, int marker); - //! \brief Frees a pointer without any marker for application code //! \param[in] ptr: the pointer to free. void malloc_extras_free(void *ptr); @@ -127,7 +59,7 @@ void malloc_extras_free(void *ptr); //! and size recordings. //! \param[in] bytes: the number of bytes to allocate from SDRAM. //! \return the pointer to the location in SDRAM to use in application code. -void *malloc_extras_sdram_malloc_wrapper(uint bytes); +void *malloc_extras_sdram_malloc(uint bytes); //! \brief Allows a search of the 2 heaps available. (DTCM, stolen SDRAM) //! \note Commented out as this can cause stack overflow issues quickly. diff --git a/c_common/front_end_common_lib/src/malloc_extras.c b/c_common/front_end_common_lib/src/malloc_extras.c index 37a96af233..f2e08453e5 100644 --- a/c_common/front_end_common_lib/src/malloc_extras.c +++ b/c_common/front_end_common_lib/src/malloc_extras.c @@ -19,10 +19,6 @@ #include #include -//! debug flag to lock in safety features -#define SAFETY_FLAG 0xDEADBEEF -//! amount of extra space _per allocation_ to add for the safety checker code -#define EXTRA_BYTES 64 //! offset used to compute location of the heap block metadata #define MINUS_POINT 60 //! the number of bytes in a word @@ -35,9 +31,6 @@ //============================================================================ // control flags -//! debug flag to lock in safety features -bool safety = true; - //! \brief flag to help with debugging bool to_print = false; @@ -46,24 +39,10 @@ bool to_print = false; bool use_dtcm = true; //============================================================================ -// global variables - -//! a extra heap, that exploits SDRAM which can be easily regenerated. -static heap_t *stolen_sdram_heap = NULL; - -//! tracker for mallocs -void **malloc_points = NULL; - -//! base line for the tracker array size. will grow with usage -int malloc_points_size = 4; // =========================================================================== // functions -void malloc_extras_turn_off_safety(void) { - safety = false; -} - void malloc_extras_turn_on_print(void) { to_print = true; } @@ -72,10 +51,6 @@ void malloc_extras_turn_off_print(void) { to_print = false; } -heap_t *malloc_extras_get_stolen_heap(void) { - return stolen_sdram_heap; -} - #if 0 static inline void terminate(uint result_code) __attribute__((noreturn)); #endif @@ -90,568 +65,55 @@ void malloc_extras_terminate(uint result_code) { spin1_exit(0); } -bool malloc_extras_check(void *ptr) { - // only check if safety is turned on. else its not possible to check. - if (safety) { - int *int_pointer = (int *) ptr; - int_pointer = int_pointer - 1; - int words = int_pointer[0]; - - for (int buffer_index = 0; buffer_index < BUFFER_WORDS; - buffer_index++) { - uint32_t flag = int_pointer[words + buffer_index]; - if (flag != SAFETY_FLAG) { - bool found = false; - for (int index = 0; index < malloc_points_size; index ++) { - if ((malloc_points[index] != 0) && - (malloc_points[index] == ptr)) { - found = true; - } - } - if (found) { - log_error("flag is actually %x for ptr %x", flag, ptr); - } else { - log_error("Unexpected ptr %x", ptr); - } - return false; - } - } - return true; - } - return true; -} - -//! \brief allows the ability to read the size of a malloc. -//! \param[in] ptr: the pointer to get the size in words of. -//! \return returns the size of a given malloc in words. -int malloc_extras_malloc_size(void *ptr) { - // only able to be figured out if safety turned on. - if (safety) { - // locate and return the len at the front of the malloc. - int *int_pointer = (int *) ptr; - int_pointer = int_pointer - 1; - return int_pointer[0]; - } - - log_error("there is no way to measure size when the safety is off."); - //Not know so return 0 - return 0; -} - -//! \brief checks a given pointer with a marker -//! \param[in] ptr: the pointer marker for whats being checked. -//! \param[in] marker: the numerical marker for this test. allowing easier -//! tracking of where this check was called in the user application code -//! (probably should be a string. but meh) -void malloc_extras_check_marked(void *ptr, int marker) { - // only check if safety turned on - if (safety) { - if (!malloc_extras_check(ptr)) { - log_error("test failed with marker %d", marker); - malloc_extras_terminate(DETECTED_MALLOC_FAILURE); - } - } else { - log_error("check cannot operate with safety turned off."); - } -} - -void malloc_extras_check_all_marked(int marker) { - // only check if safety turned on. else pointless. - if (safety) { - bool failed = false; - for (int index = 0; index < malloc_points_size; index ++) { - if (malloc_points[index] != 0 && - !malloc_extras_check(malloc_points[index])) { - log_error("the malloc with index %d has overran", index); - log_error("this test is marked by marker %d", marker); - failed = true; - } - } - - if (failed) { - malloc_extras_terminate(DETECTED_MALLOC_FAILURE); - } - } else { - log_error("cannot do checks with safety turned off"); - } -} - -void malloc_extras_check_all(void) { - malloc_extras_check_all_marked(-1); -} - -//! \brief cycles through the true heap and figures how many blocks there are -//! to steal. -//! \param[in] sdram_heap: the true SDRAM heap -//! \return the number of SDRAM blocks to utilise -static inline int find_n_available_mallocs(heap_t *sdram_heap) { - int n_available_true_malloc = 0; - block_t *free_blk = sdram_heap->free; - - // traverse blocks till none more available - while (free_blk != NULL) { - free_blk = free_blk->free; - n_available_true_malloc += 1; - } - return n_available_true_malloc; -} - -//! \brief builds a tracker for mallocs. for debug purposes -static void build_malloc_tracker(void) { - // malloc tracker - malloc_points = sark_xalloc( - stolen_sdram_heap, malloc_points_size * sizeof(void*), 0, - ALLOC_LOCK); - if (malloc_points == NULL) { - log_error("FAILED to allocate the tracker code!"); - rt_error(RTE_SWERR); - } - - // set tracker. - for (int index = 0; index < malloc_points_size; index ++) { - malloc_points[index] = 0; - } -} - -//! \brief count how much space available given expected block costs -//! \param[in] sizes_region: the SDRAM loc where addresses to steal are located -//! \return size available given expected block costs -static inline uint find_free_space_available( - available_sdram_blocks *sizes_region) { - uint free = 0; - for (int index = 0; index < sizes_region->n_blocks; index++) { - free += sizes_region->blocks[index].size - sizeof(block_t); - } - return free; -} - -//! \brief steals all SDRAM spaces from true heap -//! \param[in] list_of_available_blocks: location for stolen heap bits to go -//! \return true if successful. false otherwise -static inline bool add_heap_to_collection( - sdram_block *list_of_available_blocks) { - // go through true heap and allocate and add to end of list. - int position = 0; - - // loop over the true heap and add accordingly. - while (sv->sdram_heap->free != NULL) { - block_t *next_blk = sv->sdram_heap->free->next; - - // get next size minus the size it'll need to put in when alloc'ing - int size = ((uchar *) next_blk - (uchar *) sv->sdram_heap->free) - - sizeof(block_t); - - // make life easier by saying blocks have to be bigger than the heap. - // so all spaces can be used for heaps - uchar *b_address = sark_xalloc(sv->sdram_heap, size, 0, ALLOC_LOCK); - if (b_address == NULL) { - log_error("failed to allocate %d", size); - return false; - } - list_of_available_blocks[position].sdram_base_address = b_address; - list_of_available_blocks[position].size = size; - stolen_sdram_heap->free_bytes += size; - position++; - } - return true; -} - -//! \brief builds the new heap struct over our stolen and proper claimed -//! SDRAM spaces. -//! \param[in] sizes_region: the struct that contains addresses and sizes that -//! have already been allocated, but which we can use. -//! \param[in] n_mallocs: the number of mallocs expected to be done. -//! \param[in] list_of_available_blocks: the mallocs from the original heap. -static inline void make_heap_structure( - available_sdram_blocks *sizes_region, int n_mallocs, - sdram_block *list_of_available_blocks) { - // generate position pointers - int stolen_current_index = 0; - int heap_current_index = 0; - bool first = true; - block_t *previous = NULL; - block_t *previous_free = NULL; - - // generate heap pointers - while (stolen_current_index < sizes_region->n_blocks || - heap_current_index < n_mallocs) { - // build pointers to try to reduce code space - int *to_process; - sdram_block *to_process_blocks; - - // determine which tracker to utilise - uint top_stolen = (uint) sizes_region->blocks[ - stolen_current_index].sdram_base_address; - uint top_true = (uint) list_of_available_blocks[ - heap_current_index].sdram_base_address; - - // determine which one to utilise now - if ((stolen_current_index < sizes_region->n_blocks) && - top_stolen < top_true) { - to_process = &stolen_current_index; - to_process_blocks = sizes_region->blocks; - } else { - to_process = &heap_current_index; - to_process_blocks = list_of_available_blocks; - } - - // if has not already set up the heap struct, set it up - if (first) { - // set flag to not need to do this again - first = false; - - // set up stuff we can - stolen_sdram_heap->free = (block_t *) - to_process_blocks[*to_process].sdram_base_address; - - stolen_sdram_heap->free->next = (block_t *) ( - to_process_blocks[*to_process].sdram_base_address + - to_process_blocks[*to_process].size - sizeof(block_t)); - - stolen_sdram_heap->free->free = NULL; - stolen_sdram_heap->first = stolen_sdram_heap->free; - - // previous block in chain - previous = stolen_sdram_heap->free->next; - previous_free = stolen_sdram_heap->free; - } else { - // set up block in block - block_t *free = (block_t *) - to_process_blocks[*to_process].sdram_base_address; - free->free = NULL; - - // update next block - free->next = (block_t *) ( - to_process_blocks[*to_process].sdram_base_address + - to_process_blocks[*to_process].size - sizeof(block_t)); - free->next->free = NULL; - free->next->next = NULL; - - // update previous links - previous->next = free; - previous->free = free; - previous_free->free = free; - - // update previous pointers - previous = free->next; - previous_free = free; - } - // update pointers - (*to_process)++; - } - - // update last - stolen_sdram_heap->last = previous; - stolen_sdram_heap->last->free = NULL; - stolen_sdram_heap->last->next = NULL; -} - -//! prints out the fake heap as if the spin1 alloc was operating over it -static inline void print_free_sizes_in_heap(void) { - block_t *free_blk = stolen_sdram_heap->free; - uint total_size = 0; - uint index = 0; - - // traverse blocks till none more available - while (free_blk) { - uint size = (uchar *) free_blk->next - (uchar *) free_blk; - log_info("free block %d has address %x and size of %d", - index, free_blk, size); - - total_size += size; - free_blk = free_blk->free; - index++; - } - - log_info("total free size is %d", total_size); -} - -//! \details sets up trackers for this core if asked. -//! \note DOES NOT REBUILD THE FAKE HEAP! -bool malloc_extras_initialise_with_fake_heap( - heap_t *heap_location) { - stolen_sdram_heap = heap_location; - - // if no real stolen SDRAM heap. point at the original SDRAM heap. - if (stolen_sdram_heap == NULL) { - stolen_sdram_heap = sv->sdram_heap; - } - - // only build tracker if not already built and its expected - if (malloc_points == NULL && safety) { - build_malloc_tracker(); - } - return true; -} - -bool malloc_extras_initialise_and_build_fake_heap( - available_sdram_blocks *sizes_region) { - // hard set stolen sdram heap to the default heap. in case no fake heap - stolen_sdram_heap = sv->sdram_heap; - - /* if planning to track all mallocs and frees to verify no - overwrites/corruption. build the initial malloc tracker*/ - if (safety) { - build_malloc_tracker(); - } - - // only build the fake heap if there's bits to build with - if (sizes_region == NULL) { - return true; - } - - // allocate blocks store for figuring out block order - uint n_mallocs = find_n_available_mallocs(sv->sdram_heap); - sdram_block *list_of_available_blocks = sark_alloc( - n_mallocs * sizeof(sdram_block), 1); - - // if fail to alloc dtcm blow up - if (list_of_available_blocks == NULL) { - return false; - } - - // alloc space for a heap object (stealing from steal store if not by - // normal means. - stolen_sdram_heap = - sark_xalloc(sv->sdram_heap, MIN_SIZE_HEAP, 0, ALLOC_LOCK); - if (stolen_sdram_heap == NULL) { - // check we can steal - if (sizes_region->n_blocks == 0) { - log_error("cant find space for the heap"); - return false; - } - - // deallocate 32 bytes from first handed down to be the heap object - stolen_sdram_heap = (heap_t *) - sizes_region->blocks[0].sdram_base_address; - sizes_region->blocks[0].sdram_base_address += MIN_SIZE_HEAP; - sizes_region->blocks[0].size -= MIN_SIZE_HEAP; - } - - // determine how much spare space there is. - stolen_sdram_heap->free_bytes = find_free_space_available(sizes_region); - - // go through true heap and allocate and add to end of list. - bool success = add_heap_to_collection(list_of_available_blocks); - if (!success) { - log_error("failed to add heap"); - return false; - } - - // build the heap struct if there is a heap structure. - make_heap_structure(sizes_region, n_mallocs, list_of_available_blocks); - - // free the allocated dtcm for the true alloc stuff. - sark_free(list_of_available_blocks); - - // printer for sanity purposes - if (to_print) { - print_free_sizes_in_heap(); - } - - return true; -} - -//! \brief builds a new heap with no stolen SDRAM and sets up the malloc -//! tracker if required. -//! \return bool where true is a successful initialisation and false otherwise. -bool malloc_extras_initialise_no_fake_heap_data(void) { - return malloc_extras_initialise_and_build_fake_heap(NULL); -} - -void malloc_extras_free_marked(void *ptr, int marker) { - // only print if its currently set to print (saves iobuf) - if (to_print) { - log_info("freeing %x", ptr); - } - - // track if the pointer has been corrupted before trying to free it. - // only possible if safety been turned on - int *int_pointer = (int *) ptr; - if (safety) { - if (!malloc_extras_check(ptr)) { - log_error("over ran whatever is being freed"); - log_error("marker is %d", marker); - malloc_extras_terminate(DETECTED_MALLOC_FAILURE); - } - - bool found = false; - int index = 0; - while (!found && index < malloc_points_size) { - if (malloc_points[index] == ptr) { - found = true; - malloc_points[index] = 0; - } else { - index++; - } - } - - // if set to print and there was a free index, print it - if (found && to_print) { - log_info("freeing index %d", index); - } - - // shift pointer if in safety - int_pointer--; - } +void malloc_extras_free(void *ptr) { // if safe to free, free from the correct heap based off position. if ((int) ptr >= DTCM_BASE && (int) ptr <= DTCM_TOP) { - sark_xfree(sark.heap, int_pointer, ALLOC_LOCK); + if (to_print) { + log_info("freeing 0x%08x from DTCM", ptr); + } + sark_xfree(sark.heap, ptr, ALLOC_LOCK); } else { - sark_xfree(stolen_sdram_heap, int_pointer, ALLOC_LOCK); - } -} - -void malloc_extras_free(void *ptr) { - malloc_extras_free_marked(ptr, -1); -} - -//! \brief doubles the size of the SDRAM malloc tracker -static inline void build_bigger_size(void) { - // make twice as big tracker - int new_malloc_points_size = malloc_points_size * 2; - - // make new tracker - void **temp_pointer = sark_xalloc( - stolen_sdram_heap, new_malloc_points_size * sizeof(void*), 0, - ALLOC_LOCK); - - // check for null - if (temp_pointer == NULL) { - log_error("failed to allocate space for next range."); - rt_error(RTE_SWERR); - } - - // init the new store - for (int index = 0; index < new_malloc_points_size; index ++) { - temp_pointer[index] = 0; - } - - // move from old to new - for (int index = 0; index < malloc_points_size; index ++) { - temp_pointer[index] = malloc_points[index]; - } - - // free old and update pointers - sark_xfree(stolen_sdram_heap, malloc_points, ALLOC_LOCK); - malloc_points = temp_pointer; - malloc_points_size = new_malloc_points_size; -} - -//! \brief locates a new spot in the malloc tracker. may force a new -//! allocation of malloc markers if full already. -//! \return the index in the current malloc tracker to put this new malloc -//! pointer. -static inline int find_free_malloc_index(void) { - int index; - for (index = 0; index < malloc_points_size; index ++) { - if (malloc_points[index] == 0) { - return index; + if (to_print) { + log_info("freeing 0x%08x from SDRAM", ptr); } + sark_xfree(sv->sdram_heap, ptr, ALLOC_LOCK); } - // full. rebuild twice as big - build_bigger_size(); - return index + 1; } -//! \brief allows a search of the SDRAM heap. -//! \param[in] bytes: the number of bytes to allocate. -//! \return the address of the block of memory to utilise. -static void *safe_sdram_malloc(uint bytes) { +void *malloc_extras_sdram_malloc(uint bytes) { + // try SDRAM stolen from the cores synaptic matrix areas. - uint32_t *p = sark_xalloc(stolen_sdram_heap, bytes, 0, ALLOC_LOCK); + void *p = sark_xalloc(sv->sdram_heap, bytes, 0, ALLOC_LOCK); if (p == NULL) { log_error("Failed to malloc %u bytes.\n", bytes); } - - return (void *) p; -} - -//! \brief adds the len and buffers to a given malloc pointer. -//! \details Stores in the malloc tracker and prints index if required. -//! \param[in] p: The allocated buffer -//! \param[in] bytes: The size of the buffer in \p p -static void add_safety_len_and_padding(int *p, uint bytes) { - // add len - int n_words = (int) ((bytes - MINUS_POINT) / BYTE_TO_WORD); - p[0] = n_words; - - // fill in buffer at end of malloc. - for (int buffer_word = 0; buffer_word < BUFFER_WORDS; buffer_word++) { - p[n_words + buffer_word] = SAFETY_FLAG; - } - - // add malloc to the malloc tracker. - int malloc_point_index = find_free_malloc_index(); - if (malloc_point_index == -1) { - log_error("cant track this malloc. failing"); - rt_error(RTE_SWERR); - } - malloc_points[malloc_point_index] = (void *) &p[1]; - - // only print if its currently set to print (saves iobuf) if (to_print) { - log_info("index %d", malloc_point_index); - log_info("address is %x", &p[1]); + log_info("Allocated %u bytes from SDRAM at 0x%08x", bytes, p); } -} - -void *malloc_extras_sdram_malloc_wrapper(uint bytes) { - // if using safety. add the extra bytes needed for buffer and len. - if (safety) { - bytes = bytes + EXTRA_BYTES; - } - - // malloc from SDRAM heap. - int * p = safe_sdram_malloc(bytes); - - // if safety, add the len and buffers and return location for app code. - if (safety) { - add_safety_len_and_padding(p, bytes); - // return the point were user code can use from. - return (void *) &p[1]; - } - - // if no safety, the point is the point used by the application code. - return (void *) p; + return p; } void *malloc_extras_malloc(uint bytes) { - if (safety) { - bytes = bytes + EXTRA_BYTES; - } // try DTCM if allowed (not safe if overused, due to stack overflows) - int *p = NULL; + void *p = NULL; if (use_dtcm) { p = sark_alloc(bytes, 1); // if DTCM failed to malloc, go to SDRAM. if (p == NULL) { - if (to_print) { - log_info("went to SDRAM"); - } - p = safe_sdram_malloc(bytes); + p = malloc_extras_sdram_malloc(bytes); + } else if (to_print) { + log_info("Allocated %u bytes from DTCM at 0x%08x", bytes, p); } // only use SDRAM. (safer to avoid stack overflows) } else { - if (to_print) { - log_info("went to SDRAM without checking DTCM. as requested"); - } - p = safe_sdram_malloc(bytes); - } - - // if safety, add the len and buffers and return location for app code. - if (safety) { - add_safety_len_and_padding(p, bytes); - - // return the point were user code can use from. - return (void *) &p[1]; + p = malloc_extras_sdram_malloc(bytes); } // if no safety, the point is the point used by the application code. - return (void *) p; + return p; } diff --git a/c_common/models/compressors/Makefile b/c_common/models/compressors/Makefile index da58e58fc4..32f505d166 100644 --- a/c_common/models/compressors/Makefile +++ b/c_common/models/compressors/Makefile @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -APPS = sorter.mk \ - bit_field_ordered_covering_compressor.mk \ - bit_field_pair_compressor.mk \ - simple_pair_compressor.mk \ +APPS = simple_pair_compressor.mk \ simple_unordered_compressor.mk all: $(APPS) diff --git a/c_common/models/compressors/bit_field_ordered_covering_compressor.mk b/c_common/models/compressors/bit_field_ordered_covering_compressor.mk deleted file mode 100644 index 6146540292..0000000000 --- a/c_common/models/compressors/bit_field_ordered_covering_compressor.mk +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2019 The University of Manchester -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -APP = bit_field_ordered_covering_compressor - -SOURCES = bit_field_compressor.c - -FEC_OPT = $(OSPACE) - -include ../fec_models.mk diff --git a/c_common/models/compressors/bit_field_pair_compressor.mk b/c_common/models/compressors/bit_field_pair_compressor.mk deleted file mode 100644 index 82524a8452..0000000000 --- a/c_common/models/compressors/bit_field_pair_compressor.mk +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2019 The University of Manchester -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -APP = bit_field_pair_compressor - -SOURCES = bit_field_compressor.c - -FEC_OPT = $(OSPACE) - -include ../fec_models.mk - -CFLAGS += -DUSE_PAIR \ No newline at end of file diff --git a/c_common/models/compressors/sorter.mk b/c_common/models/compressors/sorter.mk deleted file mode 100644 index 6e3fb972de..0000000000 --- a/c_common/models/compressors/sorter.mk +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2019 The University of Manchester -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -APP = bit_field_sorter_and_searcher - -SOURCES = sorter/bit_field_sorter_and_searcher.c - -CFLAGS += -DSPINNAKER -Wshadow -O0 -FEC_OPT = $(OSPACE) -#FEC_OPT = -O0 - -include ../fec_models.mk - - diff --git a/c_common/models/compressors/src/bit_field_common/bit_field_table_generator.h b/c_common/models/compressors/src/bit_field_common/bit_field_table_generator.h deleted file mode 100644 index 8d0ee36fa2..0000000000 --- a/c_common/models/compressors/src/bit_field_common/bit_field_table_generator.h +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \file -//! \brief The table generator support code -#ifndef __BIT_FIELD_TABLE_GENERATOR_H__ -#define __BIT_FIELD_TABLE_GENERATOR_H__ - -#include "../common/constants.h" -#include "routing_tables.h" -#include -#include - -//! max number of links on a router -#define MAX_LINKS_PER_ROUTER 6 - -//! neuron level mask -#define NEURON_LEVEL_MASK 0xFFFFFFFF - -//! \brief Count the number of unique keys in the list up to the midpoint. -//! \details Works on the assumption that the list is grouped (sorted) by key -//! \param[in] sorted_bit_fields: the pointer to the sorted bit field struct. -//! \param[in] midpoint: where in the sorted bitfields to go to -//! \return the number of unique keys -int count_unique_keys( - sorted_bit_fields_t *restrict sorted_bit_fields, int midpoint) { - // semantic sugar - filter_info_t **restrict bit_fields = sorted_bit_fields->bit_fields; - int *restrict sort_order = sorted_bit_fields->sort_order; - int n_bit_fields = sorted_bit_fields->n_bit_fields; - - // as the sorted bitfields are sorted by key. checking key changes when - // within the midpoint will work. - int count = 0; - uint32_t last_key = -1; - for (int i = 0; i < n_bit_fields; i++) { - if ((sort_order[i] < midpoint) && (last_key != bit_fields[i]->key)) { - count++; - last_key = bit_fields[i]->key; - } - } - return count; -} - -//! \brief Generate a routing tables by merging an entry and a list of -//! bitfields by processor. -//! \param[in] original_entry: The Routing Table entry in the original table -//! \param[in] filters: List of the bitfields to me merged in -//! \param[in] bit_field_processors: List of the processors for each bitfield -//! \param[in] bf_found: Number of bitfields found. -//! \param[in/out] core_atom: The core-atom to start from, updated with where -//! we got to -//! \return Whether more calls are needed for the bit field in question -bool generate_table( - entry_t original_entry, filter_info_t **restrict filters, - uint32_t *restrict bit_field_processors, int bf_found, - struct core_atom *core_atom) { - - // Remove the processor bits from the route that match the bitfields - uint32_t stripped_route = original_entry.route; - for (int i = 0; i < bf_found; i++) { - bit_field_clear(&stripped_route, - bit_field_processors[i] + MAX_LINKS_PER_ROUTER); - } - - // Go through the atoms, potentially starting where we left off - uint32_t first_atom = global_atom(filters[0], core_atom); - uint32_t n_atoms = filters[0]->n_atoms; - for (uint32_t atom = first_atom; atom < n_atoms; atom++) { - - // Stop when the route no longer matches the key from the bit field, - // as this will resume with another entry later - uint32_t atom_key = get_bf_key(filters[0], core_atom); - if ((atom_key & original_entry.key_mask.mask) - != original_entry.key_mask.key) { - // We need to continue this later, so say yes! - return true; - } - - // Start with a copy of the stripped route - uint32_t new_route = stripped_route; - - // Add the processor for each bit field where the bit for the atom is set - for (int bf_index = 0; bf_index < bf_found; bf_index++) { - log_debug("data address is %x", filters[bf_index]->data); - if (bit_field_test(filters[bf_index]->data, atom)) { - log_debug( - "setting for atom %d from bitfield index %d so proc %d", - atom, bf_index, bit_field_processors[bf_index]); - bit_field_set(&new_route, - MAX_LINKS_PER_ROUTER + bit_field_processors[bf_index]); - } - } - - // Add a new entry based on the bit fields - routing_tables_append_new_entry( - original_entry.key_mask.key + (atom - first_atom), - NEURON_LEVEL_MASK, new_route, original_entry.source); - - // Get the next core atom for the next round - next_core_atom(filters[0], core_atom); - } - - log_debug("key %d atoms %d size %d", - original_entry.key_mask.key, n_atoms, - routing_table_get_n_entries()); - // We got through all atoms, so say no! - return false; -} - -//! \brief Take a midpoint and read the sorted bitfields, -//! computing the max size of the routing table. -//! \param[in] mid_point: where in the sorted bitfields to go to -//! \param[in] uncompressed_table: the uncompressed router table -//! \param[in] sorted_bit_fields: the pointer to the sorted bit field struct. -//! \return size of table(s) to be generated in entries -static inline uint32_t bit_field_table_generator_max_size( - int mid_point, table_t *restrict uncompressed_table, - sorted_bit_fields_t *restrict sorted_bit_fields) { - // semantic sugar to avoid referencing - filter_info_t **restrict bit_fields = sorted_bit_fields->bit_fields; - int *restrict sort_order = sorted_bit_fields->sort_order; - - // Start with the size of the uncompressed table - uint32_t max_size = uncompressed_table->size; - log_debug("keys %d", max_size); - - // Check every bitfield to see if is to be used - // Only need each key once to track last used as tables is sorted by key - uint32_t last_key = 0xFFFFFFFF; - bool is_last_key = false; - for (int bf_i = 0; bf_i < sorted_bit_fields->n_bit_fields; bf_i++) { - if (sort_order[bf_i] < mid_point) { - if (!is_last_key || last_key != bit_fields[bf_i]->key) { - last_key = bit_fields[bf_i]->key; - is_last_key = true; - - // One entry per atom but we can remove the uncompressed one - max_size += bit_fields[bf_i]->n_atoms - 1; - log_debug("key %d size %d", - last_key, bit_fields[bf_i]->n_atoms); - } - } - } - log_debug("Using mid_point %d, counted size of table is %d", - mid_point, max_size); - return max_size; -} - -//! \brief Take a midpoint and read the sorted bitfields up to that point, -//! generating bitfield routing tables and loading them into SDRAM -//! \param[in] mid_point: where in the sorted bitfields to go to -//! \param[in] uncompressed_table: the uncompressed router table -//! \param[in] sorted_bit_fields: the pointer to the sorted bit field struct. -static inline void bit_field_table_generator_create_bit_field_router_tables( - int mid_point, - table_t *restrict uncompressed_table, - sorted_bit_fields_t *restrict sorted_bit_fields) { - // semantic sugar to avoid referencing - filter_info_t **restrict bit_fields = sorted_bit_fields->bit_fields; - int *restrict processor_ids = sorted_bit_fields->processor_ids; - int *restrict sort_order = sorted_bit_fields->sort_order; - entry_t *restrict original = uncompressed_table->entries; - uint32_t original_size = uncompressed_table->size; - uint32_t n_bit_fields = sorted_bit_fields->n_bit_fields; - - filter_info_t * filters[MAX_PROCESSORS]; - uint32_t bit_field_processors[MAX_PROCESSORS]; - log_debug("pre size %d", routing_table_get_n_entries()); - - // Go through key-sorted bit fields and routing entries in tandem. - // Note: there may be multiple routing entries for each bit field, - // but there must be only one bit field per processor per routing entry! - uint32_t rt_i = 0; - uint32_t bf_i = 0; - while (bf_i < n_bit_fields && rt_i < original_size) { - // Find a routing entry that starts at the current bit field (there - // must be one, because combined entries must be from the same source - // at this point). - while (rt_i < original_size && - original[rt_i].key_mask.key != bit_fields[bf_i]->key) { - routing_tables_append_entry(original[rt_i++]); - } - - // Get out while you still can! - if (rt_i >= original_size) { - break; - } - - // Now find all bit fields with the same key, which will have the same - // remaining properties too (like atoms per core etc.) since they will - // be from the same source. - uint32_t key = original[rt_i].key_mask.key; - uint32_t bf_found = 0; - while (bf_i < n_bit_fields && bit_fields[bf_i]->key == key) { - if (sort_order[bf_i] < mid_point) { - filters[bf_found] = bit_fields[bf_i]; - bit_field_processors[bf_found] = processor_ids[bf_i]; - bf_found++; - } - bf_i++; - } - - // If we found any bit fields that now match, create entries for each - // routing entry that continues to match the keys - if (bf_found > 0) { - // While the bit field is not finished from this entry, keep - // generating more - struct core_atom core_atom = {0, 0}; - while (rt_i < original_size - && generate_table(original[rt_i], filters, - bit_field_processors, bf_found, &core_atom)) { - rt_i++; - } - // The last one will return false, so increment one more time - rt_i++; - } - } - - // At this point, we might still not have finished the routing table; - // all remaining entries must be outside of the bit fields, so just copy - // them. - while (rt_i < original_size) { - routing_tables_append_entry(original[rt_i++]); - } -} - -//! \brief debugging print for a pointer to a table. -//! \param[in] table: the table pointer to print -void print_table(table_t *table) { - entry_t *entries = table->entries; - for (uint32_t i = 0; i < table->size; i++) { - log_debug("i %u, key %u, mask %u, route %u, source %u", - i, entries[i].key_mask.key, entries[i].key_mask.mask, - entries[i].route, entries[i].source); - } -} - -//! \brief How to compare two entries -//! \param[in] ent_1: The first entry -//! \param[in] ent_2: The second entry -//! \return Whether the first entry is greater than the second -static inline bool compare_entries(const entry_t *ent_1, const entry_t *ent_2) { - return ent_1->key_mask.key > ent_2->key_mask.key; -} - -//! \brief Sort a given table so that the entries in the table are by key -//! value. -//! \details Uses insertion sort. -//! \param[in] table: the table to sort. -void sort_table_by_key(table_t *table) { - uint32_t size = table->size; - entry_t *entries = table->entries; - - uint32_t i, j; - for (i = 1; i < size; i++) { - const entry_t temp = entries[i]; - for (j = i; j > 0 && compare_entries(&entries[j - 1], &temp); j--) { - entries[j] = entries[j - 1]; - } - entries[j] = temp; - } -} - -#endif // __BIT_FIELD_TABLE_GENERATOR_H__ diff --git a/c_common/models/compressors/src/bit_field_common/compressor_sorter_structs.h b/c_common/models/compressors/src/bit_field_common/compressor_sorter_structs.h deleted file mode 100644 index fa95951ec7..0000000000 --- a/c_common/models/compressors/src/bit_field_common/compressor_sorter_structs.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \file -//! \brief Structures and enumerations for the bitfield compressor sorter. -#ifndef __COMPRESSOR_SORTER_STRUCTS_H__ -#define __COMPRESSOR_SORTER_STRUCTS_H__ - -#include -#include -#include - -//!=========================================================================== -//! enums - -//! \brief the acceptable finish states -typedef enum compressor_states { - //! Flag to say this core has never been used or prepared - UNUSED_CORE = 30, - //! Flag to say compressor is ready to run. This clears previous results - PREPARED = 31, - //! Flag to say compressor is acticvely compressing - COMPRESSING = 32, - //! Flag to say the last compression run ended due to a malloc failure - FAILED_MALLOC = 33, - //! Flag to say sorter force seen and compression has ended or been stopped - //! Note: It use may be replaced with the PREPARED flag - FORCED_BY_COMPRESSOR_CONTROL = 34, - //! Flag to say previous run was successful - SUCCESSFUL_COMPRESSION = 35, - //! Flag to say previous run finished but without a small enough table - FAILED_TO_COMPRESS = 36, - //! Flag to say previous run was aborted as it ran out of time - RAN_OUT_OF_TIME = 37 -} compressor_states; - -//! \brief The commands sent to a compressor core -typedef enum instructions_to_compressor { - //! Flag for saying processor is not a compressor - NOT_COMPRESSOR = 40, - //! Flag for saying compression processor will not be used any more - DO_NOT_USE = 41, - //! Flag for saying compression processor needs to be prepared for the - //! first time - TO_BE_PREPARED = 42, - //! Flag to ask compressor to setup and clear any previous result - PREPARE = 43, - //! Flag to say processor shoukd run - RUN = 44, - //! Flag to say processor should stop as result no longer needed - FORCE_TO_STOP = 45 -} instructions_to_compressor; - -//!========================================================================= -//! structs - -//! \brief Holds the data to initialise routing_table.h -typedef struct multi_table_t { - //! The individual subtables - table_t** sub_tables; - //! The number of individual subtables - uint32_t n_sub_tables; - //! The number of entry_t entries actually in the tables. - // NOTE: is a int because ordered covering uses ints for len of tables - // and we did not feel safe to change that. - int n_entries; - //! The max number of entries supported by this multitable. - uint32_t max_entries; -} multi_table_t; - -//! \brief the list of cores that can be used as compressor processor -typedef struct compressor_processors_top_t { - //! The number of processor_id(s) in the list - uint32_t n_processors; - //! List of the ids of processors that can be used as compressors - uint32_t processor_id[]; -} compressor_processors_top_t; - -//! \brief uncompressed routing table region -typedef struct uncompressed_table_region_data_t { - //! the application ID - uint32_t app_id; - //! table struct - table_t uncompressed_table; -} uncompressed_table_region_data_t; - -//! \brief Holds the list of bitfield associated processor IDs. -//! \details sorted order based off best effort linked to sorted_bit_fields(), -//! but separate to avoid SDRAM rewrites -typedef struct sorted_bit_fields_t { - //! length of the arrays - int n_bit_fields; - //! list of bitfield associated processor IDs. - int* processor_ids; - //! the list of bitfields in sorted order based off best effect. - filter_info_t** bit_fields; - //! the sort order based on best contribution to reducing redundancy - int* sort_order; -} sorted_bit_fields_t; - -//! \brief SDRAM area to communicate between sorter and compressor -typedef struct comms_sdram_t { - //! The state the compressor is in - compressor_states compressor_state; - //! The last instruction passed from the sorter to the compressor - instructions_to_compressor sorter_instruction; - //! how many bit fields were used to make those tables - int mid_point; - //! Pointer to the shared version of the uncompressed routing table - table_t* uncompressed_router_table; - //! Pointer to the uncompressed tables metadata - multi_table_t *routing_tables; - //! Pointer to the whole sorted_bit_fields data - sorted_bit_fields_t *sorted_bit_fields; - //! initialise value for malloc_extras (Same for all compressors) - heap_t *fake_heap_data; -} comms_sdram_t; - -//! \brief a single mapping in the addresses area -typedef struct bitfield_proc_t { - //! The bitfield wrapper - filter_region_t *filter; - //! The core associated with the bitfield - int processor; -} bitfield_proc_t; - -//! \brief top-level structure in the addresses area -typedef struct region_addresses_t { - //! Minimum percentage of bitfields to be merge in (currently ignored) - uint32_t threshold; - //! Number of times that the sorters should set of the compressions again - uint32_t retry_count; - //! Pointer to the area malloced to hold the comms_sdram - comms_sdram_t* comms_sdram; - //! Number of processors in the list - int n_processors; - //! The data for the processors - bitfield_proc_t processors[]; -} region_addresses_t; - -#endif // __COMPRESSOR_SORTER_STRUCTS_H__ diff --git a/c_common/models/compressors/src/bit_field_common/routing_tables.h b/c_common/models/compressors/src/bit_field_common/routing_tables.h deleted file mode 100644 index a77ed23053..0000000000 --- a/c_common/models/compressors/src/bit_field_common/routing_tables.h +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \file -//! \brief Utilities for a single routing table -#ifndef __ROUTING_TABLES_H__ -#define __ROUTING_TABLES_H__ - -#include -#include -#include -#include "routing_tables_utils.h" - -//============================================================================= -// location for variables - -//! holder in DTCM for top level pointers in SDRAM, used for performance. -multi_table_t multi_table; - -//! \brief Gets a pointer to where this entry is stored -//! -//! Will not check if there is an entry with this id but will RTE if the id -//! is too large -//! \param[in] entry_id_to_find: Id of entry to find pointer to -//! \param[in] marker: int that should be different in every call so we can -//! detect where MUNDY was reading past the end of the table -//! \return pointer to the entry's location -entry_t* routing_tables_get_entry_marked(uint32_t entry_id_to_find, int marker) { - uint32_t table_id = entry_id_to_find >> TABLE_SHIFT; - if (table_id >= multi_table.n_sub_tables) { - log_error("Id %d to big for %d tables marker %d", - entry_id_to_find, multi_table.n_sub_tables, marker); - malloc_extras_terminate(RTE_SWERR); - } - uint32_t local_id = entry_id_to_find & LOCAL_ID_ADD; - if (local_id >= multi_table.sub_tables[table_id]->size) { - log_error("Id %d has local_id %d which is too big for " - "table of size %d marker %d", - entry_id_to_find, local_id, - multi_table.sub_tables[table_id]->size, marker); - malloc_extras_terminate(RTE_SWERR); - } - return &multi_table.sub_tables[table_id]->entries[local_id]; -} - -entry_t* routing_table_get_entry(uint32_t entry_id_to_find) { - return routing_tables_get_entry_marked(entry_id_to_find, -1); -} - -//! \brief Gets a pointer to where to append an entry to the routing table. -//! \return pointer to the entry's location -entry_t* routing_tables_append_get_entry(void) { - // check that we're not hitting the max entries supported by the table - if (multi_table.n_entries == (int) multi_table.max_entries) { - log_error( - "There is no more space out of %d entries in this multi-table" - "for this entry.", multi_table.max_entries); - malloc_extras_terminate(RTE_SWERR); - } - - // locate right table index - uint32_t table_id = multi_table.n_entries >> TABLE_SHIFT; - if (table_id >= multi_table.n_sub_tables) { - log_error("Id %d to big for %d tables", - multi_table.n_entries, multi_table.n_sub_tables); - malloc_extras_terminate(RTE_SWERR); - } - - // locate entry index - uint32_t local_id = multi_table.n_entries & LOCAL_ID_ADD; - if (local_id != multi_table.sub_tables[table_id]->size) { - log_error("Id %d has local_id %d which is big for %d table", - multi_table.n_entries, local_id, - multi_table.sub_tables[table_id]->size); - malloc_extras_terminate(RTE_SWERR); - } - - // update trackers. - multi_table.n_entries++; - multi_table.sub_tables[table_id]->size++; - return &multi_table.sub_tables[table_id]->entries[local_id]; -} - -//! \brief Inserts a deep copy of an entry after the last known entry in the -//! table. -//! \details will RTE if is this appended fails. -//! \param[in] original_entry: The Routing Table entry to be copied in -void routing_tables_append_entry(entry_t original_entry) { - entry_t *new_entry = routing_tables_append_get_entry(); - new_entry->key_mask.key = original_entry.key_mask.key; - new_entry->key_mask.mask = original_entry.key_mask.mask; - new_entry->source = original_entry.source; - new_entry->route = original_entry.route; -} - -//! Inserts an new entry after the last known entry in the table. -//! -//! will RTE if is this appended fails. -//! \param[in] key: The key for the new entry to be added -//! \param[in] mask: The key for the new entry to be added -//! \param[in] route: The key for the new entry to be added -//! \param[in] source: The key for the new entry to be added -void routing_tables_append_new_entry( - uint32_t key, uint32_t mask, uint32_t route, uint32_t source) { - entry_t *restrict new_entry = routing_tables_append_get_entry(); - new_entry->key_mask.key = key; - new_entry->key_mask.mask = mask; - new_entry->source = source; - new_entry->route = route; -} - -int routing_table_get_n_entries(void) { - return multi_table.n_entries; -} - -//! \brief Prepares the Routing table based on passed in pointers and counts -//! -//! NOTE: Will NOT Free the space any previous tables held -//! \param[in] table: Pointer to the metadata to init -void routing_tables_init(multi_table_t* table) { - multi_table.sub_tables = table->sub_tables; - multi_table.n_sub_tables = table->n_sub_tables; - multi_table.n_entries = table->n_entries; - multi_table.max_entries = table->max_entries; - log_debug("init with n table %d entries %d", - multi_table.n_sub_tables, multi_table.n_entries); - - for (uint32_t i = 0; i < multi_table.n_sub_tables; i++) { - log_debug("table %d size %d", i, multi_table.sub_tables[i]->size); - } -} - -//! \brief Saves the metadata to the multi_table object we are managing -//! \param[in] tables: Pointer to the metadata to save to -void routing_tables_save(multi_table_t *restrict tables) { - tables->sub_tables = multi_table.sub_tables; - tables->n_sub_tables = multi_table.n_sub_tables; - tables->n_entries = multi_table.n_entries; - tables->max_entries = multi_table.max_entries; - log_info("saved table with %d entries over %d tables", - tables->n_entries, tables->n_sub_tables); -} - -void routing_table_remove_from_size(int size_to_remove) { - if (size_to_remove > multi_table.n_entries) { - log_error("Remove %d large than n_entries %d", - size_to_remove, multi_table.n_entries); - malloc_extras_terminate(RTE_SWERR); - } - multi_table.n_entries -= size_to_remove; -} - -//! \brief Clones an original table into this format. -//! \details Will _not_ free the space any previous tables held. -//! Makes a Deep copy of the original. -//! \param[in] original: the table to be cloned -void routing_tables_clone_table(table_t *restrict original) { - for (uint32_t i = 0; i < original->size; i++) { - routing_tables_append_entry(original->entries[i]); - } -} - -//! \brief Gets a pointer to several entries -//! \param[in] start_entry: The first entry to get -//! \param[in] n_entries: The number of entries to get -//! \param[out] output: Where to put the entries read - must have enough space! -//! \return: Whether the entries are available now, or should be waited for -bool routing_table_get_entries(uint32_t start_entry, uint32_t n_entries, - entry_t *output) { - for (uint32_t i = 0; i < n_entries; i++) { - output[i] = *routing_table_get_entry(start_entry + i); - } - return true; -} - -//! \brief Waits for the last transfer from routing_table_get_entries to complete -//! \details Returns immediately if the last transfer is already done -void routing_table_wait_for_last_transfer(void) { - return; -} - -#endif // __ROUTING_TABLES_H__ diff --git a/c_common/models/compressors/src/bit_field_common/routing_tables_utils.h b/c_common/models/compressors/src/bit_field_common/routing_tables_utils.h deleted file mode 100644 index f91ece0bb1..0000000000 --- a/c_common/models/compressors/src/bit_field_common/routing_tables_utils.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \file -//! \brief Compound routing table utilities. -#ifndef __ROUTING_TABLES_UTILS_H__ -#define __ROUTING_TABLES_UTILS_H__ - -#include -#include -#include "compressor_sorter_structs.h" - -//! number of entries in each sub table -#define TABLE_SIZE 1024 - -//! \brief Shift to go from entry \p _id to table id. -//! -//! 2TABLE_SHIFT needs to be ::TABLE_SIZE -#define TABLE_SHIFT 10 - -//! \brief bitwise add to get sub table id. -//! -//! Needs to be ::TABLE_SIZE - 1; -#define LOCAL_ID_ADD 1023 - -//============================================================================= -//state for reduction in parameters being passed around - -//! \brief Does all frees for the multi_table object par ones before -//! the start point -//! \param[in] tables: pointer to the metadata to be freed -//! \param[in] start_point: where in the array to start freeing from. -static void routing_tables_utils_free( - multi_table_t *restrict tables, uint32_t start_point) { - if (tables->n_sub_tables == 0) { - // Already freed or never malloced - return; - } - for (uint32_t i = start_point; i > tables->n_sub_tables; i++) { - FREE_MARKED(tables->sub_tables[i]->entries, 70999); - FREE_MARKED(tables->sub_tables[i], 70100); - } - FREE_MARKED(tables->sub_tables, 70101); - tables->n_sub_tables = 0; - tables->n_entries = 0; -} - -//! \brief Does all frees for the multi_table object -//! \param[in] tables: pointer to the metadata to be freed -static void routing_tables_utils_free_all(multi_table_t *restrict tables) { - routing_tables_utils_free(tables, 0); -} - -//! \brief Prepares the Routing table to handle at least n_entries -//! -//! Will do all the the mallocs needed to hold at least max_entries -//! The actual size may be rounded up but this behaviour should not be counted -//! on in the future. -//! -//! Will NOT Free the space any previous tables held -//! \param[in] tables: the collection of tables to prepare -//! \param[in] max_entries: maximum number of entries table should hold -//! \return True if and only if all table(s) could be malloced -static inline bool routing_tables_utils_malloc( - multi_table_t *restrict tables, uint32_t max_entries) { - tables->n_sub_tables = ((max_entries - 1) >> TABLE_SHIFT) + 1; - tables->max_entries = max_entries; - log_debug("n table %d max entries %d", tables->n_sub_tables, max_entries); - tables->n_entries = 0; - tables->sub_tables = MALLOC_SDRAM(tables->n_sub_tables * sizeof(table_t*)); - - // check array malloced successfully - if (tables->sub_tables == NULL) { - log_error("failed to allocate memory for routing tables"); - tables->n_sub_tables = 0; - return false; - } - - // run through full tables mallocing max sizes. - int entries_covered = 0; - for (uint32_t i = 0; i < tables->n_sub_tables - 1; i++) { - tables->sub_tables[i] = MALLOC_SDRAM( - sizeof(uint32_t) + (sizeof(entry_t) * TABLE_SIZE)); - if (tables->sub_tables[i] == NULL) { - log_error("failed to allocate memory for routing tables"); - tables->n_sub_tables = i; - routing_tables_utils_free_all(tables); - return false; - } - tables->sub_tables[i]->size = 0; - entries_covered += TABLE_SIZE; - log_debug("created table %d size %d", i, tables->sub_tables[i]->size); - } - - // create last table with correct size - int last_table_size = tables->max_entries - entries_covered; - tables->sub_tables[tables->n_sub_tables - 1] = MALLOC_SDRAM( - sizeof(uint32_t) + (sizeof(entry_t) * last_table_size)); - if (tables->sub_tables[tables->n_sub_tables - 1] == NULL) { - log_error("failed to allocate memory for routing tables"); - tables->n_sub_tables = tables->n_sub_tables - 1; - routing_tables_utils_free_all(tables); - return false; - } - // init the size - tables->sub_tables[tables->n_sub_tables - 1]->size = 0; - log_debug("created table %d size %d", - tables->n_sub_tables - 1, - tables->sub_tables[tables->n_sub_tables - 1]->size); - - // debugging please keep. - log_debug("n table %d entries %d", - tables->n_sub_tables, tables->n_entries); - for (uint32_t i = 0; i < tables->n_sub_tables; i++) { - log_debug("table %d size %d", i, tables->sub_tables[i]->size); - } - return true; -} - -//! \brief Converts the multitable to a single routing table and free the rest -//! -//! will RTE if the routing table has too many entries to fit into a router -//! \param[in] tables: the multitable to convert -//! \return A pointer to a traditional router table -static inline table_t* routing_tables_utils_convert( - multi_table_t *restrict tables) { - log_debug("converting table with %d entries over %d tables", - tables->n_sub_tables, tables->n_entries); - - // if table too big for a router. RTE. - if (tables->n_entries > TABLE_SIZE) { - log_error("At %d There are too many entries to convert to a table_t", - tables->n_entries); - malloc_extras_terminate(RTE_SWERR); - } - - // Assume size of subtable not set so set it - tables->sub_tables[0]->size = tables->n_entries; - - // claim the first pointer before freeing to avoid bad memory access - table_t* first_table = tables->sub_tables[0]; - - // Free the rest - routing_tables_utils_free(tables, 1); - return first_table; -} - -#endif // __ROUTING_TABLES_UTILS_H__ diff --git a/c_common/models/compressors/src/bit_field_compressor.c b/c_common/models/compressors/src/bit_field_compressor.c deleted file mode 100644 index ddd5c14c6e..0000000000 --- a/c_common/models/compressors/src/bit_field_compressor.c +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \file -//! \brief The bitfield compressor -#include -#include -#include -#include -#include "common-typedefs.h" -#include "common/constants.h" -#include "bit_field_common/compressor_sorter_structs.h" -#include "bit_field_common/bit_field_table_generator.h" -#include "common/minimise.h" -#include "compressor_includes/compressor.h" -#include "bit_field_common/routing_tables.h" -#include "bit_field_common/bit_field_table_generator.h" - -/*****************************************************************************/ - -//! interrupt priorities -enum compressor_interrupt_priorities { - TIMER_TICK_PRIORITY = -1, //!< Timer uses FIQ! - COMPRESSION_START_PRIORITY = 3 //!< Compression start is low priority -}; - -//! \brief Number of timer iterations to ensure close to matching tracker -#define TIMER_ITERATIONS 1000 - -//! \brief Timer controls, as it seems timer in massive waits doesn't -//! necessarily engage properly. Ticks once per millisecond. -int counter = 0; -//! \brief Maximum value of ::counter, at which point the compressor should -//! shut itself down. Number of milliseconds to allow for a compressor run. -int max_counter = 0; - -//! Whether the compressor should shut down -volatile bool stop_compressing = false; - -//! Allows minimise to report if it failed due to malloc issues -bool failed_by_malloc = false; - -//! Whether to compress as much as possible -bool compress_as_much_as_possible = false; - -//! Debugging for wait_for_instructions(): old state of sorter -instructions_to_compressor previous_sorter_state = NOT_COMPRESSOR; -//! Debugging for wait_for_instructions(): old state of compressor -compressor_states previous_compressor_state = UNUSED_CORE; - -//! SDRAM are used for communication between sorter and THIS compressor -comms_sdram_t *restrict comms_sdram; - -// DEBUG stuff please leave -#ifdef DEBUG_COMPRESSOR -bool hack_malloc_failed = false; -#endif -// --------------------------------------------------------------------- - -//! \brief Handle the compression process -void start_compression_process(void) { - log_debug("in compression phase"); - - // restart timer (also puts us in running state) - spin1_resume(SYNC_NOWAIT); - -#ifdef DEBUG_COMPRESSOR - if (comms_sdram->mid_point >= 100) { - log_warning("HACK fail at 100 plus bitfeilds!"); - comms_sdram->compressor_state = FAILED_TO_COMPRESS; - return; - } -#endif - -#ifdef DEBUG_COMPRESSOR - if ((comms_sdram->mid_point > 0) || (!hack_malloc_failed)) { - log_warning("HACK malloc fail!"); - hack_malloc_failed = true; - comms_sdram->compressor_state = FAILED_MALLOC; - return; - } -#endif - - // run compression - bool success = run_compressor( - compress_as_much_as_possible, &failed_by_malloc, &stop_compressing); - - // turn off timer and set us into pause state - spin1_pause(); - - // Decode whether we succeeded or failed. - int max_length = rtr_alloc_max(); - if (success && (routing_table_get_n_entries() <= max_length)) { - log_info("Passed minimise_run() with success code: %d", success); - routing_tables_save(comms_sdram->routing_tables); - comms_sdram->compressor_state = SUCCESSFUL_COMPRESSION; - return; - } - - // Not a success, could be one of 4 failure states - log_info("Failed minimise_run() with success code: %d", success); - if (failed_by_malloc) { // malloc failed somewhere - log_debug("failed malloc response"); - comms_sdram->compressor_state = FAILED_MALLOC; - } else if (comms_sdram->sorter_instruction != RUN) { // control killed it - log_debug("force fail response"); - comms_sdram->compressor_state = FORCED_BY_COMPRESSOR_CONTROL; - log_debug("send ack"); - } else if (stop_compressing) { // ran out of time - log_debug("time fail response"); - comms_sdram->compressor_state = RAN_OUT_OF_TIME; - } else { // after finishing compression, still could not fit into table. - log_debug("failed by space response"); - comms_sdram->compressor_state = FAILED_TO_COMPRESS; - } -} - -//! \brief Initialise the abstraction layer of many routing tables as single -//! big table. -void setup_routing_tables(void) { - routing_tables_init(comms_sdram->routing_tables); - - if (comms_sdram->mid_point == 0) { - routing_tables_clone_table(comms_sdram->uncompressed_router_table); - } else { - bit_field_table_generator_create_bit_field_router_tables( - comms_sdram->mid_point, comms_sdram->uncompressed_router_table, - comms_sdram->sorted_bit_fields); - } -} - -//! \brief Run the compressor process as requested -void run_compression_process(void) { - if (comms_sdram->mid_point > 0) { - log_info("Run with %d tables and %d mid_point out of %d bitfields", - comms_sdram->routing_tables->n_sub_tables, - comms_sdram->mid_point, - comms_sdram->sorted_bit_fields->n_bit_fields); - } else { - log_info("Run with %d tables and no bitfields", - comms_sdram->routing_tables->n_sub_tables); - } - log_debug("setting up fake heap for sdram usage"); - malloc_extras_initialise_with_fake_heap(comms_sdram->fake_heap_data); - log_debug("set up fake heap for sdram usage"); - - // Set all status flags - failed_by_malloc = false; - stop_compressing = false; - counter = 0; - - setup_routing_tables(); - - log_debug("starting compression attempt with %d entries", - routing_table_get_n_entries()); - - // start compression process - start_compression_process(); -} - -//! \brief Check what to do if anything as sorter has asked to ::RUN -//! \details May do nothing if the previous run has already finished -//! \param[in] compressor_state: The current state of the compressor -//! \returns Whether the ::RUN made sense with the current compressor state -static inline bool process_run(compressor_states compressor_state) { - switch (compressor_state) { - case PREPARED: - comms_sdram->compressor_state = COMPRESSING; - run_compression_process(); - return true; - case COMPRESSING: - // Should not be back in this loop before result set - return false; - case FAILED_MALLOC: - case FORCED_BY_COMPRESSOR_CONTROL: - case SUCCESSFUL_COMPRESSION: - case FAILED_TO_COMPRESS: - case RAN_OUT_OF_TIME: - // waiting for sorter to pick up result - return true; - case UNUSED_CORE: - // Should never happen - return false; - } - return false; -} - -//! \brief Check what to do if anything as sorter has asked to ::PREPARE -//! \details Mainly used to clear result of previous run -//! \param[in] compressor_state: The current state of the compressor -//! \returns Whether the ::PREPARE made sense with the current compressor state -static inline bool process_prepare(compressor_states compressor_state) { - switch (compressor_state) { - case UNUSED_CORE: - // First prepare - log_info("Prepared for the first time"); - comms_sdram->compressor_state = PREPARED; - return true; - case FAILED_MALLOC: - case FORCED_BY_COMPRESSOR_CONTROL: - case SUCCESSFUL_COMPRESSION: - case FAILED_TO_COMPRESS: - case RAN_OUT_OF_TIME: - // clear previous result - log_info("prepared"); - comms_sdram->compressor_state = PREPARED; - return true; - case PREPARED: - // waiting for sorter to pick up result - return true; - case COMPRESSING: - // Should never happen - return false; - } - return false; -} - -//! \brief Check what to do if anything as sorter has asked to ::FORCE_TO_STOP -//! \details Mainly used to clear result of previous run -//! The wait loop that calls this does not run during compressing; -//! timer_callback() picks up the sorter change during compression -//! \param[in] compressor_state: The current state of the compressor -//! \returns Whether the ::FORCE_TO_STOP made sense with the current compressor -//! state -static inline bool process_force(compressor_states compressor_state) { - switch (compressor_state) { - case COMPRESSING: - // passed to compressor as *sorter_instruction - // Do nothing until compressor notices changed - return true; - case FORCED_BY_COMPRESSOR_CONTROL: - // Waiting for sorter to pick up - return true; - case FAILED_MALLOC: - case SUCCESSFUL_COMPRESSION: - case FAILED_TO_COMPRESS: - case RAN_OUT_OF_TIME: - log_info("Force detected so changing result to ack"); - // The results other than MALLOC no longer matters - comms_sdram->compressor_state = FORCED_BY_COMPRESSOR_CONTROL; - return true; - case PREPARED: - case UNUSED_CORE: - // Should never happen - return false; - } - return false; -} - -//! \brief Busy-wait until there is a new instruction from the sorter. -//! \details Note that this is done at very low priority so that interrupts -//! (including to deliver instructions to us to work) will breeze past. -//! \param[in] unused0: unused -//! \param[in] unused1: unused -static void wait_for_instructions(UNUSED uint unused0, UNUSED uint unused1) { - // set if combination of user2 and user3 is expected - bool users_match = true; - - // cache the states so they dont change inside one loop - compressor_states compressor_state = comms_sdram->compressor_state; - instructions_to_compressor sorter_state = comms_sdram->sorter_instruction; - // When debugging Log if changed - if (sorter_state != previous_sorter_state) { - previous_sorter_state = sorter_state; - log_debug("Sorter state changed sorter: %d compressor %d", - sorter_state, compressor_state); - } - if (compressor_state != previous_compressor_state) { - previous_compressor_state = compressor_state; - log_debug("Compressor state changed sorter: %d compressor %d", - sorter_state, compressor_state); - } - - switch (sorter_state) { - case PREPARE: - users_match = process_prepare(compressor_state); - break; - case RUN: - users_match = process_run(compressor_state); - break; - case FORCE_TO_STOP: - users_match = process_force(compressor_state); - break; - case NOT_COMPRESSOR: - // For some reason compressor sees this state too - case TO_BE_PREPARED: - users_match = (compressor_state == UNUSED_CORE); - break; - case DO_NOT_USE: - log_warning("DO_NOT_USE detected exiting wait"); - spin1_pause(); - return; - } - if (users_match) { - spin1_schedule_callback( - wait_for_instructions, 0, 0, COMPRESSION_START_PRIORITY); - } else { - log_error("Unexpected combination of sorter_state %d and " - "compressor_state %d", sorter_state, compressor_state); - malloc_extras_terminate(RTE_SWERR); - } -} - -//! \brief Timer interrupt for controlling stopping compression. -//! \details Could be due to time taken to try to compress table. -//! Could be because sorter has cancelled run request. -//! \param[in] unused0: not used -//! \param[in] unused1: not used -static void timer_callback(UNUSED uint unused0, UNUSED uint unused1) { - counter++; - - if (counter >= max_counter) { - stop_compressing = true; - log_info("passed timer point"); - spin1_pause(); - } - - // check that the sorter has told the compressor to finish for any reason - if (comms_sdram->sorter_instruction != RUN) { - stop_compressing = true; - if (comms_sdram->compressor_state == COMPRESSING) { - log_info("Sorter cancelled run request"); - } else if (comms_sdram->sorter_instruction == DO_NOT_USE) { - log_info("Compressor no longer to be used"); - } else { - log_warning("timer weirdness %d %d", - comms_sdram->sorter_instruction, - comms_sdram->compressor_state); - } - spin1_pause(); - } -} - -//! \brief Set up the callback for setting off the router compressor. -static void initialise(void) { - log_info("Setting up stuff to allow bitfield compressor to occur."); - - log_debug("reading time_for_compression_attempt"); - vcpu_t *restrict sark_virtual_processor_info = (vcpu_t *) SV_VCPU; - vcpu_t *restrict this_vcpu_info = - &sark_virtual_processor_info[spin1_get_core_id()]; - - uint32_t time_for_compression_attempt = this_vcpu_info->user1; - log_info("time_for_compression_attempt = %d", - time_for_compression_attempt); - - // 0 = compress_only_when_needed, 1 = compress_as_much_as_possible - uint32_t int_value = this_vcpu_info->user2; - if (int_value & 1) { - compress_as_much_as_possible = true; - } - log_info("int %d, compress_as_much_as_possible = %d", - int_value, compress_as_much_as_possible); - - // Get the pointer for all cores - comms_sdram = (comms_sdram_t *) this_vcpu_info->user3; - - // Now move the pointer to the comms for this core - comms_sdram += spin1_get_core_id(); - - // sort out timer (this is shrank to be called 1000 times, so that we can - // check for sorter controls. e.g. is the sorter forces the compressor - // to stop early). - max_counter = time_for_compression_attempt / TIMER_ITERATIONS; - spin1_set_timer_tick(TIMER_ITERATIONS); - spin1_callback_on(TIMER_TICK, timer_callback, TIMER_TICK_PRIORITY); - log_info("my processor id is %d", spin1_get_core_id()); -} - -//! \brief says this is NOT a standalone compressor. -//! \return Always false -bool standalone(void) { - return false; -} - - -//! \brief the main entrance. -void c_main(void) { - log_debug("%u bytes of free DTCM", sark_heap_max(sark.heap, 0)); - - // set up params - initialise(); - - // kick-start the process - spin1_schedule_callback( - wait_for_instructions, 0, 0, COMPRESSION_START_PRIORITY); - - // go - log_info("waiting for synchronisation %d %d", - comms_sdram->sorter_instruction, comms_sdram->compressor_state); - spin1_start(SYNC_WAIT); -} diff --git a/c_common/models/compressors/src/compressor_includes/pair_minimize.h b/c_common/models/compressors/src/compressor_includes/pair_minimize.h index 625d220838..63f6231e47 100644 --- a/c_common/models/compressors/src/compressor_includes/pair_minimize.h +++ b/c_common/models/compressors/src/compressor_includes/pair_minimize.h @@ -353,9 +353,9 @@ bool minimise_run(int target_length, bool *failed_by_malloc, } } - log_debug("before sort %u", routes_count); + log_info("before sort %u", routes_count); for (uint i = 0; i < routes_count; i++) { - log_debug("%u", routes[i]); + log_info("%u", routes[i]); } sort_routes(); @@ -364,12 +364,12 @@ bool minimise_run(int target_length, bool *failed_by_malloc, return false; } - log_debug("after sort %u", routes_count); + log_info("after sort %u", routes_count); for (uint i = 0; i < routes_count; i++) { - log_debug("%u", routes[i]); + log_info("%u", routes[i]); } - log_debug("do sort_table by route %u", table_size); + log_info("do sort_table by route %u", table_size); tc[T2_LOAD] = 0xFFFFFFFF; tc[T2_CONTROL] = 0x83; sort_table(); @@ -388,14 +388,14 @@ bool minimise_run(int target_length, bool *failed_by_malloc, while (left <= max_index) { int right = left; uint32_t left_route = routing_table_get_entry(left)->route; - log_debug("A %u %u %u %u", left, max_index, right, left_route); + log_info("A %u %u %u %u", left, max_index, right, left_route); while ((right < table_size - 1) && routing_table_get_entry(right+1)->route == left_route) { right++; } remaining_index = right + 1; - log_debug("compress %u %u", left, right); + log_info("compress %u %u", left, right); tc[T2_LOAD] = 0xFFFFFFFF; tc[T2_CONTROL] = 0x83; compress_by_route(left, right); @@ -417,7 +417,6 @@ bool minimise_run(int target_length, bool *failed_by_malloc, left = right + 1; } - log_debug("done %u %u", table_size, write_index); //for (uint i = 0; i < write_index; i++) { // entry_t *entry1 = routing_table_get_entry(i); diff --git a/c_common/models/compressors/src/simple/rt_single.h b/c_common/models/compressors/src/simple/rt_single.h index 96a763cf4a..57c27cca1b 100644 --- a/c_common/models/compressors/src/simple/rt_single.h +++ b/c_common/models/compressors/src/simple/rt_single.h @@ -168,20 +168,6 @@ bool load_routing_table(uint32_t app_id) { return TRUE; } -//! \brief Free memory allocated and call spin1_exit() and sets the user0 -//! error code correctly. -//! \param[in] header: the header object -void cleanup_and_exit(header_t *header) { - // Free the memory used by the routing table. - log_debug("free sdram blocks which held router tables"); - FREE(table->entries); - // Free the block of SDRAM used to load the routing table. - sark_xfree(sv->sdram_heap, (void *) header, ALLOC_LOCK); - - log_info("completed router compressor"); - malloc_extras_terminate(EXITED_CLEANLY); -} - //! \brief Gets a pointer to several entries //! \param[in] start_entry: The first entry to get //! \param[in] n_entries: The number of entries to get diff --git a/c_common/models/compressors/src/simple/simple_compressor.c b/c_common/models/compressors/src/simple/simple_compressor.c index 3d12c9ef74..a1950e0ee7 100644 --- a/c_common/models/compressors/src/simple/simple_compressor.c +++ b/c_common/models/compressors/src/simple/simple_compressor.c @@ -74,7 +74,8 @@ void compress_start(UNUSED uint unused0, UNUSED uint unused1) { // Try to load the routing table log_debug("try loading tables"); if (load_routing_table(header->app_id)) { - cleanup_and_exit(header); + log_info("completed router compressor"); + malloc_extras_terminate(EXITED_CLEANLY); } else { // Otherwise give up and exit with an error log_error("Failed to minimise routing table to fit %u entries. " @@ -84,7 +85,6 @@ void compress_start(UNUSED uint unused0, UNUSED uint unused1) { // Free the block of SDRAM used to load the routing table. log_debug("free sdram blocks which held router tables"); - FREE((void *) header); // set the failed flag and exit malloc_extras_terminate(EXIT_FAIL); @@ -100,8 +100,7 @@ bool standalone(void) { //! \brief the main entrance. void c_main(void) { log_debug("%u bytes of free DTCM", sark_heap_max(sark.heap, 0)); - malloc_extras_turn_off_safety(); - malloc_extras_initialise_no_fake_heap_data(); + malloc_extras_turn_on_print(); // kick-start the process spin1_schedule_callback(compress_start, 0, 0, 3); diff --git a/c_common/models/compressors/src/sorter/bit_field_reader.h b/c_common/models/compressors/src/sorter/bit_field_reader.h deleted file mode 100644 index d472c1ccf5..0000000000 --- a/c_common/models/compressors/src/sorter/bit_field_reader.h +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \dir -//! \brief Support files for the bitfield sorter -//! \file -//! \brief Code for reading bitfields in SDRAM -#ifndef __BIT_FIELD_READER_H__ -#define __BIT_FIELD_READER_H__ - -#include -#include -#include "common/constants.h" -#include "bit_field_common/compressor_sorter_structs.h" - -//! For each possible processor the first index of a row for that processor -static int processor_heads[MAX_PROCESSORS]; - -//! Sum of packets per processor for bitfields with redundancy not yet ordered -static uint32_t processor_totals[MAX_PROCESSORS]; - -//! \brief Detemine how many bits are not set in a bit field -//! \param[in] filter: The bitfield to look for redundancy in -//! \return How many redundant packets there are -static uint32_t n_redundant(filter_info_t *restrict filter) { - uint32_t n_atoms = filter->n_atoms; - uint32_t n_words = get_bit_field_size(n_atoms); - return n_atoms - count_bit_field(filter->data, n_words); -} - -//! \brief Fill in the order column based on packet reduction -//! \param[in] sorted_bit_fields: Data to be ordered -static inline void order_bitfields( - sorted_bit_fields_t *restrict sorted_bit_fields) { - // Semantic sugar to avoid extra lookup all the time - int *restrict processor_ids = sorted_bit_fields->processor_ids; - int *restrict sort_order = sorted_bit_fields->sort_order; - filter_info_t **restrict bit_fields = sorted_bit_fields->bit_fields; - - // To label each row in sort order - for (int sorted_index = 0; sorted_index < sorted_bit_fields->n_bit_fields; - sorted_index++) { - - // Find the processor with highest number of packets coming in - int worst_processor = 0; - uint32_t highest_neurons = 0; - for (int c = 0; c < MAX_PROCESSORS; c++) { - if (processor_totals[c] > highest_neurons){ - worst_processor = c; - highest_neurons = processor_totals[c]; - } - } - - // Label the row pointer to be the header as next - int index = processor_heads[worst_processor]; - sort_order[index] = sorted_index; - log_debug("processor %u index %u total %u", - worst_processor, index, processor_totals[worst_processor]); - - // If there is another row with the same processor - if ((index < sorted_bit_fields->n_bit_fields - 1) && - (processor_ids[index] == processor_ids[index + 1])) { - log_debug("i %u processor %u index %u more %u total %u", - sorted_index, worst_processor, index, - sorted_bit_fields->n_bit_fields, - processor_totals[worst_processor]); - - // reduce the packet count bu redundancy - processor_totals[worst_processor] -= n_redundant(bit_fields[index]); - - // move the pointer - processor_heads[worst_processor] += 1; - } else { - // otherwise set the counters to ignore this processor - processor_totals[worst_processor] = NO_BIT_FIELDS; - processor_heads[worst_processor] = DO_NOT_USE; - - log_debug("i %u processor %u index %u last %u total %u", - sorted_index, worst_processor, index, - sorted_bit_fields->n_bit_fields, - processor_totals[worst_processor]); - } - } -} - -//! \brief Sort the data based on the bitfield key -//! \param[in] sorted_bit_fields: Data to be ordered -static inline void sort_by_key( - sorted_bit_fields_t *restrict sorted_bit_fields) { - // Semantic sugar to avoid extra lookup all the time - int *restrict processor_ids = sorted_bit_fields->processor_ids; - int *restrict sort_order = sorted_bit_fields->sort_order; - filter_info_t **restrict bit_fields = sorted_bit_fields->bit_fields; - int i, j; - - for (i = 1; i < sorted_bit_fields->n_bit_fields; i++) { - const int temp_processor_id = processor_ids[i]; - const uint32_t temp_sort_order = sort_order[i]; - filter_info_t *const bit_field_temp = bit_fields[i]; - register uint32_t key = bit_field_temp->key; - - for (j = i; j > 0 && bit_fields[j - 1]->key > key; j--) { - processor_ids[j] = processor_ids[j - 1]; - sort_order[j] = sort_order[j - 1]; - bit_fields[j] = bit_fields[j - 1]; - } - - processor_ids[j] = temp_processor_id; - sort_order[j] = temp_sort_order; - bit_fields[j] = bit_field_temp; - } -} - -//! \brief Debugging support for bit_field_reader_read_in_bit_fields(); -//! prints sorted bitfields and tests memory allocation -//! \param[in] sorted_bit_fields: The sorted bitfields -static inline void print_structs( - sorted_bit_fields_t *restrict sorted_bit_fields) { - // useful for debugging - for (int i = 0; i < sorted_bit_fields->n_bit_fields; i++) { - log_debug("index %u processor: %u, key: %u, data %u redundant %u order %u", - i, sorted_bit_fields->processor_ids[i], - sorted_bit_fields->bit_fields[i]->key, - sorted_bit_fields->bit_fields[i]->data, - n_redundant(sorted_bit_fields->bit_fields[i]), - sorted_bit_fields->sort_order[i]); - } -} - -//! \brief Sort a subset of the bit fields by the redundancy -//! \param[in,out] sorted_bit_fields: -//! The bit fields to sort. -//! The bit field order is actually changed by this function. -//! \param[in] start: The index of the first bit field to sort -//! \param[in] end: The index after the last bit field to sort -static inline void sort_by_redundancy(sorted_bit_fields_t *sorted_bit_fields, - uint32_t start, uint32_t end) { - // We only need to sort the bit fields, as this assumes it is called - // before the index is filled in, and where start and n_items covers items - // with the same processor id - filter_info_t **bit_fields = sorted_bit_fields->bit_fields; - for (uint32_t i = start + 1; i < end; i++) { - filter_info_t *temp_bf = bit_fields[i]; - - uint32_t j; - for (j = i; j > start && n_redundant(bit_fields[j - 1]) < n_redundant(temp_bf); j--) { - bit_fields[j] = bit_fields[j - 1]; - } - bit_fields[j] = temp_bf; - } -} - -//! \brief Fill in the sorted bit-field struct and builds tracker of -//! incoming packet counts. -//! \param[in] region_addresses: The addresses of the regions to read -//! \param[in] sorted_bit_fields: The sorted bitfield struct with bitfields -//! in sorted order. -static inline void fills_in_sorted_bit_fields_and_tracker( - region_addresses_t *restrict region_addresses, - sorted_bit_fields_t *restrict sorted_bit_fields) { - // iterate through a processors bitfield region and add to the bf by - // processor struct, whilst updating num of total param. - for (int r_id = 0, index = 0; r_id < region_addresses->n_processors; r_id++) { - // locate data for malloc memory calcs - filter_region_t *restrict filter_region = - region_addresses->processors[r_id].filter; - int processor = region_addresses->processors[r_id].processor; - - // store the index in bitfields list where this processors bitfields - // start being read in at. (not sorted) - processor_heads[processor] = index; - - // read in the processors bitfields. - filter_info_t *filters = filter_region->filters; - for (uint32_t bf_id = 0; bf_id < filter_region->n_filters; bf_id++) { - // update trackers. - if (!filters[bf_id].all_ones) { - sorted_bit_fields->processor_ids[index] = processor; - sorted_bit_fields->bit_fields[index] = &filters[bf_id]; - index++; - } - - // also accum the incoming packets from bitfields which have no - // redundancy - processor_totals[processor] += filters[bf_id].n_atoms; - } - sort_by_redundancy(sorted_bit_fields, processor_heads[processor], index); - } -} - - -//! \brief Read in bitfields -//! \param[in] region_addresses: The addresses of the regions to read -//! \param[in] sorted_bit_fields: The sorted bitfield structure to which -//! bitfields in sorted order will be populated. -static inline void bit_field_reader_read_in_bit_fields( - region_addresses_t *restrict region_addresses, - sorted_bit_fields_t *restrict sorted_bit_fields) { - //init data tracking structures - for (int i = 0; i < MAX_PROCESSORS; i++) { - processor_heads[i] = DO_NOT_USE; - processor_totals[i] = 0; - } - - // track positions and incoming packet counts. - fills_in_sorted_bit_fields_and_tracker(region_addresses, sorted_bit_fields); - - //TODO safety code to be removed. - for (int i = 0; i < MAX_PROCESSORS; i++) { - log_debug("i: %d, head: %d count: %d", - i, processor_heads[i], processor_totals[i]); - } -#if 0 - print_structs(sorted_bit_fields); -#endif - - // sort bitfields so that bit-fields are in order of most impact on worse - // affected cores. so that merged bitfields should reduce packet rates - // fairly across cores on the chip. - order_bitfields(sorted_bit_fields); - - //TODO safety code to be removed. -#if 0 - print_structs(sorted_bit_fields); -#endif - - // sort the bitfields by key. - // NOTE: does not affect previous sort, as this directly affects the index - // in the bitfield array, whereas the previous sort affects the sort index - // array. - sort_by_key(sorted_bit_fields); - - // useful for debugging -#if 0 - print_structs(sorted_bit_fields); -#endif -} - -//! \brief Sets up the initial sorted bitfield struct -//! \param[in] region_addresses: -//! The address that holds all the chips' bitfield addresses -//! \return The pointer to the sorted memory tracker, or `NULL` if any of the -//! #MALLOC's failed for any reason. -static inline sorted_bit_fields_t * bit_field_reader_initialise( - region_addresses_t *restrict region_addresses) { - sorted_bit_fields_t *restrict sorted_bit_fields = MALLOC_SDRAM( - sizeof(sorted_bit_fields_t)); - if (sorted_bit_fields == NULL) { - log_error("failed to allocate DTCM for sorted bitfields."); - return NULL; - } - - // figure out how many bitfields we need - log_debug("n_processors of addresses = %d", region_addresses->n_processors); - int n_bit_fields = 0; - for (int r_id = 0; r_id < region_addresses->n_processors; r_id++) { - filter_region_t *restrict filter = region_addresses->processors[r_id].filter; - uint32_t n_filters = filter->n_filters; - filter_info_t *filters = filter->filters; - uint32_t n_usable = 0; - for (uint32_t f_id = 0; f_id < n_filters; f_id++) { - n_usable += !filters[f_id].all_ones; - } - n_bit_fields += n_usable; - log_debug("Core %d has %u bitfields of which %u have redundancy", - region_addresses->processors[r_id].processor, - filter->n_filters, n_usable); - } - sorted_bit_fields->n_bit_fields = n_bit_fields; - log_info("Number of bitfields with redundancy found is %u", - sorted_bit_fields->n_bit_fields); - - // if there are no bit-fields just return sorted bitfields. - if (n_bit_fields != 0) { - // malloc the separate bits of the sorted bitfield struct - sorted_bit_fields->bit_fields = MALLOC_SDRAM( - n_bit_fields * sizeof(filter_info_t*)); - if (sorted_bit_fields->bit_fields == NULL) { - log_error( - "cannot allocate memory for the sorted bitfield addresses"); - FREE(sorted_bit_fields); - return NULL; - } - - sorted_bit_fields->processor_ids = MALLOC_SDRAM( - n_bit_fields * sizeof(int)); - if (sorted_bit_fields->processor_ids == NULL) { - log_error("cannot allocate memory for the sorted bitfields with " - "processors ids"); - FREE(sorted_bit_fields->bit_fields); - FREE(sorted_bit_fields); - return NULL; - } - - sorted_bit_fields->sort_order = MALLOC_SDRAM( - n_bit_fields * sizeof(int)); - if (sorted_bit_fields->sort_order == NULL) { - log_error("cannot allocate memory for the sorted bitfields with " - "sort_order"); - FREE(sorted_bit_fields->bit_fields); - FREE(sorted_bit_fields->processor_ids); - FREE(sorted_bit_fields); - return NULL; - } - - // init to -1, else random data (used to make prints cleaner) - for (int sorted_index = 0; sorted_index < n_bit_fields; - sorted_index++) { - sorted_bit_fields->sort_order[sorted_index] = FAILED_TO_FIND; - } - } - - // return - return sorted_bit_fields; -} - -#endif // __BIT_FIELD_READER_H__ diff --git a/c_common/models/compressors/src/sorter/bit_field_sorter_and_searcher.c b/c_common/models/compressors/src/sorter/bit_field_sorter_and_searcher.c deleted file mode 100644 index cb5b46e303..0000000000 --- a/c_common/models/compressors/src/sorter/bit_field_sorter_and_searcher.c +++ /dev/null @@ -1,1163 +0,0 @@ -/* - * Copyright (c) 2019 The University of Manchester - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//! \file -//! \brief SpiNNaker routing table minimisation with bitfield integration -//! control processor. -//! -//! Controls the attempt to minimise the router entries with bitfield -//! components. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "bit_field_common/routing_tables_utils.h" -#include "bit_field_common/compressor_sorter_structs.h" -#include "bit_field_common/bit_field_table_generator.h" -#include "bit_field_reader.h" - -//============================================================================ -// #defines and enums - -//! Time step for safety timer tick interrupt -#define TIME_STEP 1000 - -//! After how many time steps to kill the process -#define KILL_TIME 20000 - -//! Delay between checks of SDRAM polling -#define SDRAM_POLL_DELAY 50 - -//! Number of attempts for SDRAM poll -#define SDRAM_POLL_ATTEMPTS 20 - -//! The magic +1 for inclusive coverage that 0 index is no bitfields -#define ADD_INCLUSIVE_BIT 1 - -//! Flag for if a rtr_mc failure. -#define RTR_MC_FAILED 0 - -//! Bit shift for the app id for the route -#define ROUTE_APP_ID_BIT_SHIFT 24 - -//! Callback priorities -typedef enum priorities { - COMPRESSION_START_PRIORITY = 3, //!< General processing is low priority - TIMER_TICK_PRIORITY = 0 //!< Timer tick is high priority -} priorities; - -//============================================================================ -// global params - -//! DEBUG variable: counter of how many time steps have passed -uint32_t time_steps = 0; - -//! Whether we found a stopping position -volatile bool terminated = false; - -//! \brief The uncompressed router table -//! \details Address comes from `vcpu->user1` -uncompressed_table_region_data_t *restrict uncompressed_router_table; - -//! \brief The locations of bitfields from application processors -//! \details Address comes from `vcpu->user2` -region_addresses_t *restrict region_addresses; - -//! \brief SDRAM blocks that the fake heap can use -//! \details Address comes from `vcpu->user3` -available_sdram_blocks *restrict usable_sdram_regions; - -//! Best midpoint that record a success -int best_success = FAILED_TO_FIND; - -//! Lowest midpoint that record failure -int lowest_failure; - -//! The minimum number of bitfields to be merged in -uint32_t threshold_in_bitfields; - -//! The store for the last routing table that was compressed -table_t *restrict last_compressed_table = NULL; - -//! The compressor's SARK application id -uint32_t app_id = 0; - -//! \brief the list of bitfields in sorted order based off best effect, and -//! processor ids. -sorted_bit_fields_t *restrict sorted_bit_fields; - -//! Stores which values have been tested -bit_field_t tested_mid_points; - -//! SDRAM used to communicate with the compressors -comms_sdram_t *restrict comms_sdram; - -//! Record if the last action was to reduce cores due to malloc -bool just_reduced_cores_due_to_malloc = false; - -//! Number of ties after the first search compressor cores should be tasked -//! to find a better solution -uint32_t retires_left; - -//============================================================================ - -//! \brief Load the best routing table to the router. -//! \return Whether the table was loaded into the router or not -static inline bool load_routing_table_into_router(void) { - // Try to allocate sufficient room for the routing table. - int start_entry = rtr_alloc_id(last_compressed_table->size, app_id); - if (start_entry == RTR_MC_FAILED) { - log_error("Unable to allocate routing table of size %d\n", - last_compressed_table->size); - return false; - } - - // Load entries into the table (provided the allocation succeeded). - // Note that although the allocation included the specified - // application ID we also need to include it as the most significant - // byte in the route (see `sark_hw.c`). - log_debug("loading %d entries into router", last_compressed_table->size); - for (uint32_t entry_id = 0; entry_id < last_compressed_table->size; - entry_id++) { - entry_t entry = last_compressed_table->entries[entry_id]; - uint32_t route = entry.route | (app_id << ROUTE_APP_ID_BIT_SHIFT); - uint success = rtr_mc_set( - start_entry + entry_id, - entry.key_mask.key, entry.key_mask.mask, route); - - // chekc that the entry was set - if (success == RTR_MC_FAILED) { - log_error("failed to set a router table entry at index %d", - start_entry + entry_id); - return false; - } - } - - // Indicate we were able to allocate routing table entries. - return true; -} - -//! \brief Send a message forcing the processor to stop its compression -//! attempt -//! \param[in] processor_id: the processor ID to send a ::FORCE_TO_STOP to -void send_force_stop_message(int processor_id) { - if (comms_sdram[processor_id].sorter_instruction == RUN) { - log_debug("sending stop to processor %d", processor_id); - comms_sdram[processor_id].sorter_instruction = FORCE_TO_STOP; - } -} - -//! \brief Send a message telling the processor to prepare for the next run -//! \details This is critical as it tells the processor to clear the result -//! field -//! \param[in] processor_id: the processor ID to send a ::PREPARE to -void send_prepare_message(int processor_id) { - // set message params - log_debug("sending prepare to processor %d", processor_id); - comms_sdram[processor_id].sorter_instruction = PREPARE; - comms_sdram[processor_id].mid_point = FAILED_TO_FIND; -} - -//! \brief Set up the search bitfields. -//! \return True if the setup succeeded -static inline bool set_up_tested_mid_points(void) { - log_debug("set_up_tested_mid_point n bf addresses is %d", - sorted_bit_fields->n_bit_fields); - - uint32_t words = get_bit_field_size( - sorted_bit_fields->n_bit_fields + ADD_INCLUSIVE_BIT); - tested_mid_points = MALLOC(words * sizeof(bit_field_t)); - - // check the malloc worked - if (tested_mid_points == NULL) { - return false; - } - - // clear the bitfields - clear_bit_field(tested_mid_points, words); - - // return if successful - return true; -} - -//! \brief Store the addresses for freeing when response code is sent. -//! \param[in] processor_id: The compressor processor ID -//! \param[in] mid_point: The point in the bitfields to work from. -//! \param[in] table_size: Number of entries that the uncompressed routing -//! tables need to hold. -//! \return True if stored -static inline bool pass_instructions_to_compressor( - uint32_t processor_id, uint32_t mid_point, uint32_t table_size) { - bool success = routing_tables_utils_malloc( - comms_sdram[processor_id].routing_tables, table_size); - if (!success) { - log_warning("failed to create bitfield tables for midpoint %d", - mid_point); - return false; - } - - // set the midpoint for the given compressor processor. - comms_sdram[processor_id].mid_point = mid_point; - - if (comms_sdram[processor_id].mid_point == 0) { - // Info stuff but local sorted_bit_fields as compressor not set yet - log_info("using processor %d with %d entries for %d bitfields out of %d", - processor_id, table_size, - comms_sdram[processor_id].mid_point, - sorted_bit_fields->n_bit_fields); - } else { - // Info stuff using compressor data - log_info("using processor %d with %d entries for %d bitfields out of %d", - processor_id, table_size, - comms_sdram[processor_id].mid_point, - comms_sdram[processor_id].sorted_bit_fields->n_bit_fields); - } - - comms_sdram[processor_id].sorter_instruction = RUN; - return true; -} - -//! \brief Build tables and tries to set off a compressor processor based off -//! a mid-point. -//! \details If there is a problem will set reset the mid_point as untested and -//! set this and all unused compressors to ::DO_NOT_USE state. -//! \param[in] mid_point: The mid-point to start at -//! \param[in] processor_id: The processor to run the compression on -static inline void malloc_tables_and_set_off_bit_compressor( - int mid_point, int processor_id) { - // free any previous routing tables - routing_tables_utils_free_all(comms_sdram[processor_id].routing_tables); - - // malloc space for the routing tables - uint32_t table_size = bit_field_table_generator_max_size( - mid_point, &uncompressed_router_table->uncompressed_table, - sorted_bit_fields); - - // if successful, try setting off the bitfield compression - comms_sdram[processor_id].sorted_bit_fields = sorted_bit_fields; - bool success = pass_instructions_to_compressor( - processor_id, mid_point, table_size); - - if (!success) { - // OK, lets turn this and all ready processors off to save space. - // At least default no bitfield handled elsewhere so of to reduce. - comms_sdram[processor_id].sorter_instruction = DO_NOT_USE; - - for (int p_id = 0; p_id < MAX_PROCESSORS; p_id++) { - if ((comms_sdram[p_id].sorter_instruction == PREPARE) || - (comms_sdram[p_id].sorter_instruction == TO_BE_PREPARED)) { - comms_sdram[p_id].sorter_instruction = DO_NOT_USE; - } - } - - // Ok that midpoint did not work so need to try it again - bit_field_clear(tested_mid_points, mid_point); - } -} - -#if 0 -//! \brief Find the region ID in the region addresses for this processor ID -//! \param[in] processor_id: The processor ID to find the region ID in the -//! addresses -//! \return The address in the addresses region for the processor ID, or -//! `NULL` if none found. -static inline filter_region_t *find_processor_bit_field_region( - int processor_id) { - // find the right bitfield region - for (int r_id = 0; r_id < region_addresses->n_processors; r_id++) { - int region_proc_id = region_addresses->processors[r_id].processor; - log_debug("is looking for %d and found %d", - processor_id, region_proc_id); - if (region_proc_id == processor_id) { - return region_addresses->processors[r_id].filter; - } - } - - // if not found - log_error("failed to find the right region. WTF"); - malloc_extras_terminate(EXIT_SWERR); - return NULL; -} -#endif - -//! \brief Set the flag for the merged filters -static inline void set_merged_filters(void) { - log_debug("best_success %d", best_success); - for (int i = 0; i < best_success; i++) { - // Find the actual index of this bitfield - int bf_i = sorted_bit_fields->sort_order[i]; - // Update the flag - sorted_bit_fields->bit_fields[bf_i]->merged = 1; - } -} - -//! \brief Locate the next valid midpoint to test -//! \return The midpoint, or #FAILED_TO_FIND if no midpoints left -static inline int locate_next_mid_point(void) { - if (sorted_bit_fields->n_bit_fields == 0) { - return FAILED_TO_FIND; - } - - // if not tested yet / reset test all - if (!bit_field_test(tested_mid_points, sorted_bit_fields->n_bit_fields)) { - log_debug("Retrying all which is mid_point %d", - sorted_bit_fields->n_bit_fields); - return sorted_bit_fields->n_bit_fields; - } - - if (retires_left == 0) { - log_warning("Stopping compression due to retry count"); - return FAILED_TO_FIND; - } - retires_left -= 1; - - // need to find a midpoint - log_debug("n_bf_addresses %d tested_mid_points %d", - sorted_bit_fields->n_bit_fields, - bit_field_test(tested_mid_points, - sorted_bit_fields->n_bit_fields)); - - // the last point of the longest space - int best_end = FAILED_TO_FIND; - - // the length of the longest space to test - int best_length = 0; - - // the current length of the currently detected space - int current_length = 0; - - log_debug("best_success %d lowest_failure %d", - best_success, lowest_failure); - - // iterate over the range to binary search, looking for biggest block to - // explore, then take the middle of that block - - // NOTE: if there are no available bitfields, this will result in best end - // being still set to -1, as every bit is set, so there is no blocks with - // any best length, and so best end is never set and lengths will still be - // 0 at the end of the for loop. -1 is a special midpoint which higher - // code knows to recognise as no more exploration needed. - for (int index = best_success + 1; index <= lowest_failure; index++) { - log_debug("index: %d, value: %u current_length: %d", - index, bit_field_test(tested_mid_points, index), - current_length); - - // verify that the index has been used before - if (bit_field_test(tested_mid_points, index)) { - // if used before and is the end of the biggest block seen so far. - // Record and repeat. - if (current_length > best_length) { - best_length = current_length; - best_end = index - 1; - log_debug("found best_length: %d best_end %d", - best_length, best_end); - // if not the end of the biggest block, ignore (log for debugging) - } else { - log_debug("not best: %d best_end %d", best_length, best_end); - } - // if its seen a set we're at the end of a block. so reset the - // current block len, as we're about to start another block. - current_length = 0; - // not set, so still within a block, increase len. - } else { - current_length += 1; - } - } - - // use the best less half (shifted) of the best length - int new_mid_point = best_end - (best_length >> 1); - log_debug("returning mid point %d", new_mid_point); - - // set the mid point to be tested. (safe as we de-set if we fail later on) - if (new_mid_point >= 0) { - log_debug("setting new mid point %d", new_mid_point); - - // just a safety check, as this has caught us before. - if (bit_field_test(tested_mid_points, new_mid_point)) { - log_error("HOW ON EARTH DID YOU GET HERE!"); - malloc_extras_terminate(EXIT_SWERR); - } - } - - return new_mid_point; -} - -//! \brief Clean up when we've found a good compression -//! \details Handles the freeing of memory from compressor processors, waiting -//! for compressor processors to finish and removing merged bitfields from -//! the bitfield regions. -static inline void handle_best_cleanup(void) { - // load routing table into router - load_routing_table_into_router(); - log_debug("finished loading table"); - - log_info("setting set_n_merged_filters"); - set_merged_filters(); - - // This is to allow the host report to know how many bitfields on the chip - // merged without reading every cores bit-field region. - vcpu_t *sark_virtual_processor_info = (vcpu_t *) SV_VCPU; - uint processor_id = spin1_get_core_id(); - sark_virtual_processor_info[processor_id].user2 = best_success; - - // Safety to break out of loop in check_buffer_queue as terminate wont - // stop this interrupt - terminated = true; - - // set up user registers etc to finish cleanly - malloc_extras_terminate(EXITED_CLEANLY); -} - -//! \brief Prepare a processor for the first time. -//! \details This includes mallocing the comp_instruction_t user -//! \param[in] processor_id: The ID of the processor to prepare -//! \return True if the preparation succeeded. -bool prepare_processor_first_time(int processor_id) { - comms_sdram[processor_id].sorter_instruction = PREPARE; - - // Create the space for the routing table meta data - comms_sdram[processor_id].routing_tables = - MALLOC_SDRAM(sizeof(multi_table_t)); - if (comms_sdram[processor_id].routing_tables == NULL) { - comms_sdram[processor_id].sorter_instruction = DO_NOT_USE; - log_error("Error mallocing routing bake pointer on %d", processor_id); - return false; - } - - comms_sdram[processor_id].routing_tables->sub_tables = NULL; - comms_sdram[processor_id].routing_tables->n_sub_tables = 0; - comms_sdram[processor_id].routing_tables->n_entries = 0; - - // Pass the fake heap stuff - comms_sdram[processor_id].fake_heap_data = malloc_extras_get_stolen_heap(); - log_debug("fake_heap_data %u", comms_sdram[processor_id].fake_heap_data); - - // Check the processor is live - int count = 0; - while (comms_sdram[processor_id].compressor_state != PREPARED) { - // give chance for compressor to read - spin1_delay_us(SDRAM_POLL_DELAY); - if (++count > SDRAM_POLL_ATTEMPTS) { - comms_sdram[processor_id].sorter_instruction = DO_NOT_USE; - log_error("compressor failed to reply %d", processor_id); - return false; - } - } - return true; -} - -//! \brief Get the next processor id which is ready to run a compression. -//! \details May result in preparing a processor in the process. -//! \return The processor ID of the next available processor, or -//! #FAILED_TO_FIND if none -int find_prepared_processor(void) { - // Look for a prepared one - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - if (comms_sdram[processor_id].sorter_instruction == PREPARE && - comms_sdram[processor_id].compressor_state == PREPARED) { - log_debug("found prepared %d", processor_id); - return processor_id; - } - } - - // NOTE: This initialization component exists here due to a race condition - // with the compressors, where we dont know if they are reacting to - // "messages" before sync signal has been sent. We also have this here to - // save the 16 bytes per compressor core we dont end up using. - - // Look for a processor never used and prepare it - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - log_debug("processor_id %d status %d", - processor_id, comms_sdram[processor_id].sorter_instruction); - if (comms_sdram[processor_id].sorter_instruction == TO_BE_PREPARED) { - if (prepare_processor_first_time(processor_id)) { - log_debug("found to be prepared %d", processor_id); - return processor_id; - } - log_debug("first failed %d", processor_id); - } - } - log_debug("FAILED %d", FAILED_TO_FIND); - return FAILED_TO_FIND; -} - -//! \brief Get the next processor ID which is ready to run a compression. -//! \param[in] midpoint: The mid-point this processor will use -//! \return The processor ID of the next available processor, or -//! #FAILED_TO_FIND if none -int find_compressor_processor_and_set_tracker(int midpoint) { - int processor_id = find_prepared_processor(); - if (processor_id > FAILED_TO_FIND) { - // allocate this core to do this midpoint. - comms_sdram[processor_id].mid_point = midpoint; - // set the tracker to use this midpoint - bit_field_set(tested_mid_points, midpoint); - // return processor id - } - log_debug("returning %d", processor_id); - return processor_id; -} - -//! \brief Set up the compression attempt for the no bitfield version. -//! \return Whether setting off the compression attempt was successful. -bool setup_no_bitfields_attempt(void) { - if (threshold_in_bitfields > 0) { - log_info("No bitfields attempt skipped due to threshold of %d percent", - region_addresses->threshold); - return true; - } - - int processor_id = - find_compressor_processor_and_set_tracker(NO_BIT_FIELDS); - if (processor_id == FAILED_TO_FIND) { - log_error("No processor available for no bitfield attempt"); - malloc_extras_terminate(RTE_SWERR); - } - // set off a none bitfield compression attempt, to pipe line work - log_info("setting off the no bitfield version of the search on %u", - processor_id); - - pass_instructions_to_compressor(processor_id, NO_BIT_FIELDS, - uncompressed_router_table->uncompressed_table.size); - return true; -} - -//! \brief Check if a compressor processor is available. -//! \return Whether at least one processor is ready to compress -bool all_compressor_processors_busy(void) { - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - log_debug("processor_id %d status %d", - processor_id, comms_sdram[processor_id].sorter_instruction); - switch (comms_sdram[processor_id].sorter_instruction) { - case TO_BE_PREPARED: - return false; - case PREPARE: - if (comms_sdram[processor_id].compressor_state == PREPARED) { - return false; - } - break; - default: - // This processor is busy; continue to next one - break; - } - } - return true; -} - -//! \brief Check to see if all compressor processor are done and not ready -//! \return true if all processors are done and not set ready -bool all_compressor_processors_done(void) { - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - if (comms_sdram[processor_id].sorter_instruction >= PREPARE) { - return false; - } - } - return true; -} - -//! \brief Check if all processors are done; if yes, run best and exit -//! \return False if at least one compressors is not done. -//! True if termination fails (which shouldn't happen...) -bool exit_carry_on_if_all_compressor_processors_done(void) { - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - if (comms_sdram[processor_id].sorter_instruction >= PREPARE) { - return false; - } - } - - // Check there is nothing left to do - int mid_point = locate_next_mid_point(); - if (mid_point != FAILED_TO_FIND) { - log_error("Ran out of processors while still having mid_point %d to do", - mid_point); - malloc_extras_terminate(RTE_SWERR); - // Should never get here but break out of the loop - terminated = true; - return true; - } - - // Should never get here if above check worked but just in case - if (just_reduced_cores_due_to_malloc) { - log_error("Last result was a malloc fail! Use host"); - malloc_extras_terminate(RTE_SWERR); - // Should never get here but break out of the loop - terminated = true; - return true; - } - - // Check there was actually a result - if (best_success == FAILED_TO_FIND) { - log_error("No usable result found! Use host"); - malloc_extras_terminate(RTE_SWERR); - // Should never get here but break out of the loop - terminated = true; - return true; - } - - // Should never get here if above check failed but just in case - if (best_success < (int) threshold_in_bitfields) { - log_error("The threshold is %d bitfields. " - "Which is %d percent of the total of %d", - threshold_in_bitfields, region_addresses->threshold, - sorted_bit_fields->n_bit_fields); - log_error("Best result found was %d Which is below the threshold! " - "Use host", best_success); - malloc_extras_terminate(RTE_SWERR); - // Should never get here but break out of the loop - terminated = true; - return true; - } - - handle_best_cleanup(); - - // Should never get here but break out of the loop - terminated = true; - return true; -} - -//! \brief Start the binary search on another compressor if one available -void carry_on_binary_search(void) { - if (exit_carry_on_if_all_compressor_processors_done()) { - return; // Should never get here but just in case - } - if (all_compressor_processors_busy()) { - log_debug("all_compressor_processors_busy"); - return; //Pass back to check_buffer_queue - } - log_debug("start carry_on_binary_search"); - - int mid_point = locate_next_mid_point(); - log_debug("available with midpoint %d", mid_point); - - if (mid_point == FAILED_TO_FIND) { - // OK, lets turn all ready processors off as done. - for (int p_id = 0; p_id < MAX_PROCESSORS; p_id++) { - if (comms_sdram[p_id].sorter_instruction == PREPARE) { - comms_sdram[p_id].sorter_instruction = DO_NOT_USE; - } else if (comms_sdram[p_id].sorter_instruction > PREPARE) { - log_debug( - "waiting for processor %d status %d doing midpoint %u", - p_id, comms_sdram[p_id].sorter_instruction, - comms_sdram[p_id].mid_point); - } - } - return; - } - - int processor_id = find_compressor_processor_and_set_tracker(mid_point); - log_debug("start create at time step: %u", time_steps); - malloc_tables_and_set_off_bit_compressor(mid_point, processor_id); - log_debug("end create at time step: %u", time_steps); -} - -//! \brief Timer interrupt for controlling time taken to try to compress table -//! \param[in] unused0: unused -//! \param[in] unused1: unused -void timer_callback(UNUSED uint unused0, UNUSED uint unused1) { - time_steps++; - // Debug stuff please keep -#if 0 - if ((time_steps & 1023) == 0) { - log_info("time_steps: %u", time_steps); - } - if (time_steps > KILL_TIME) { - log_error("timer overran %u", time_steps); - malloc_extras_terminate(RTE_SWERR); - } -#endif -} - -//! \brief Handle the fact that a midpoint was successful. -//! \param[in] mid_point: The mid-point that succeeded. -//! \param[in] processor_id: The compressor processor ID -void process_success(int mid_point, int processor_id) { - // if the mid point is better than seen before, store results for final. - if (best_success <= mid_point) { - best_success = mid_point; - - // If we have a previous table free it as no longer needed - if (last_compressed_table != NULL) { - FREE(last_compressed_table); - } - - // Get last table and free the rest - last_compressed_table = routing_tables_utils_convert( - comms_sdram[processor_id].routing_tables); - log_debug("n entries is %d", last_compressed_table->size); - } else { - routing_tables_utils_free_all(comms_sdram[processor_id].routing_tables); - } - - // kill any search below this point, as they all redundant as - // this is a better search. - for (int proc_id = 0; proc_id < MAX_PROCESSORS; proc_id++) { - if (comms_sdram[proc_id].mid_point < mid_point) { - send_force_stop_message(proc_id); - } - } - - just_reduced_cores_due_to_malloc = false; - log_debug("finished process of successful compression"); -} - -//! \brief Handle the fact that a midpoint failed due to insufficient memory -//! \param[in] mid_point: The mid-point that failed -//! \param[in] processor_id: The compressor processor ID -void process_failed_malloc(int mid_point, int processor_id) { - routing_tables_utils_free_all(comms_sdram[processor_id].routing_tables); - // Remove the flag that say this midpoint has been checked - bit_field_clear(tested_mid_points, mid_point); - - // Add a retry to recover from the failure - retires_left += 1; - if (just_reduced_cores_due_to_malloc) { - log_info("Multiple malloc detected on %d keeping processor %d", - mid_point, processor_id); - // Not thresholding as just did a threshold - just_reduced_cores_due_to_malloc = false; - } else { - comms_sdram[processor_id].sorter_instruction = DO_NOT_USE; - log_info("Malloc detected on %d removing processor %d", - mid_point, processor_id); - just_reduced_cores_due_to_malloc = true; - } -} - -//! \brief Handle the fact that a midpoint failed for reasons other than -//! memory allocation. -//! \param[in] mid_point: The mid-point that failed -//! \param[in] processor_id: The compressor processor ID -void process_failed(int mid_point, int processor_id) { - // safety check to ensure we dont go on if the uncompressed failed - if (mid_point <= (int) threshold_in_bitfields) { - if (threshold_in_bitfields == NO_BIT_FIELDS) { - log_error("The no bitfields attempted failed! Giving up"); - } else { - log_error("The threshold is %d, " - "which is %d percent of the total of %d", - threshold_in_bitfields, region_addresses->threshold, - sorted_bit_fields->n_bit_fields); - log_error("The attempt with %d bitfields failed. ! Giving up", - mid_point); - } - malloc_extras_terminate(EXIT_FAIL); - } - if (lowest_failure > mid_point) { - log_debug("Changing lowest_failure from: %d to mid_point:%d", - lowest_failure, mid_point); - lowest_failure = mid_point; - } else { - log_debug("lowest_failure: %d already lower than mid_point:%d", - lowest_failure, mid_point); - } - routing_tables_utils_free_all(comms_sdram[processor_id].routing_tables); - - // tell all compression processors trying midpoints above this one - // to stop, as its highly likely a waste of time. - for (int proc_id = 0; proc_id < MAX_PROCESSORS; proc_id++) { - if (comms_sdram[proc_id].mid_point > mid_point) { - send_force_stop_message(proc_id); - } - } - - // handler to say this message has changed the last to not be a malloc fail - just_reduced_cores_due_to_malloc = false; -} - -//! \brief Process the response from a compressor's attempt to compress. -//! \param[in] processor_id: The compressor processor ID -//! \param[in] finished_state: The response code -void process_compressor_response( - int processor_id, compressor_states finished_state) { - // locate this responses midpoint - int mid_point = comms_sdram[processor_id].mid_point; - log_debug("received response %d from processor %d doing %d midpoint", - finished_state, processor_id, mid_point); - - // free the processor for future processing - send_prepare_message(processor_id); - - // process compressor response based off state. - switch (finished_state) { - case SUCCESSFUL_COMPRESSION: - // compressor was successful at compressing the tables. - log_debug("successful from processor %d doing mid point %d " - "best so far was %d", - processor_id, mid_point, best_success); - process_success(mid_point, processor_id); - break; - - case FAILED_MALLOC: - // compressor failed as a malloc request failed. - log_debug("failed by malloc from processor %d doing mid point %d", - processor_id, mid_point); - process_failed_malloc(mid_point, processor_id); - break; - - case FAILED_TO_COMPRESS: - // compressor failed to compress the tables as no more merge options. - log_debug("failed to compress from processor %d doing mid point %d", - processor_id, mid_point); - process_failed(mid_point, processor_id); - break; - - case RAN_OUT_OF_TIME: - // compressor failed to compress as it ran out of time. - log_debug("failed by time from processor %d doing mid point %d", - processor_id, mid_point); - process_failed(mid_point, processor_id); - break; - - case FORCED_BY_COMPRESSOR_CONTROL: - // compressor stopped at the request of the sorter. - log_debug("ack from forced from processor %d doing mid point %d", - processor_id, mid_point); - routing_tables_utils_free_all(comms_sdram[processor_id].routing_tables); - break; - - case UNUSED_CORE: - case PREPARED: - case COMPRESSING: - // states that shouldn't occur - log_error("no idea what to do with finished state %d, " - "from processor %d", finished_state, processor_id); - malloc_extras_terminate(RTE_SWERR); - } -} - -//! \brief Check compressors' state till they're finished. -//! \param[in] unused0: unused -//! \param[in] unused1: unused -void check_compressors(UNUSED uint unused0, UNUSED uint unused1) { - log_debug("Entering the check_compressors loop"); - // iterate over the compressors buffer until we have the finished state - while (!terminated) { - bool no_new_result = true; - bool failed_cpu = false; - - // iterate over processors looking for a new result - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - // Check each compressor asked to run or forced - compressor_states finished_state = - comms_sdram[processor_id].compressor_state; - uchar state = sv_vcpu[processor_id].cpu_state; - if (finished_state > COMPRESSING) { - no_new_result = false; - process_compressor_response(processor_id, finished_state); - } else if (state != CPU_STATE_RUN && state != CPU_STATE_PAUSE - && state != CPU_STATE_DEAD) { - log_error("CPU %u Failed!", processor_id); - failed_cpu = true; - no_new_result = false; - } - } - if (failed_cpu) { - rt_error(RTE_SWERR); - } - if (no_new_result) { - log_debug("no_new_result"); - // Check if another processor could be started or even done - carry_on_binary_search(); - } else { - log_debug("result"); - } - } - // Safety code in case exit after setting best_found fails - log_debug("exiting the interrupt, to allow the binary to finish"); -} - -//! \brief Start binary search on all compressors dividing the bitfields as -//! evenly as possible. -void start_binary_search(void) { - // Find the number of available processors - uint32_t available = 0; - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - if (comms_sdram[processor_id].sorter_instruction == TO_BE_PREPARED) { - available++; - } - } - - // Set off the worse acceptable (note no bitfield would have been set off - // earlier) - if (threshold_in_bitfields > 0) { - int processor_id = find_compressor_processor_and_set_tracker( - threshold_in_bitfields); - malloc_tables_and_set_off_bit_compressor( - threshold_in_bitfields, processor_id); - } - - // create slices and set off each slice. - uint32_t mid_point = sorted_bit_fields->n_bit_fields; - while ((available > 0) && (mid_point > threshold_in_bitfields)) { - int processor_id = find_compressor_processor_and_set_tracker(mid_point); - // Check the processor replied and has not been turned of by previous - if (processor_id == FAILED_TO_FIND) { - log_error("No processor available in start_binary_search"); - return; - } - malloc_tables_and_set_off_bit_compressor(mid_point, processor_id); - - // Find the next step which may change due to rounding - int step = ((mid_point - threshold_in_bitfields) / available); - if (step < 1) { - step = 1; - } - mid_point -= step; - available--; - } - - // Dont need all processors so turn the rest off - if (available > 0) { - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - if (comms_sdram[processor_id].sorter_instruction == TO_BE_PREPARED) { - comms_sdram[processor_id].sorter_instruction = DO_NOT_USE; - } - } - } -} - -//! \brief Ensure that for each router table entry there is at most 1 bitfield -//! per processor -static inline void check_bitfield_to_routes(void) { - filter_info_t **bit_fields = sorted_bit_fields->bit_fields; - int *processor_ids = sorted_bit_fields->processor_ids; - entry_t *entries = uncompressed_router_table->uncompressed_table.entries; - uint32_t n_bf = sorted_bit_fields->n_bit_fields; - uint32_t bf_i = 0; - - for (uint32_t i = 0; i < uncompressed_router_table->uncompressed_table.size; i++) { - // Bit field of seen processors (assumes less than 33 processors) - uint32_t seen_processors = 0; - // Go through all bitfields that match the key - while (bf_i < n_bf && (entries[i].key_mask.mask & bit_fields[bf_i]->key) == - entries[i].key_mask.key) { - - if (seen_processors & (1 << processor_ids[bf_i])) { - log_error("Routing key 0x%08x matches more than one bitfield key" - " on processor %d (last found 0x%08x)", - entries[i].key_mask.key, processor_ids[bf_i], - bit_fields[bf_i]->key); - malloc_extras_terminate(EXIT_SWERR); - } - seen_processors |= (1 << processor_ids[bf_i]); - bf_i++; - } - } -} - -//! \brief Start the work for the compression search -//! \param[in] unused0: unused -//! \param[in] unused1: unused -void start_compression_process(UNUSED uint unused0, UNUSED uint unused1) { - // malloc the struct and populate n bit-fields. DOES NOT populate the rest. - sorted_bit_fields = bit_field_reader_initialise(region_addresses); - // check state to fail if not read in - if (sorted_bit_fields == NULL) { - log_error("failed to read in bitfields, quitting"); - malloc_extras_terminate(EXIT_MALLOC); - } - - // Set the threshold - if (region_addresses->threshold == 0) { - threshold_in_bitfields = 0; - } else { - threshold_in_bitfields = (sorted_bit_fields->n_bit_fields * - region_addresses->threshold) / 100; - best_success = threshold_in_bitfields; - } - log_info("threshold_in_bitfields %d which is %d percent of %d", - threshold_in_bitfields, region_addresses->threshold, - sorted_bit_fields->n_bit_fields); - - // set up mid point trackers. NEEDED here as setup no bitfields attempt - // will use it during processor allocation. - set_up_tested_mid_points(); - - // set off the first compression attempt (aka no bitfields). - bool success = setup_no_bitfields_attempt(); - if (!success) { - log_error("failed to set up uncompressed attempt"); - malloc_extras_terminate(EXIT_MALLOC); - } - - log_debug("populating sorted bitfields at time step: %d", time_steps); - bit_field_reader_read_in_bit_fields(region_addresses, sorted_bit_fields); - check_bitfield_to_routes(); - - // the first possible failure is all bitfields so set there. - lowest_failure = sorted_bit_fields->n_bit_fields; - log_debug("finished reading bitfields at time step: %d", time_steps); - - //TODO: safety code to be removed - for (int bit_field_index = 0; - bit_field_index < sorted_bit_fields->n_bit_fields; - bit_field_index++) { - // get key - filter_info_t *bf_pointer = - sorted_bit_fields->bit_fields[bit_field_index]; - if (bf_pointer == NULL) { - log_error("failed at index %d", bit_field_index); - malloc_extras_terminate(RTE_SWERR); - return; - } - } - - // start the binary search by slicing the search space by the compressors. - start_binary_search(); - - // set off checker which in turn sets of the other compressor processors - spin1_schedule_callback( - check_compressors, 0, 0, COMPRESSION_START_PRIORITY); -} - -//! \brief Get a handle to this CPU's vcpu structure -//! \return the vcpu structure -static inline vcpu_t *get_this_vcpu_info(void) { - vcpu_t *sark_virtual_processor_info = (vcpu_t *) SV_VCPU; - vcpu_t *restrict this_vcpu_info = - &sark_virtual_processor_info[spin1_get_core_id()]; - return this_vcpu_info; -} - -//! \brief Set up a tracker for the user registers so that its easier to use -//! during coding. -static void initialise_user_register_tracker(void) { - log_debug("set up user register tracker (easier reading)"); - vcpu_t *restrict this_vcpu_info = get_this_vcpu_info(); - - // convert user registers to struct pointers - data_specification_metadata_t *restrict app_ptr_table = - (data_specification_metadata_t *) this_vcpu_info->user0; - uncompressed_router_table = - (uncompressed_table_region_data_t *) this_vcpu_info->user1; - region_addresses = (region_addresses_t *) this_vcpu_info->user2; - - comms_sdram = region_addresses->comms_sdram; - for (int processor_id = 0; processor_id < MAX_PROCESSORS; processor_id++) { - comms_sdram[processor_id].compressor_state = UNUSED_CORE; - comms_sdram[processor_id].sorter_instruction = NOT_COMPRESSOR; - comms_sdram[processor_id].mid_point = FAILED_TO_FIND; - comms_sdram[processor_id].routing_tables = NULL; - comms_sdram[processor_id].uncompressed_router_table = - &uncompressed_router_table->uncompressed_table; - comms_sdram[processor_id].sorted_bit_fields = NULL; - comms_sdram[processor_id].fake_heap_data = NULL; - } - usable_sdram_regions = (available_sdram_blocks *) this_vcpu_info->user3; - - retires_left = region_addresses->retry_count; - - log_debug("finished setting up register tracker:\n\n" - "user0 = %d\n user1 = %d\n user2 = %d\n user3 = %d\n", - app_ptr_table, uncompressed_router_table, - region_addresses, usable_sdram_regions); -} - -//! \brief Read in router table setup parameters. -static void initialise_routing_control_flags(void) { - app_id = uncompressed_router_table->app_id; - log_debug("app id %d, uncompress total entries %d", - app_id, uncompressed_router_table->uncompressed_table.size); -} - -//! \brief Set things up for the compressor processors so they are ready to be -//! compressing -//! \return Whether the initialisation of compressors succeeded -bool initialise_compressor_processors(void) { - // allocate DTCM memory for the processor status trackers - log_debug("allocate and step compressor processor status"); - compressor_processors_top_t *compressor_processors_top = (void *) - ®ion_addresses->processors[region_addresses->n_processors]; - - // Switch compressor processors to TO_BE_PREPARED - for (uint32_t processor_index = 0; - processor_index < compressor_processors_top->n_processors; - processor_index++) { - int processor_id = - compressor_processors_top->processor_id[processor_index]; - comms_sdram[processor_id].sorter_instruction = TO_BE_PREPARED; - } - return true; -} - -//! \brief Callback to set off the router compressor. -//! \return Whether the initialisation was successful -static bool initialise(void) { - log_debug("Setting up stuff to allow bitfield compressor control process" - " to occur."); - - // Get pointer to 1st virtual processor info struct in SRAM - initialise_user_register_tracker(); - - // ensure the original table is sorted by key - // (done here instead of by host for performance) - sort_table_by_key(&uncompressed_router_table->uncompressed_table); - - // get the compressor data flags (app id, compress only when needed, - //compress as much as possible, x_entries - initialise_routing_control_flags(); - - // build the fake heap for allocating memory - log_info("setting up fake heap for sdram usage"); - bool heap_creation = malloc_extras_initialise_and_build_fake_heap( - usable_sdram_regions); - if (!heap_creation) { - log_error("failed to setup stolen heap"); - return false; - } - - // allows us to not be forced to use the safety code - // (used in production mode) - malloc_extras_turn_off_safety(); - - log_debug("finished setting up fake heap for sdram usage"); - - // get the compressor processors stored in an array - log_info("start init of compressor processors"); - if (!initialise_compressor_processors()) { - log_error("failed to init the compressor processors."); - return false; - } - - // finished init - return true; -} - -//! \brief The main entrance. -void c_main(void) { - if (!initialise()) { - log_error("failed to init"); - malloc_extras_terminate(EXIT_FAIL); - } - - // set up interrupts - spin1_set_timer_tick(TIME_STEP); - spin1_callback_on(TIMER_TICK, timer_callback, TIMER_TICK_PRIORITY); - - // kick-start the process - spin1_schedule_callback( - start_compression_process, 0, 0, COMPRESSION_START_PRIORITY); - - // go - log_debug("waiting for sync"); - spin1_start(SYNC_WAIT); -} diff --git a/spinn_front_end_common/interface/interface_functions/machine_bit_field_router_compressor.py b/spinn_front_end_common/interface/interface_functions/machine_bit_field_router_compressor.py deleted file mode 100644 index e944a4b4fc..0000000000 --- a/spinn_front_end_common/interface/interface_functions/machine_bit_field_router_compressor.py +++ /dev/null @@ -1,799 +0,0 @@ -# Copyright (c) 2019 The University of Manchester -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools -import logging -import struct -from collections import defaultdict -from spinn_utilities.config_holder import get_config_bool, get_config_int -from spinn_utilities.log import FormatAdapter -from spinn_utilities.progress_bar import ProgressBar -from spinn_machine import CoreSubsets, Router -from spinnman.exceptions import ( - SpinnmanInvalidParameterException, - SpinnmanUnexpectedResponseCodeException, SpiNNManCoresNotInStateException) -from spinnman.model import ExecutableTargets -from spinnman.model.enums import CPUState, ExecutableType -from pacman.model.routing_tables import MulticastRoutingTables -from pacman.operations.router_compressors.ordered_covering_router_compressor\ - import ( - get_generality as - ordered_covering_generality) -from spinn_front_end_common.abstract_models.\ - abstract_supports_bit_field_routing_compression import ( - AbstractSupportsBitFieldRoutingCompression) -from spinn_front_end_common.data import FecDataView -from spinn_front_end_common.utilities.report_functions.\ - bit_field_compressor_report import ( - generate_provenance_item) -from spinn_front_end_common.utilities.exceptions import ( - CantFindSDRAMToUseException) -from spinn_front_end_common.utilities.helpful_functions import ( - get_defaultable_source_id, n_word_struct) -from spinn_front_end_common.utilities.system_control_logic import ( - run_system_application) -from spinn_front_end_common.utilities.constants import ( - BIT_FIELD_COMMS_SDRAM_TAG, BIT_FIELD_USABLE_SDRAM_TAG, - BIT_FIELD_ADDRESSES_SDRAM_TAG, BIT_FIELD_ROUTING_TABLE_SDRAM_TAG) -from .load_executable_images import filter_targets -from .host_bit_field_router_compressor import ( - generate_key_to_atom_map, generate_report_path, - start_compression_selection_process) - -logger = FormatAdapter(logging.getLogger(__name__)) - -#: Size of SDRAM allocation for addresses -SIZE_OF_SDRAM_ADDRESS_IN_BYTES = (17 * 2 * 4) + (3 * 4) - -# 7 pointers or int for each core. 4 Bytes for each 18 cores max -SIZE_OF_COMMS_SDRAM = 7 * 4 * 18 - -SECOND_TO_MICRO_SECOND = 1000000 - - -class _MachineBitFieldRouterCompressor(object): - """ - On-machine bitfield-aware routing table compression. - """ - - __slots__ = ["_compressor_aplx", "_compressor_type"] - - #: SDRAM tag the router compressor expects to find there routing tables in - ROUTING_TABLE_SDRAM_TAG = 1 - - #: SDRAM tag for the addresses the router compressor expects to find the - #: bitfield addresses for the chip. - BIT_FIELD_ADDRESSES_SDRAM_TAG = 2 - - # - TIMES_CYCLED_ROUTING_TABLES = 3 - - #: the successful identifier - SUCCESS = 0 - - #: How many header elements are in the region addresses (1, n addresses) - N_REGIONS_ELEMENT = 1 - - #: Minimum size a heap object needs in SDRAM. (limit on the size of useful - #: SDRAM regions to borrow) - _MIN_SIZE_FOR_HEAP = 32 - - # bit offset for compress only when needed - _ONLY_WHEN_NEEDED_BIT_OFFSET = 1 - - # bit offset for compress as much as possible - _AS_MUCH_AS_POSS_BIT_OFFSET = 2 - - # structs for performance requirements. - _FOUR_WORDS = struct.Struct("= size_to_borrow: - new_size = size - size_to_borrow - matrix_addresses_and_size[pos] = (base_address, new_size) - return base_address - raise CantFindSDRAMToUseException() - - def _add_to_addresses( - self, vertex, placement, region_addresses, - sdram_block_addresses_and_sizes): - """ - Adds data about the API-based vertex. - - :param AbstractSupportsBitFieldRoutingCompression vertex: - vertex which utilises the API - :param ~.Placement placement: placement of vertex - :param dict(tuple(int,int),list(tuple(int,int))) region_addresses: - store for data regions - :param sdram_block_addresses_and_sizes: store for surplus SDRAM - :type sdram_block_addresses_and_sizes: - dict(tuple(int,int),list(tuple(int,int))) - """ - # store the region sdram address's - bit_field_sdram_address = vertex.bit_field_base_address(placement) - region_addresses[placement.x, placement.y].append( - (bit_field_sdram_address, placement.p)) - - # store the available space from the matrix to borrow - blocks = vertex.regeneratable_sdram_blocks_and_sizes(placement) - - for (address, size) in blocks: - if size != 0 and size > self._MIN_SIZE_FOR_HEAP: - sdram_block_addresses_and_sizes[ - placement.x, placement.y].append((address, size)) - sorted( - sdram_block_addresses_and_sizes[placement.x, placement.y], - key=lambda data: data[0]) - - def _generate_addresses(self, progress_bar): - """ - Generates the bitfield SDRAM addresses. - - :param ~.ProgressBar progress_bar: the progress bar - :return: region_addresses and the executable targets to load the - router table compressor with bitfield. and the SDRAM blocks - available for use on each core that we plan to use - :rtype: tuple(dict(tuple(int,int),tuple(int,int)), - dict(tuple(int,int),list(tuple(int,int)))) - """ - # data holders - region_addresses = defaultdict(list) - sdram_block_addresses_and_sizes = defaultdict(list) - - for app_vertex in progress_bar.over( - FecDataView.iterate_vertices(), finish_at_end=False): - for m_vertex in app_vertex.machine_vertices: - if isinstance( - m_vertex, AbstractSupportsBitFieldRoutingCompression): - placement = FecDataView.get_placement_of_vertex(m_vertex) - self._add_to_addresses( - m_vertex, placement, region_addresses, - sdram_block_addresses_and_sizes) - return region_addresses, sdram_block_addresses_and_sizes - - def _generate_chip_data( - self, address_list, cores, comms_sdram, retry_count): - """ - Generate the region_addresses_t data. - - * Minimum percentage of bitfields to be merge in (currently ignored) - - * Number of times that the sorters should set of the compressions again - - * Pointer to the area malloc'ed to hold the comms_sdram - - * Number of processors in the list - - * The data for the processors - - :param list(tuple(int,int)) address_list: - the list of SDRAM addresses - :param ~.CoreSubset cores: compressor cores on this chip. - :param int comms_sdram: Address for communications block - :param retry_count: - Number of times that the sorters should set of the compressions - again. `None` for as much as needed - :type retry_count: int or None - :return: the byte array - :rtype: bytes - """ - threshold_percentage = get_config_int( - "Mapping", - "router_table_compression_with_bit_field_acceptance_threshold") - - data = self._FOUR_WORDS.pack( - threshold_percentage, - retry_count if retry_count is not None else 0xFFFFFFFF, - comms_sdram, len(address_list)) - for (bit_field, processor_id) in address_list: - data += self._TWO_WORDS.pack(bit_field, processor_id) - data += self._ONE_WORD.pack(len(cores)) - data += n_word_struct(len(cores)).pack(*list(cores.processor_ids)) - return data - - -def machine_bit_field_ordered_covering_compressor( - compress_as_much_as_possible=False): - """ - Compression with bit field and ordered covering. - - :param bool compress_as_much_as_possible: - whether to compress as much as possible - :return: where the compressors ran - """ - compressor = _MachineBitFieldRouterCompressor( - "bit_field_ordered_covering_compressor.aplx", "OrderedCovering") - return compressor.run(compress_as_much_as_possible) - - -def machine_bit_field_pair_router_compressor( - compress_as_much_as_possible=False): - """ - Compression with bit field pairing. - - :param bool compress_as_much_as_possible: - whether to compress as much as possible - :return: where the compressors ran - """ - compressor = _MachineBitFieldRouterCompressor( - "bit_field_pair_compressor.aplx", "Pair") - return compressor.run(compress_as_much_as_possible) From 2bbb29d820ec2cad239a1cb036178dd1253f126b Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Fri, 26 May 2023 07:57:10 +0100 Subject: [PATCH 05/23] Remove remaining references --- .../interface/abstract_spinnaker_base.py | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/spinn_front_end_common/interface/abstract_spinnaker_base.py b/spinn_front_end_common/interface/abstract_spinnaker_base.py index 1d265a36cc..39952031f3 100644 --- a/spinn_front_end_common/interface/abstract_spinnaker_base.py +++ b/spinn_front_end_common/interface/abstract_spinnaker_base.py @@ -94,10 +94,6 @@ sdram_outgoing_partition_allocator, spalloc_allocator, system_multicast_routing_generator, tags_loader, virtual_machine_generator, add_command_senders) -from spinn_front_end_common.interface.interface_functions.\ - machine_bit_field_router_compressor import ( - machine_bit_field_ordered_covering_compressor, - machine_bit_field_pair_router_compressor) from spinn_front_end_common.interface.interface_functions.\ host_no_bitfield_router_compression import ( ordered_covering_compression, pair_compression) @@ -1432,40 +1428,6 @@ def _execute_host_bitfield_compressor(self): compressed = host_based_bit_field_router_compressor() return compressed - def _execute_machine_bitfield_ordered_covering_compressor(self): - """ - Runs, times and logs the MachineBitFieldOrderedCoveringCompressor. - - .. note:: - Calling of this method is based on the configuration compressor or - virtual_compressor value - """ - with FecTimer( - "Machine bitfield ordered covering compressor", - TimerWork.COMPRESSING) as timer: - if timer.skip_if_virtual_board(): - return None - machine_bit_field_ordered_covering_compressor() - self._multicast_routes_loaded = True - return None - - def _execute_machine_bitfield_pair_compressor(self): - """ - Runs, times and logs the MachineBitFieldPairRouterCompressor. - - .. note:: - Calling of this method is based on the configuration compressor or - virtual_compressor value - """ - with FecTimer( - "Machine bitfield pair router compressor", - TimerWork.COMPRESSING) as timer: - if timer.skip_if_virtual_board(): - return None - self._multicast_routes_loaded = True - machine_bit_field_pair_router_compressor() - return None - def _execute_ordered_covering_compressor(self): """ Runs, times and logs the OrderedCoveringCompressor. @@ -1624,11 +1586,6 @@ def _do_early_compression(self, name): None, list(ProvenanceDataItem)) :raise ConfigurationException: if the name is not expected """ - if name == "MachineBitFieldOrderedCoveringCompressor": - return \ - self._execute_machine_bitfield_ordered_covering_compressor() - if name == "MachineBitFieldPairRouterCompressor": - return self._execute_machine_bitfield_pair_compressor() if name == "OrderedCoveringCompressor": return self._execute_ordered_covering_compressor() if name == "OrderedCoveringOnChipRouterCompression": From d9cb125e9d10b501310b88988bacbcf9d50b5985 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 13 Jun 2023 14:15:16 +0100 Subject: [PATCH 06/23] Remove hi_atom --- .../utility_models/reverse_ip_tag_multi_cast_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py b/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py index 92f366d8f4..a02e717195 100644 --- a/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py +++ b/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py @@ -176,7 +176,7 @@ def send_buffer_times(self, send_buffer_times): if hasattr(self._send_buffer_times[0], "__len__"): vertex_slice = vertex.vertex_slice send_buffer_times_to_set = self._send_buffer_times[ - vertex_slice.lo_atom:vertex_slice.hi_atom + 1] + vertex_slice.get_raster_ids()] vertex.send_buffer_times = send_buffer_times_to_set def enable_recording(self, new_state=True): From b9bdac910ac90acd56fe16dcad52474955866237 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 18 Sep 2023 10:07:35 +0100 Subject: [PATCH 07/23] Fix --- spinn_front_end_common/utilities/database/database_writer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index 5f60448216..e3709912a4 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -15,9 +15,9 @@ import logging import os from spinn_utilities.log import FormatAdapter -from pacman.utilities.utility_calls import get_field_based_keys from pacman.model.graphs.application.abstract import ( AbstractOneAppOneMachineVertex) +from pacman.utilities.utility_calls import get_keys from spinnman.spalloc import SpallocJob from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.sqlite_db import SQLiteDB @@ -25,7 +25,6 @@ AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) from spinn_front_end_common.utility_models import LivePacketGather from spinn_front_end_common.utility_models import LivePacketGatherMachineVertex -from pacman.utilities.utility_calls import get_keys logger = FormatAdapter(logging.getLogger(__name__)) DB_NAME = "input_output_database.sqlite3" From fe9c2de0e7d787da20a68b6232d1269aa5cc7f32 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 24 Oct 2023 07:35:08 +0100 Subject: [PATCH 08/23] Ensure we don't mistranslate keys we can't handle --- .../utility_models/live_packet_gather_machine_vertex.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spinn_front_end_common/utility_models/live_packet_gather_machine_vertex.py b/spinn_front_end_common/utility_models/live_packet_gather_machine_vertex.py index 7ae764230d..99fb5be483 100644 --- a/spinn_front_end_common/utility_models/live_packet_gather_machine_vertex.py +++ b/spinn_front_end_common/utility_models/live_packet_gather_machine_vertex.py @@ -28,6 +28,7 @@ from spinn_front_end_common.utilities.constants import ( SYSTEM_BYTES_REQUIREMENT, SIMULATION_N_BYTES, BYTES_PER_WORD) from spinn_front_end_common.utilities.exceptions import ConfigurationException +from pacman.model.graphs.common.mdslice import MDSlice _ONE_SHORT = struct.Struct(" Date: Tue, 24 Oct 2023 15:56:37 +0100 Subject: [PATCH 09/23] Reduce printing --- .../src/compressor_includes/pair_minimize.h | 15 ++++++++------- .../compressors/src/simple/simple_compressor.c | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/c_common/models/compressors/src/compressor_includes/pair_minimize.h b/c_common/models/compressors/src/compressor_includes/pair_minimize.h index 63f6231e47..625d220838 100644 --- a/c_common/models/compressors/src/compressor_includes/pair_minimize.h +++ b/c_common/models/compressors/src/compressor_includes/pair_minimize.h @@ -353,9 +353,9 @@ bool minimise_run(int target_length, bool *failed_by_malloc, } } - log_info("before sort %u", routes_count); + log_debug("before sort %u", routes_count); for (uint i = 0; i < routes_count; i++) { - log_info("%u", routes[i]); + log_debug("%u", routes[i]); } sort_routes(); @@ -364,12 +364,12 @@ bool minimise_run(int target_length, bool *failed_by_malloc, return false; } - log_info("after sort %u", routes_count); + log_debug("after sort %u", routes_count); for (uint i = 0; i < routes_count; i++) { - log_info("%u", routes[i]); + log_debug("%u", routes[i]); } - log_info("do sort_table by route %u", table_size); + log_debug("do sort_table by route %u", table_size); tc[T2_LOAD] = 0xFFFFFFFF; tc[T2_CONTROL] = 0x83; sort_table(); @@ -388,14 +388,14 @@ bool minimise_run(int target_length, bool *failed_by_malloc, while (left <= max_index) { int right = left; uint32_t left_route = routing_table_get_entry(left)->route; - log_info("A %u %u %u %u", left, max_index, right, left_route); + log_debug("A %u %u %u %u", left, max_index, right, left_route); while ((right < table_size - 1) && routing_table_get_entry(right+1)->route == left_route) { right++; } remaining_index = right + 1; - log_info("compress %u %u", left, right); + log_debug("compress %u %u", left, right); tc[T2_LOAD] = 0xFFFFFFFF; tc[T2_CONTROL] = 0x83; compress_by_route(left, right); @@ -417,6 +417,7 @@ bool minimise_run(int target_length, bool *failed_by_malloc, left = right + 1; } + log_debug("done %u %u", table_size, write_index); //for (uint i = 0; i < write_index; i++) { // entry_t *entry1 = routing_table_get_entry(i); diff --git a/c_common/models/compressors/src/simple/simple_compressor.c b/c_common/models/compressors/src/simple/simple_compressor.c index a1950e0ee7..0d2b1f1c0d 100644 --- a/c_common/models/compressors/src/simple/simple_compressor.c +++ b/c_common/models/compressors/src/simple/simple_compressor.c @@ -85,6 +85,7 @@ void compress_start(UNUSED uint unused0, UNUSED uint unused1) { // Free the block of SDRAM used to load the routing table. log_debug("free sdram blocks which held router tables"); + FREE((void *) header); // set the failed flag and exit malloc_extras_terminate(EXIT_FAIL); @@ -100,7 +101,7 @@ bool standalone(void) { //! \brief the main entrance. void c_main(void) { log_debug("%u bytes of free DTCM", sark_heap_max(sark.heap, 0)); - malloc_extras_turn_on_print(); + malloc_extras_turn_off_print(); // kick-start the process spin1_schedule_callback(compress_start, 0, 0, 3); From 4babb1f483461dbe92c61eb60c0a3449fa722910 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 20 Nov 2023 10:57:24 +0000 Subject: [PATCH 10/23] Log boards and error if not enough connections --- .../interface/interface_functions/spalloc_allocator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index cac204addf..9384d66bc0 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -420,6 +420,11 @@ def _launch_checked_job_old(n_boards: int, spalloc_kwargs: dict) -> Tuple[ job.destroy(str(ex)) raise connections = job.connections + if len(connections) < n_boards: + logger.warning("boards: {}", + str(connections).replace("{", "[").replace( + "}", "]")) + raise ValueError("Not enough connections detected") if logger.isEnabledFor(logging.DEBUG): logger.debug("boards: {}", str(connections).replace("{", "[").replace( From d20da72575792edba0f1c7e9abe1ca25699fee23 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 20 Nov 2023 11:05:14 +0000 Subject: [PATCH 11/23] flake8 --- .../interface/interface_functions/spalloc_allocator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py index 9384d66bc0..fdff7e58ca 100644 --- a/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py +++ b/spinn_front_end_common/interface/interface_functions/spalloc_allocator.py @@ -421,9 +421,9 @@ def _launch_checked_job_old(n_boards: int, spalloc_kwargs: dict) -> Tuple[ raise connections = job.connections if len(connections) < n_boards: - logger.warning("boards: {}", - str(connections).replace("{", "[").replace( - "}", "]")) + logger.warning( + "boards: {}", + str(connections).replace("{", "[").replace("}", "]")) raise ValueError("Not enough connections detected") if logger.isEnabledFor(logging.DEBUG): logger.debug("boards: {}", From cea75840e616477e8caf1ac21a04550e98183c1e Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 21 Nov 2023 11:17:57 +0000 Subject: [PATCH 12/23] run_time does not need to be an int --- spinn_front_end_common/interface/abstract_spinnaker_base.py | 6 +++--- spinn_front_end_common/interface/config_handler.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spinn_front_end_common/interface/abstract_spinnaker_base.py b/spinn_front_end_common/interface/abstract_spinnaker_base.py index 46d1929e92..de16577c71 100644 --- a/spinn_front_end_common/interface/abstract_spinnaker_base.py +++ b/spinn_front_end_common/interface/abstract_spinnaker_base.py @@ -383,7 +383,7 @@ def run_until_complete(self, n_steps: Optional[int] = None): self._run(n_steps, sync_time=0.0) FecTimer.end_category(TimerCategory.RUN_OTHER) - def run(self, run_time: Optional[int], sync_time: float = 0): + def run(self, run_time: Optional[float], sync_time: float = 0): """ Run a simulation for a fixed amount of time. @@ -453,7 +453,7 @@ def _calc_run_time(self, run_time: Optional[float]) -> Union[ f"{self._data_writer.get_hardware_time_step_us()} us") return n_machine_time_steps, total_run_time - def _run(self, run_time: Optional[int], sync_time: float): + def _run(self, run_time: Optional[float], sync_time: float): self._data_writer.start_run() try: @@ -477,7 +477,7 @@ def __is_main_thread() -> bool: """ return threading.get_ident() == threading.main_thread().ident - def __run(self, run_time: Optional[int], sync_time: float): + def __run(self, run_time: Optional[float], sync_time: float): """ The main internal run function. diff --git a/spinn_front_end_common/interface/config_handler.py b/spinn_front_end_common/interface/config_handler.py index c3afad6f19..86715b1666 100644 --- a/spinn_front_end_common/interface/config_handler.py +++ b/spinn_front_end_common/interface/config_handler.py @@ -122,7 +122,7 @@ def _error_on_previous(self, option) -> None: "See https://spinnakermanchester.github.io/common_pages/" "Algorithms.html.") - def _adjust_config(self, runtime: Union[int, bool, None]): + def _adjust_config(self, runtime: Optional[float]): """ Adjust and checks the configuration based on runtime From 20ff29aaec60d5aceafc2a66fa6da49d22bf9fbd Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 21 Nov 2023 11:30:58 +0000 Subject: [PATCH 13/23] remove unused import --- spinn_front_end_common/interface/config_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/config_handler.py b/spinn_front_end_common/interface/config_handler.py index 86715b1666..806e5d1613 100644 --- a/spinn_front_end_common/interface/config_handler.py +++ b/spinn_front_end_common/interface/config_handler.py @@ -17,7 +17,7 @@ import os import shutil import traceback -from typing import Optional, Type, Union +from typing import Optional, Type from spinn_utilities.log import FormatAdapter from spinn_utilities.config_holder import ( config_options, load_config, get_config_bool, get_config_int, From dbd855b62497d8248bfddba7106971cabc9465e6 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 27 Nov 2023 10:11:49 +0000 Subject: [PATCH 14/23] Allow multidimensional atoms per core --- .../utility_models/reverse_ip_tag_multi_cast_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py b/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py index ce1f85b242..58e4651b0c 100644 --- a/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py +++ b/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py @@ -14,7 +14,7 @@ import sys import numpy -from typing import List, Optional, Union +from typing import List, Optional, Union, Tuple from spinn_utilities.overrides import overrides from spinn_machine.tags import IPTag from spinnman.messages.eieio import EIEIOPrefix @@ -44,7 +44,7 @@ class ReverseIpTagMultiCastSource(ApplicationVertex, LegacyPartitionerAPI): def __init__( self, n_keys: int, label: Optional[str] = None, - max_atoms_per_core: int = sys.maxsize, + max_atoms_per_core: Optional[int, Tuple[int, ...]] = sys.maxsize, # Live input parameters receive_port: Optional[int] = None, From 2a8a6675fb1a636cf0efac4a3bfc6e3befce45d8 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 27 Nov 2023 10:15:12 +0000 Subject: [PATCH 15/23] Fix typing --- .../utility_models/reverse_ip_tag_multi_cast_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py b/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py index 58e4651b0c..269e9faffc 100644 --- a/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py +++ b/spinn_front_end_common/utility_models/reverse_ip_tag_multi_cast_source.py @@ -44,7 +44,8 @@ class ReverseIpTagMultiCastSource(ApplicationVertex, LegacyPartitionerAPI): def __init__( self, n_keys: int, label: Optional[str] = None, - max_atoms_per_core: Optional[int, Tuple[int, ...]] = sys.maxsize, + max_atoms_per_core: Optional[ + Union[int, Tuple[int, ...]]] = sys.maxsize, # Live input parameters receive_port: Optional[int] = None, From 4cca8a13c2cb5935b29792636e2fc9c68dddccdc Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 28 Nov 2023 12:01:20 +0000 Subject: [PATCH 16/23] Make this work with devices like this --- .../abstract_models/__init__.py | 3 +- .../abstract_models/live_output_device.py | 36 +++++++++++++++++++ spinn_front_end_common/data/fec_data_view.py | 23 +++++++++++- .../interface_functions/database_interface.py | 2 ++ .../utilities/database/database_writer.py | 21 +++++++++-- 5 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 spinn_front_end_common/abstract_models/live_output_device.py diff --git a/spinn_front_end_common/abstract_models/__init__.py b/spinn_front_end_common/abstract_models/__init__.py index bb52bc5cde..d02f97bca7 100644 --- a/spinn_front_end_common/abstract_models/__init__.py +++ b/spinn_front_end_common/abstract_models/__init__.py @@ -29,6 +29,7 @@ AbstractSupportsBitFieldRoutingCompression) from .abstract_can_reset import AbstractCanReset from .has_custom_atom_key_map import HasCustomAtomKeyMap +from .live_output_device import LiveOutputDevice __all__ = ("AbstractGeneratesDataSpecification", "AbstractHasAssociatedBinary", @@ -38,4 +39,4 @@ "AbstractVertexWithEdgeToDependentVertices", "AbstractCanReset", "AbstractSupportsBitFieldGeneration", "AbstractSupportsBitFieldRoutingCompression", - "HasCustomAtomKeyMap") + "HasCustomAtomKeyMap", "LiveOutputDevice") diff --git a/spinn_front_end_common/abstract_models/live_output_device.py b/spinn_front_end_common/abstract_models/live_output_device.py new file mode 100644 index 0000000000..f95b18de93 --- /dev/null +++ b/spinn_front_end_common/abstract_models/live_output_device.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 The University of Manchester +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from spinn_utilities.abstract_base import ( + AbstractBase, abstractmethod) +from pacman.model.graphs.machine.machine_vertex import MachineVertex +from typing import Dict, Tuple + + +class LiveOutputDevice(object, metaclass=AbstractBase): + """ + Indicates a device that will live-output other vertices, and so has a + different mapping of keys to atoms. + """ + __slots__ = () + + @abstractmethod + def get_device_output_keys(self) -> Dict[MachineVertex, Tuple[int, int]]: + """ + Get the atom key mapping to be output for each machine vertex received + by the device to be output. Note that the device may change the keys + as they pass through it, and this needs to be recognized here. + + :rtype: Dict[MachineVertex, Tuple[int, int]] + """ diff --git a/spinn_front_end_common/data/fec_data_view.py b/spinn_front_end_common/data/fec_data_view.py index 5d98071905..cf2f395df2 100644 --- a/spinn_front_end_common/data/fec_data_view.py +++ b/spinn_front_end_common/data/fec_data_view.py @@ -15,7 +15,7 @@ import logging import os from typing import ( - Dict, Iterable, Iterator, Optional, Set, Tuple, Union, TYPE_CHECKING) + Dict, Iterable, Iterator, Optional, Set, Tuple, Union, List, TYPE_CHECKING) from spinn_utilities.log import FormatAdapter from spinn_utilities.socket_address import SocketAddress from spinn_utilities.typing.coords import XY @@ -43,6 +43,7 @@ from spinn_front_end_common.utilities.notification_protocol import ( NotificationProtocol) from spinn_front_end_common.utility_models import LivePacketGather + from spinn_front_end_common.abstract_models import LiveOutputDevice logger = FormatAdapter(logging.getLogger(__name__)) _EMPTY_CORE_SUBSETS = CoreSubsets() @@ -88,6 +89,7 @@ class _FecDataModel(object): "_java_caller", "_live_packet_recorder_params", "_live_output_vertices", + "_live_output_devices", "_n_boards_required", "_n_chips_required", "_n_chips_in_graph", @@ -134,6 +136,7 @@ def _clear(self) -> None: LivePacketGatherParameters, LivePacketGather]] = None self._live_output_vertices: Set[Tuple[ApplicationVertex, str]] = set() + self._live_output_devices: List[LiveOutputDevice] = list() self._java_caller: Optional[JavaCaller] = None self._n_boards_required: Optional[int] = None self._n_chips_required: Optional[int] = None @@ -1321,3 +1324,21 @@ def get_next_ds_references(cls, number): cls.__fec_data._next_ds_reference+number) cls.__fec_data._next_ds_reference += number return list(references) + + @classmethod + def add_live_output_device(cls, device: LiveOutputDevice): + """ + Add a live output device. + + :param device: The device to be added + """ + cls.__fec_data._live_output_devices.append(device) + + @classmethod + def iterate_live_output_devices(cls) -> Iterable[LiveOutputDevice]: + """ + Iterate over live output devices. + + :rtype: iterable(LiveOutputDevice) + """ + return iter(cls.__fec_data._live_output_devices) diff --git a/spinn_front_end_common/interface/interface_functions/database_interface.py b/spinn_front_end_common/interface/interface_functions/database_interface.py index ce228fab64..9017c06d47 100644 --- a/spinn_front_end_common/interface/interface_functions/database_interface.py +++ b/spinn_front_end_common/interface/interface_functions/database_interface.py @@ -90,4 +90,6 @@ def _write_to_db(w: DatabaseWriter, runtime: Optional[float]): else: machine_vertices.add((vertex, part_id)) w.create_atom_to_event_id_mapping(machine_vertices) + w.create_device_atom_event_id_mapping( + FecDataView.iterate_live_output_devices()) p.update() diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index f14fb4d8da..85a6391234 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -27,7 +27,7 @@ from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.sqlite_db import SQLiteDB from spinn_front_end_common.abstract_models import ( - AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) + AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap, LiveOutputDevice) from spinn_front_end_common.utility_models import ( LivePacketGather, LivePacketGatherMachineVertex) if TYPE_CHECKING: @@ -262,9 +262,26 @@ def create_atom_to_event_id_mapping( INSERT INTO event_to_atom_mapping( vertex_id, event_id, atom_id) VALUES (?, ?, ?) - """, ((m_vertex_id, int(key), i) for i, key in atom_keys) + """, ((m_vertex_id, int(key), int(i)) for i, key in atom_keys) ) + def create_device_atom_event_id_mapping( + self, devices: Iterable[LiveOutputDevice]): + """ + Add output mappings for devices. + """ + for device in devices: + for m_vertex, atom_keys in device.get_device_output_keys(): + m_vertex_id = self.__vertex_to_id[m_vertex] + self.executemany( + """ + INSERT INTO event_to_atom_mapping( + vertex_id, event_id, atom_id) + VALUES (?, ?, ?) + """, ((m_vertex_id, int(key), int(i)) + for i, key in atom_keys) + ) + def _get_machine_lpg_mappings( self, part: AbstractEdgePartition) -> Iterable[ Tuple[MachineVertex, str, MachineVertex]]: From c87d2351036bd3dfe4289f040b30fab280a639b7 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 28 Nov 2023 12:14:56 +0000 Subject: [PATCH 17/23] Iterate correctly --- spinn_front_end_common/utilities/database/database_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index 85a6391234..e469db3ad2 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -271,7 +271,7 @@ def create_device_atom_event_id_mapping( Add output mappings for devices. """ for device in devices: - for m_vertex, atom_keys in device.get_device_output_keys(): + for m_vertex, atom_keys in device.get_device_output_keys().items(): m_vertex_id = self.__vertex_to_id[m_vertex] self.executemany( """ From 9c7392e92e9566ae39de89458c4db331e73746d4 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 28 Nov 2023 12:15:40 +0000 Subject: [PATCH 18/23] Mypy fix --- spinn_front_end_common/abstract_models/live_output_device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spinn_front_end_common/abstract_models/live_output_device.py b/spinn_front_end_common/abstract_models/live_output_device.py index f95b18de93..c051b633bb 100644 --- a/spinn_front_end_common/abstract_models/live_output_device.py +++ b/spinn_front_end_common/abstract_models/live_output_device.py @@ -34,3 +34,4 @@ def get_device_output_keys(self) -> Dict[MachineVertex, Tuple[int, int]]: :rtype: Dict[MachineVertex, Tuple[int, int]] """ + raise NotImplementedError From 3bfd76be49fa40d3c2bbe71838e2e5d8ccc351bf Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 28 Nov 2023 12:24:13 +0000 Subject: [PATCH 19/23] Fix type here --- .../abstract_models/live_output_device.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spinn_front_end_common/abstract_models/live_output_device.py b/spinn_front_end_common/abstract_models/live_output_device.py index c051b633bb..7a7de5861d 100644 --- a/spinn_front_end_common/abstract_models/live_output_device.py +++ b/spinn_front_end_common/abstract_models/live_output_device.py @@ -15,7 +15,7 @@ from spinn_utilities.abstract_base import ( AbstractBase, abstractmethod) from pacman.model.graphs.machine.machine_vertex import MachineVertex -from typing import Dict, Tuple +from typing import Dict, Tuple, List class LiveOutputDevice(object, metaclass=AbstractBase): @@ -26,12 +26,13 @@ class LiveOutputDevice(object, metaclass=AbstractBase): __slots__ = () @abstractmethod - def get_device_output_keys(self) -> Dict[MachineVertex, Tuple[int, int]]: + def get_device_output_keys(self) -> Dict[MachineVertex, + List[Tuple[int, int]]]: """ Get the atom key mapping to be output for each machine vertex received by the device to be output. Note that the device may change the keys as they pass through it, and this needs to be recognized here. - :rtype: Dict[MachineVertex, Tuple[int, int]] + :rtype: Dict[MachineVertex, List[Tuple[int, int]]] """ raise NotImplementedError From 22958d861492b87195e8d1cf11daf693f8eb50c4 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jan 2024 11:04:58 +0000 Subject: [PATCH 20/23] python 3.12 --- .github/workflows/c_actions.yml | 4 ++-- .github/workflows/python_actions.yml | 6 +++--- doc/source/conf.py | 2 +- setup.cfg | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/c_actions.yml b/.github/workflows/c_actions.yml index a9d3449ea4..83995e3f11 100644 --- a/.github/workflows/c_actions.yml +++ b/.github/workflows/c_actions.yml @@ -35,11 +35,11 @@ jobs: uses: ./support/actions/apt-get-install with: packages: doxygen gcc-arm-none-eabi - - name: "Prepare: Set up Python 3.8" + - name: "Prepare: Set up Python 3.12" # Note: Python is needed for spinn_utilities.make_tools when building uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.12 - name: "Prepare: Set SPINN_DIRS" run: | echo "Set SPINN_DIRS to $PWD/spinnaker_tools" diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml index 3977227c63..01df7e6210 100644 --- a/.github/workflows/python_actions.yml +++ b/.github/workflows/python_actions.yml @@ -26,7 +26,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] steps: - name: Checkout @@ -60,7 +60,7 @@ jobs: uses: ./support/actions/pytest with: tests: unittests fec_integration_tests - coverage: ${{ matrix.python-version == 3.8 }} + coverage: ${{ matrix.python-version == 3.12 }} cover-packages: ${{ env.BASE_PKG }} coveralls-token: ${{ secrets.GITHUB_TOKEN }} @@ -79,7 +79,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - python-version: [3.8] + python-version: [3.12] steps: - name: Checkout diff --git a/doc/source/conf.py b/doc/source/conf.py index ee8ecff213..b3b034ecc0 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -55,7 +55,7 @@ spinnaker_doc_version = "latest" intersphinx_mapping = { - 'python': ('https://docs.python.org/3.8', None), + 'python': ('https://docs.python.org/3.12', None), 'numpy': ("https://numpy.org/doc/1.19/", None), 'spinn_utilities': ( f'https://spinnutils.readthedocs.io/en/{spinnaker_doc_version}/', diff --git a/setup.cfg b/setup.cfg index f06581540b..4c979c6eda 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,18 +30,18 @@ classifiers = Operating System :: Microsoft :: Windows Operating System :: MacOS Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 maintainer = SpiNNakerTeam maintainer_email = spinnakerusers@googlegroups.com keywords = spinnaker [options] -python_requires = >=3.7, <4 +python_requires = >=3.8, <4 packages = find: zip_safe = True include_package_data = True From 3a9f9779844f58f5fb950d1684deee5bb1f71e76 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jan 2024 11:15:44 +0000 Subject: [PATCH 21/23] flake8 --- .../interface/provenance/provenance_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/provenance/provenance_reader.py b/spinn_front_end_common/interface/provenance/provenance_reader.py index f9010451f0..852548fe03 100644 --- a/spinn_front_end_common/interface/provenance/provenance_reader.py +++ b/spinn_front_end_common/interface/provenance/provenance_reader.py @@ -148,7 +148,7 @@ def get_provenance_for_router(self, x: int, y: int) -> str: ORDER BY description """ return "\n".join( - f"{ cast(str, row[0]) }: { cast(int, row[1]) }" + f"{cast(str, row[0])}: {cast(int, row[1])}" for row in self.run_query(query, [int(x), int(y)])) def get_cores_with_provenace(self) -> List[XYP]: From 0958485595780c9b3fe151c283fa7810ebd0a3eb Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jan 2024 11:27:53 +0000 Subject: [PATCH 22/23] from collections.abc import Sized --- spinn_front_end_common/interface/provenance/fec_timer.py | 3 ++- spinn_front_end_common/utilities/iobuf_extractor.py | 3 ++- .../utility_models/command_sender_machine_vertex.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/spinn_front_end_common/interface/provenance/fec_timer.py b/spinn_front_end_common/interface/provenance/fec_timer.py index c79f59b7ab..c9e85e368c 100644 --- a/spinn_front_end_common/interface/provenance/fec_timer.py +++ b/spinn_front_end_common/interface/provenance/fec_timer.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations +from collections.abc import Sized import logging import os import time from datetime import timedelta -from typing import List, Optional, Sized, Union, TYPE_CHECKING +from typing import List, Optional, Union, TYPE_CHECKING from typing_extensions import Literal, Self from spinn_utilities.config_holder import (get_config_bool) from spinn_utilities.log import FormatAdapter diff --git a/spinn_front_end_common/utilities/iobuf_extractor.py b/spinn_front_end_common/utilities/iobuf_extractor.py index 33ffdc8955..713fa77f06 100644 --- a/spinn_front_end_common/utilities/iobuf_extractor.py +++ b/spinn_front_end_common/utilities/iobuf_extractor.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections.abc import Sized import logging import os import re -from typing import List, Optional, Pattern, Sequence, Set, Sized, Tuple, Union +from typing import List, Optional, Pattern, Sequence, Set, Tuple, Union from spinn_utilities.log import FormatAdapter from spinn_utilities.make_tools.replacer import Replacer from spinn_utilities.progress_bar import ProgressBar diff --git a/spinn_front_end_common/utility_models/command_sender_machine_vertex.py b/spinn_front_end_common/utility_models/command_sender_machine_vertex.py index b96f42a6f0..915a204ed5 100644 --- a/spinn_front_end_common/utility_models/command_sender_machine_vertex.py +++ b/spinn_front_end_common/utility_models/command_sender_machine_vertex.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations +from collections.abc import Sized from enum import IntEnum from typing import ( - Callable, Dict, Iterable, List, Sequence, Set, Sized, Tuple, Type, TypeVar, + Callable, Dict, Iterable, List, Sequence, Set, Tuple, Type, TypeVar, TYPE_CHECKING) from spinn_utilities.overrides import overrides from spinnman.model.enums import ExecutableType From 9628fa036b4edb2ed30c40a597eccb613ca5d24d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 2 Jan 2024 13:13:23 +0000 Subject: [PATCH 23/23] remove distutils --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index cd8dbf7730..0dfbe6af33 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import distutils.dir_util from setuptools import setup +import shutil import os import sys @@ -26,8 +26,8 @@ this_dir = os.path.dirname(os.path.abspath(__file__)) build_dir = os.path.join(this_dir, "build") if os.path.isdir(build_dir): - distutils.dir_util.remove_tree(build_dir) + shutil.rmtree(build_dir) egg_dir = os.path.join(this_dir, "SpiNNFrontEndCommon.egg-info") if os.path.isdir(egg_dir): - distutils.dir_util.remove_tree(egg_dir) + shutil.rmtree(egg_dir) setup()