[WIP] Router Lookahead Profiler

.github/workflows/test.yml

@@ -156,6 +156,11 @@ jobs:
             params: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVPR_USE_EZGL=on -DVPR_USE_SERVER=off',
             suite: 'vtr_reg_basic'
           },
+          {
+            name: 'Basic with PROFILE_LOOKAHEAD',
+            params: '-DVTR_ASSERT_LEVEL=3 -DVPR_PROFILE_LOOKAHEAD=on',
+            suite: 'vtr_reg_basic'
+          },
           {
             name: 'Basic with CAPNPROTO disabled',
             params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_CAPNPROTO=off',

CMakeLists.txt

@@ -42,6 +42,9 @@ option(VTR_ENABLE_CAPNPROTO "Enable capnproto binary serialization support in VP
 #Allow the user to decide whether to compile the server module
 option(VPR_USE_SERVER "Specify whether vpr enables the server mode" ON)
 
+#Allow the user to decide whether to profile the router lookahead
+option(VPR_PROFILE_LOOKAHEAD "Specify whether to enable the router LookaheadProfiler" OFF)
+
 #Allow the user to enable/disable VPR analytic placement
 #VPR option --enable_analytic_placer is also required for Analytic Placement
 option(VPR_ANALYTIC_PLACE "Enable analytic placement in VPR." ON)

requirements.txt

@@ -1,9 +1,15 @@
 prettytable
 lxml
 psutil
+scikit-learn
+matplotlib
+distinctipy
+colour
+seaborn
 pandas
 numpy
 scipy
+
 # Python linter and formatter
 click==8.0.2 # Our version of black needs an older version of click (https://stackoverflow.com/questions/71673404/importerror-cannot-import-name-unicodefun-from-click)
 black==21.4b0

utils/route_diag/src/main.cpp

@@ -130,7 +130,14 @@ static void do_one_route(const Netlist<>& net_list,
         VTR_ASSERT(cheapest.index == sink_node);
 
         vtr::optional<const RouteTreeNode&> rt_node_of_sink;
-        std::tie(std::ignore, rt_node_of_sink) = tree.update_from_heap(&cheapest, OPEN, nullptr, router_opts.flat_routing);
+        std::tie(std::ignore, rt_node_of_sink) = tree.update_from_heap(/*htpr=*/&cheapest,
+                                                                       /*target_net_pin_index=*/OPEN,
+                                                                       /*spatial_rt_lookup=*/nullptr,
+                                                                       router_opts.flat_routing,
+                                                                       router.get_router_lookahead(),
+                                                                       cost_params,
+                                                                       net_list,
+                                                                       conn_params.net_id_);
 
         //find delay
         float net_delay = rt_node_of_sink.value().Tdel;

vpr/CMakeLists.txt

@@ -51,6 +51,16 @@ else()
     message(STATUS "Server mode is disabled${SERVER_DISABILED_REASON}")
 endif()
 
+#Handle router lookahead profiler setup
+set(ROUTER_DEFINES "")
+
+if (VPR_PROFILE_LOOKAHEAD)
+    list(APPEND ROUTER_DEFINES "-DPROFILE_LOOKAHEAD")
+    message(STATUS "Router lookahead profiler enabled")
+else ()
+    message(STATUS "Router lookahead profiler disabled")
+endif ()
+
 #
 # Build Configuration
 #
@@ -164,7 +174,7 @@ if (VPR_USE_EZGL STREQUAL "on")
 
 endif()
 
-target_compile_definitions(libvpr PUBLIC ${GRAPHICS_DEFINES} ${SERVER_DEFINES})
+target_compile_definitions(libvpr PUBLIC ${GRAPHICS_DEFINES} ${SERVER_DEFINES} ${ROUTER_DEFINES})
 
 if(${VTR_ENABLE_CAPNPROTO})
     target_link_libraries(libvpr libvtrcapnproto)

vpr/src/base/SetupVPR.cpp

@@ -486,6 +486,8 @@ static void SetupRouterOpts(const t_options& Options, t_router_opts* RouterOpts)
     RouterOpts->write_intra_cluster_router_lookahead = Options.write_intra_cluster_router_lookahead;
     RouterOpts->read_intra_cluster_router_lookahead = Options.read_intra_cluster_router_lookahead;
 
+    RouterOpts->lookahead_profiling_output = Options.lookahead_profiling_output;
+
     RouterOpts->router_heap = Options.router_heap;
     RouterOpts->exit_after_first_routing_iteration = Options.exit_after_first_routing_iteration;
 

vpr/src/base/read_options.cpp

@@ -1673,6 +1673,14 @@ argparse::ArgumentParser create_arg_parser(const std::string& prog_name, t_optio
         .help("Writes the intra-cluster lookahead data to the specified file.")
         .show_in(argparse::ShowIn::HELP_ONLY);
 
+    file_grp.add_argument(args.lookahead_profiling_output, "--profile_router_lookahead")
+        .help(
+            "For every routed sink, record the cost, delay, and congestion estimated by the router lookahead and the "
+            "actual cost, delay, and congestion, from every node along each route to the sink. These results, along with many "
+            "other attributes of the node, are recorded into the file name provided. Used to assist in debugging or validating "
+            "the router lookahead. File extension must be .csv.")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
     file_grp.add_argument(args.read_placement_delay_lookup, "--read_placement_delay_lookup")
         .help(
             "Reads the placement delay lookup from the specified file instead of computing it.")

vpr/src/base/read_options.h

@@ -44,6 +44,8 @@ struct t_options {
     argparse::ArgValue<std::string> write_intra_cluster_router_lookahead;
     argparse::ArgValue<std::string> read_intra_cluster_router_lookahead;
 
+    argparse::ArgValue<std::string> lookahead_profiling_output;
+
     argparse::ArgValue<std::string> write_block_usage;
 
     /* Stage Options */

vpr/src/base/vpr_context.h

@@ -46,6 +46,9 @@ class PostClusterDelayCalculator;
 
 #endif /* NO_SERVER */
 
+// Forward declaration
+class LookaheadProfiler;
+
 /**
  * @brief A Context is collection of state relating to a particular part of VPR
  *
@@ -515,6 +518,11 @@ struct RoutingContext : public Context {
      * @brief User specified routing constraints
      */
     UserRouteConstraints constraints;
+
+    /**
+     * @brief Writes out information used to profile the accuracy of the router lookahead.
+     */
+    LookaheadProfiler lookahead_profiler;
 };
 
 /**

vpr/src/base/vpr_types.h

@@ -1400,6 +1400,10 @@ struct t_router_opts {
     std::string write_intra_cluster_router_lookahead;
     std::string read_intra_cluster_router_lookahead;
 
+    ///@brief The name of the output .csv file when PROFILE_LOOKAHEAD and --profile_router_lookahead are used.
+    ///If the string is empty, there will be no output.
+    std::string lookahead_profiling_output;
+
     e_heap_type router_heap;
     bool exit_after_first_routing_iteration;
 
@@ -1667,23 +1671,32 @@ constexpr bool is_src_sink(e_rr_type type) { return (type == SOURCE || type == S
  * @brief Extra information about each rr_node needed only during routing
  *        (i.e. during the maze expansion).
  *
- *   @param prev_edge  ID of the edge (globally unique edge ID in the RR Graph)
- *                     that was used to reach this node from the previous node.
- *                     If there is no predecessor, prev_edge = NO_PREVIOUS.
- *   @param acc_cost   Accumulated cost term from previous Pathfinder iterations.
- *   @param path_cost  Total cost of the path up to and including this node +
- *                     the expected cost to the target if the timing_driven router
- *                     is being used.
- *   @param backward_path_cost  Total cost of the path up to and including this
- *                     node.
- *   @param occ        The current occupancy of the associated rr node
+ *   @param prev_edge   ID of the edge (globally unique edge ID in the RR Graph)
+ *                      that was used to reach this node from the previous node.
+ *                      If there is no predecessor, prev_edge = NO_PREVIOUS.
+ *   @param acc_cost    Accumulated cost term from previous Pathfinder iterations.
+ *   @param path_cost   Total cost of the path up to and including this node +
+ *                      the expected cost to the target if the timing_driven router
+ *                      is being used.
+ *   @param backward_path_cost          Total cost of the path up to and including
+ *                                      this node.
+ *   @param occ         The current occupancy of the associated rr node
  */
 struct t_rr_node_route_inf {
     RREdgeId prev_edge;
 
     float acc_cost;
     float path_cost;
     float backward_path_cost;
+#ifdef PROFILE_LOOKAHEAD
+    // This data is needed for the LookaheadProfiler, when enabled. It is only conditionally
+    // compiled since this struct is a hot and large data structure.
+
+    ///@brief Total delay in the path up to and including this node.
+    float backward_path_delay;
+    ///@brief Total congestion in the path up to and including this node.
+    float backward_path_congestion;
+#endif
 
   public: //Accessors
     short occ() const { return occ_; }

vpr/src/place/timing_place_lookup.cpp

@@ -110,6 +110,22 @@ static void generic_compute_matrix_iterative_astar(
     const std::set<std::string>& allowed_types,
     bool /***/);
 
+/**
+ * @brief Compute delta delay matrix using Djikstra's algorithm to find shortest paths from a IPIN at the
+ * source location to OPINs within (start_x, start_y) to (end_x, end_y).
+ *
+ * @param route_profiler Only used to call get_net_list(), which is passed into
+ * calculate_all_path_delays_from_rr_node(), which is needed for the LookaheadProfiler.
+ * @param matrix The matrix to be filled.
+ * @param layer_num The layer of the source and sink nodes to be sampled.
+ * @param (source_x, source_y) The coordinates of the tile to sample an IPIN at.
+ * @param (start_x, start_y, end_x, end_y) The bounds within which OPINs should be sampled.
+ * @param router_opts
+ * @param measure_directconnect Whether to measure/include direct connects.
+ * @param allowed_types The allowed tile type names for the source location. If this vector is empty, all
+ * names are allowed. If the source tile type is not allowed, the matrix is filled with EMPTY_DELTA.
+ * @param is_flat Whether flat routing is being used.
+ */
 static void generic_compute_matrix_dijkstra_expansion(
     RouterDelayProfiler& route_profiler,
     vtr::Matrix<std::vector<float>>& matrix,
@@ -441,7 +457,7 @@ static void add_delay_to_matrix(
 }
 
 static void generic_compute_matrix_dijkstra_expansion(
-    RouterDelayProfiler& /*route_profiler*/,
+    RouterDelayProfiler& route_profiler,
     vtr::Matrix<std::vector<float>>& matrix,
     int layer_num,
     int source_x,
@@ -489,7 +505,7 @@ static void generic_compute_matrix_dijkstra_expansion(
         RRNodeId source_rr_node = device_ctx.rr_graph.node_lookup().find_node(layer_num, source_x, source_y, SOURCE, driver_ptc);
 
         VTR_ASSERT(source_rr_node != RRNodeId::INVALID());
-        auto delays = calculate_all_path_delays_from_rr_node(source_rr_node, router_opts, is_flat);
+        auto delays = calculate_all_path_delays_from_rr_node(source_rr_node, router_opts, is_flat, route_profiler.get_net_list());
 
         bool path_to_all_sinks = true;
         for (int sink_x = start_x; sink_x <= end_x; sink_x++) {

vpr/src/route/connection_router.cpp

@@ -239,7 +239,10 @@ t_heap* ConnectionRouter<Heap>::timing_driven_route_connection_from_heap(RRNodeI
             // This is then placed into the traceback so that the correct path is returned
             // TODO: This can be eliminated by modifying the actual traceback function in route_timing
             if (rcv_path_manager.is_enabled()) {
-                rcv_path_manager.insert_backwards_path_into_traceback(cheapest->path_data, cheapest->cost, cheapest->backward_path_cost, route_ctx);
+                rcv_path_manager.insert_backwards_path_into_traceback(cheapest->path_data,
+                                                                      cheapest->cost,
+                                                                      cheapest->backward_path_cost,
+                                                                      route_ctx);
             }
             VTR_LOGV_DEBUG(router_debug_, "  Found target %8d (%s)\n", inode, describe_rr_node(device_ctx.rr_graph, device_ctx.grid, device_ctx.rr_indexed_data, inode, is_flat_).c_str());
             break;
@@ -552,6 +555,10 @@ void ConnectionRouter<Heap>::timing_driven_add_to_heap(const t_conn_cost_params&
     // Costs initialized to current
     next.cost = std::numeric_limits<float>::infinity(); //Not used directly
     next.backward_path_cost = current->backward_path_cost;
+#ifdef PROFILE_LOOKAHEAD
+    next.backward_path_delay = current->backward_path_delay;
+    next.backward_path_congestion = current->backward_path_congestion;
+#endif
 
     // path_data variables are initialized to current values
     if (rcv_path_manager.is_enabled() && current->path_data) {
@@ -597,6 +604,10 @@ void ConnectionRouter<Heap>::timing_driven_add_to_heap(const t_conn_cost_params&
         next_ptr->cost = next.cost;
         next_ptr->R_upstream = next.R_upstream;
         next_ptr->backward_path_cost = next.backward_path_cost;
+#ifdef PROFILE_LOOKAHEAD
+        next_ptr->backward_path_delay = next.backward_path_delay;
+        next_ptr->backward_path_congestion = next.backward_path_congestion;
+#endif
         next_ptr->index = to_node;
         next_ptr->set_prev_edge(from_edge);
 
@@ -784,6 +795,10 @@ void ConnectionRouter<Heap>::evaluate_timing_driven_node_costs(t_heap* to,
     //Update the backward cost (upstream already included)
     to->backward_path_cost += (1. - cost_params.criticality) * cong_cost; //Congestion cost
     to->backward_path_cost += cost_params.criticality * Tdel;             //Delay cost
+#ifdef PROFILE_LOOKAHEAD
+    to->backward_path_delay += Tdel;
+    to->backward_path_congestion += cong_cost;
+#endif
 
     if (cost_params.bend_cost != 0.) {
         t_rr_type from_type = rr_graph_->node_type(from_node);
@@ -832,6 +847,10 @@ void ConnectionRouter<Heap>::empty_heap_annotating_node_route_inf() {
 
         rr_node_route_inf_[tmp->index].path_cost = tmp->cost;
         rr_node_route_inf_[tmp->index].backward_path_cost = tmp->backward_path_cost;
+#ifdef PROFILE_LOOKAHEAD
+        rr_node_route_inf_[tmp->index].backward_path_delay = tmp->backward_path_delay;
+        rr_node_route_inf_[tmp->index].backward_path_congestion = tmp->backward_path_congestion;
+#endif
         modified_rr_node_inf_.push_back(tmp->index);
 
         rcv_path_manager.free_path_struct(tmp->path_data);
@@ -892,6 +911,10 @@ void ConnectionRouter<Heap>::add_route_tree_node_to_heap(
     const auto& device_ctx = g_vpr_ctx.device();
     const RRNodeId inode = rt_node.inode;
     float backward_path_cost = cost_params.criticality * rt_node.Tdel;
+#ifdef PROFILE_LOOKAHEAD
+    float backward_path_delay = rt_node.Tdel;
+    float backward_path_congestion = 0.0;
+#endif
     float R_upstream = rt_node.R_upstream;
 
     /* Don't push to heap if not in bounding box: no-op for serial router, important for parallel router */
@@ -913,9 +936,27 @@ void ConnectionRouter<Heap>::add_route_tree_node_to_heap(
                        tot_cost,
                        describe_rr_node(device_ctx.rr_graph, device_ctx.grid, device_ctx.rr_indexed_data, inode, is_flat_).c_str());
 
-        push_back_node(&heap_, rr_node_route_inf_,
-                       inode, tot_cost, RREdgeId::INVALID(),
-                       backward_path_cost, R_upstream);
+#ifndef PROFILE_LOOKAHEAD
+        push_back_node(&heap_,
+                       rr_node_route_inf_,
+                       inode,
+                       tot_cost,
+                       /*prev_edge=*/RREdgeId::INVALID(),
+                       backward_path_cost,
+                       /*backward_path_delay=*/0.f,
+                       /*backward_path_congestion=*/0.f,
+                       R_upstream);
+#else
+        push_back_node(&heap_,
+                       rr_node_route_inf_,
+                       inode,
+                       tot_cost,
+                       /*prev_edge=*/RREdgeId::INVALID(),
+                       backward_path_cost,
+                       backward_path_delay,
+                       backward_path_congestion,
+                       R_upstream);
+#endif
     } else {
         float expected_total_cost = compute_node_cost_using_rcv(cost_params, inode, target_node, rt_node.Tdel, 0, R_upstream);
 

vpr/src/route/connection_router.h

@@ -2,6 +2,7 @@
 #define _CONNECTION_ROUTER_H
 
 #include "connection_router_interface.h"
+#include "lookahead_profiler.h"
 #include "rr_graph_storage.h"
 #include "route_common.h"
 #include "router_lookahead.h"
@@ -127,6 +128,11 @@ class ConnectionRouter : public ConnectionRouterInterface {
     // Ensure route budgets have been calculated before enabling this
     void set_rcv_enabled(bool enable) final;
 
+    // Get a const reference to the router's lookahead
+    const RouterLookahead& get_router_lookahead() const {
+        return router_lookahead_;
+    }
+
   private:
     // Mark that data associated with rr_node "inode" has been modified, and
     // needs to be reset in reset_path_costs.
@@ -148,6 +154,10 @@ class ConnectionRouter : public ConnectionRouterInterface {
         route_inf->prev_edge = cheapest->prev_edge();
         route_inf->path_cost = cheapest->cost;
         route_inf->backward_path_cost = cheapest->backward_path_cost;
+#ifdef PROFILE_LOOKAHEAD
+        route_inf->backward_path_delay = cheapest->backward_path_delay;
+        route_inf->backward_path_congestion = cheapest->backward_path_congestion;
+#endif
     }
 
     /** Common logic from timing_driven_route_connection_from_route_tree and

vpr/src/route/heap_type.cpp

@@ -27,6 +27,10 @@ HeapStorage::alloc() {
     temp_ptr->set_next_heap_item(nullptr);
     temp_ptr->cost = 0.;
     temp_ptr->backward_path_cost = 0.;
+#ifdef PROFILE_LOOKAHEAD
+    temp_ptr->backward_path_delay = 0.;
+    temp_ptr->backward_path_congestion = 0.;
+#endif
     temp_ptr->R_upstream = 0.;
     temp_ptr->index = RRNodeId::INVALID();
     temp_ptr->path_data = nullptr;

vpr/src/route/heap_type.h

@@ -18,6 +18,15 @@ struct t_heap {
     ///@brief The "known" cost of the path up to and including this node. Used only by the timing-driven router. In this case, the
     ///.cost member contains not only the known backward cost but also an expected cost to the target.
     float backward_path_cost = 0.;
+#ifdef PROFILE_LOOKAHEAD
+    // This data is needed for the LookaheadProfiler, when enabled. It is only conditionally
+    // compiled since there may be many instances of this struct and it is in hot code.
+
+    ///@brief The "known" delay in the path up to and including this node. Recorded for LookaheadProfiler during routing.
+    float backward_path_delay = 0.;
+    ///@brief The "known" congestion in the path up to and including this node. Recorded for LookaheadProfiler during routing.
+    float backward_path_congestion = 0.;
+#endif
     ///@brief Used only by the timing-driven router. Stores the upstream resistance to ground from this node, including the resistance
     /// of the node itself (device_ctx.rr_nodes[index].R).
     float R_upstream = 0.;