crossbar.cpp

//
// Created by root on 22-10-25. //from gpu-sim
//

#include "crossbar.h"
std::vector<int> router::sm_11_to_id;
#include <algorithm>
#include <cmath>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <utility>

xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type,
                         unsigned n_shader, unsigned n_mem,  unsigned gpm_id, unsigned gpm_num,
                         const struct inct_config& m_localinct_config,
                         std::vector<std::queue<Packet>> &in_buffers,
                         std::vector<std::queue<Packet>> &out_buffers,
                         std::vector<std::queue<Packet>> &out_buffers_next,
                         std::vector<std::queue<Packet>> &out_buffers_last,
                         std::vector<std::vector<bool>>  &issued_list) :
        _in_buffers(in_buffers), _out_buffers(out_buffers),
        _out_buffers_next(out_buffers_next), _out_buffers_last(out_buffers_last), _issued_list(issued_list) {
    m_id = router_id;
    router_type = m_type;
    _n_mem = n_mem;
    _n_shader = n_shader;
    total_nodes = n_shader + n_mem;
    _gpm_num = gpm_num;
    _gpm_id = gpm_id;
    gpm_n_shader = _n_shader / _gpm_num;
    gpm_n_mem = _n_mem / _gpm_num;
    in_buffer_limit = m_localinct_config.in_buffer_limit;
    out_buffer_limit = m_localinct_config.out_buffer_limit;
    arbit_type = m_localinct_config.arbiter_algo;
    next_node_id = 0;
    if (m_type == REQ_NET) {
        active_in_buffers = gpm_n_shader;
        active_out_buffers = gpm_n_mem;
    } else if (m_type == REPLY_NET) {
        active_in_buffers = gpm_n_mem;
        active_out_buffers = gpm_n_shader;
    }
    connect_buffer_width = total_nodes;
    make_path_list(gpm_num);
}

xbar_router::~xbar_router() {}

void xbar_router::Push(unsigned input_deviceID, unsigned output_deviceID,
                       void* data, unsigned int size) {
    unsigned rank_in_gpm = (input_deviceID < _n_shader) ? (input_deviceID % gpm_n_shader) :
                           ((input_deviceID - _n_shader) % gpm_n_mem);
    _in_buffers[rank_in_gpm].push(Packet(data, output_deviceID));
}

void* xbar_router::Pop(unsigned output_deviceID) {
    void* data = NULL;
    unsigned rank_in_gpm = (output_deviceID < _n_shader) ? (output_deviceID % gpm_n_shader) :
                           ((output_deviceID - _n_shader) % gpm_n_mem);
    if (!_out_buffers[rank_in_gpm].empty()) {
        data = _out_buffers[rank_in_gpm].front().data;
        _out_buffers[rank_in_gpm].pop();
    }
    return data;
}

bool xbar_router::Has_Buffer_In(unsigned input_deviceID, unsigned size) {
    bool has_buffer =
            (_in_buffers[input_deviceID].size() + size <= in_buffer_limit); // + 1 <= max
    return has_buffer;
}

bool xbar_router::Has_Buffer_Out(int out_buffers_num, unsigned output_deviceID, unsigned size) {
    if (out_buffers_num == 0) {
        return (_out_buffers[output_deviceID].size() + size <= out_buffer_limit);
    }
    else if (out_buffers_num == 1) {
        return (_out_buffers_next[output_deviceID].size() + size <= out_buffer_limit);
    }
    else if (out_buffers_num == 2) {
        return (_out_buffers_last[output_deviceID].size() + size <= out_buffer_limit); //引用类型
    }
    return false;
}

void xbar_router::Advance() {
    if (arbit_type == NAIVE_RR)
        RR_Advance();
    else
       abort();
}

void xbar_bypass_router::Advance() {
    if (arbit_type == NAIVE_RR)
        RR_Forward_Advance();
    else
        abort();
}

void xbar_router::RR_Advance() {
    bool active = false;
    unsigned conflict_sub = 0;
    unsigned reqs = 0;

    for (unsigned i = 0; i < active_in_buffers; ++i) {
        unsigned node_id = (i + next_node_id) % active_in_buffers; // 均匀取数，防止“饿死”

        if (!_in_buffers[node_id].empty()) {
            active = true;
            Packet _packet = _in_buffers[node_id].front();
            make_path(_packet);
            // 这里路径的设置是，已经到了相应的GPM，path中仍然会有一个相等的数
            // 比如，从0到0不用转发，那么路径中也有一个0，这样path.front然后直接判断相等就可以了
            // 又如，从0经过1到2，那么路径中就是{1, 2, 2}，第一个1为了从0转发到1，后面的2为了从1转发到2，最后一个2是判断到地方了用的
//            if (_packet.path.empty()) {};
            int out_buffers_num = 0;
            auto* out_buffers =
                    get_out_buffers(_packet.path.front(), &out_buffers_num);
            _packet.path.pop();
            // get_out_buffers用于判断要到那个out_buffers顺便整个编号
            unsigned id_in_out_buffers =
                    get_id_in_out_buffers(out_buffers_num, _packet.output_deviceID);
            // get_id_in_out_buffers就是判断在这个out_buffers中的编号
            // 例如当前在GPM1, 假设68个SM, 每个GPM分17个
            // 那么如果output_deviceID为19 = 17 + 2, 那么id_in_out_buffers就得是2
            int issued_num = (out_buffers_num == TO_LOCAL && m_id == 0) ? L2 : // m_id的用法来自380行, 初始化的部分
                             (out_buffers_num == TO_LOCAL && m_id == 1) ? SM :
                             (out_buffers_num == TO_NEXT) ? NEXT :
                             (out_buffers_num == TO_LAST) ? LAST : -1;
            // issued_num就是判断这个out_buffers对应的是_issued_list那四个中的哪个
            // 因为_issued_list在LocalInterconnect里，是传的引用进来
            if (Has_Buffer_Out(out_buffers_num, id_in_out_buffers, 1)) {
                if (!_issued_list[issued_num][id_in_out_buffers]) {
                    (*out_buffers)[id_in_out_buffers].push(_packet);
                    _in_buffers[node_id].pop();
                    _issued_list[issued_num][id_in_out_buffers] = true;
                    reqs++;
                } else
                    conflict_sub++;
            } else {
                if (_issued_list[issued_num][id_in_out_buffers]) conflict_sub++;
            }
        }
    }
    next_node_id = (++next_node_id % active_in_buffers);
}

bool xbar_router::Busy() const {
    // 这个是LocalInterconnect::Busy()调用的，
    // 但那个函数根本没被用过...就按照原来的思路随便改了下，不确定
    for (unsigned i = 0; i < gpm_n_shader; ++i) {
        if (!_in_buffers[i].empty()) return true;
    }
    for (unsigned i = 0; i < gpm_n_mem; ++i) {
        if (!_out_buffers[i].empty()) return true;
    }
    return false;
}

int xbar_router::get_dst_GPM(unsigned output_deviceID) {
    unsigned gpm_rank;
    if (output_deviceID < _n_shader)
    {
        //subnet = 0;
        gpm_rank = (output_deviceID / gpm_n_shader);
    } else {
        //subnet = 1;
        gpm_rank = ((output_deviceID - _n_shader) / gpm_n_mem);
    }
    return gpm_rank;
}

void xbar_router::make_path(Packet &packet) {
    int dst_gpm_id = get_dst_GPM(packet.output_deviceID);
    int path_len = path_list[_gpm_id][dst_gpm_id].size();
    while (path_len--) {
        packet.path.push(path_list[_gpm_id][dst_gpm_id][path_len]);
    }
}

std::vector<std::queue<Packet>>* xbar_router::get_out_buffers(int next_gpm_id, int* num) {

    if (_gpm_id == next_gpm_id) {
        *num = TO_LOCAL;
        return (std::vector<std::queue<Packet>>*) &_out_buffers;
    }
    else if ((next_gpm_id + _gpm_num- _gpm_id) % _gpm_num == 1) { // to next
        *num = TO_NEXT;
        return (std::vector<std::queue<Packet>>*) &_out_buffers_next;
    }
    else if ((_gpm_id + _gpm_num - next_gpm_id) % _gpm_num == 1) { // to last
        *num = TO_LAST;
        return (std::vector<std::queue<Packet>>*) &_out_buffers_last;
    }
    else
    {
        abort(); // 报错
    }
}

unsigned xbar_router::get_id_in_out_buffers(unsigned out_buffers_num, unsigned output_deviceID) {

    if (out_buffers_num == 0) {
        if (output_deviceID < _n_shader) {
            return output_deviceID % gpm_n_shader;
        }
        else {
            return (output_deviceID - _n_shader) % gpm_n_mem;
        }
    }
    else {
        return output_deviceID;
    }
}

void xbar_router::make_path_list(int n) {
    path_list.resize(n, std::vector<std::vector<int>>(n, std::vector<int>()));
    for (int i = 0; i < n; ++i)
    {
        for (int j = i; j < n; ++j)
        {
            if (((j - i) << 1) < n) //fix
            {
                path_list[i][j].emplace_back(j);
                for (int k = j; k > i; --k)
                {
                    path_list[i][j].emplace_back(k);
                }
                if (i != j)
                    path_list[j][i].emplace_back(i);
                for (int k = i; k < j; ++k)
                {
                    path_list[j][i].emplace_back(k);
                }
            }
            else if (((j - i) << 1) == n)
            {
                path_list[i][j].emplace_back(j);
                for (int k = j; k > i; --k)
                {
                    path_list[i][j].emplace_back(k);
                }
                if (i != j)
                    path_list[j][i].emplace_back(i);
                for (int k = j + (n - (j - i)); k > j; --k)
                {
                    path_list[j][i].emplace_back(k % n);
                }
            }
            else
            {
                path_list[i][j].emplace_back(j);
                for (int k = j; k < j + (n - (j - i)); ++k)
                {
                    path_list[i][j].emplace_back(k % n);
                }
                if (i != j)
                    path_list[j][i].emplace_back(i);
                for (int k = j + (n - (j - i)); k > j; --k)
                {
                    path_list[j][i].emplace_back(k % n);
                }
            }
        }
    } // 可以巧妙用递归反序输出
}

xbar_bypass_router::xbar_bypass_router(unsigned router_id, enum Interconnect_type m_type,
                         unsigned n_shader, unsigned n_mem,  unsigned gpm_id, unsigned gpm_num,
                         const struct inct_config& m_localinct_config,
                         std::vector<std::queue<Packet>> &in_buffers,
                         std::vector<std::queue<Packet>> &out_buffers_to_sm,
                         std::vector<std::queue<Packet>> &out_buffers_to_l2,
                         std::vector<std::queue<Packet>> &out_buffers_bybass,
                         std::vector<std::vector<bool>>  &issued_list) :
        _in_buffers(in_buffers), _out_buffers_to_sm(out_buffers_to_sm),
        _out_buffers_to_l2(out_buffers_to_l2), _out_buffers_bybass(out_buffers_bybass), _issued_list(issued_list) {
    m_id = router_id;
    router_type = m_type;
    _n_mem = n_mem;
    _n_shader = n_shader;
    total_nodes = n_shader + n_mem;
    _gpm_num = gpm_num;
    _gpm_id = gpm_id;
    gpm_n_shader = _n_shader / _gpm_num;
    gpm_n_mem = _n_mem / _gpm_num;
    in_buffer_limit = m_localinct_config.in_buffer_limit;
    out_buffer_limit = m_localinct_config.out_buffer_limit;
    arbit_type = m_localinct_config.arbiter_algo;
    next_node_id = 0;
    active_in_buffers = gpm_n_mem;
    connect_buffer_width = total_nodes;
}

xbar_bypass_router::~xbar_bypass_router() {}

bool xbar_bypass_router::Has_Buffer_Out(int out_buffers_num, unsigned output_deviceID, unsigned size) {
    if (out_buffers_num == 0) {
        return (_out_buffers_to_sm[output_deviceID].size() + size <= out_buffer_limit);
    }
    else if (out_buffers_num == 1) {
        return (_out_buffers_to_l2[output_deviceID].size() + size <= out_buffer_limit);
    }
    else if (out_buffers_num == 2) {
        return (_out_buffers_bybass[output_deviceID].size() + size <= out_buffer_limit); //引用类型
    }
    return false;
}

void xbar_bypass_router::RR_Forward_Advance() {
    bool active = false;
    unsigned conflict_sub = 0;
    unsigned reqs = 0;

    for (unsigned i = 0; i < connect_buffer_width; ++i) {
        unsigned node_id = (i + next_node_id) % connect_buffer_width;

        if (!_in_buffers[node_id].empty()) {
            active = true;
            Packet _packet = _in_buffers[node_id].front(); // 这都是转发过来的

//            if (_packet.path.empty()) {};
            int out_buffers_num = 0;
            auto* out_buffers = get_out_buffers(_packet.output_deviceID,
                                                _packet.path.front(), &out_buffers_num);
            _packet.path.pop();

            unsigned id_in_out_buffers =
                    get_id_in_out_buffers(out_buffers_num, _packet.output_deviceID);

            int issued_num = (out_buffers_num == TO_SM) ? SM :
                             (out_buffers_num == TO_L2) ? L2 :
                             (out_buffers_num == TO_BYPASS && m_id == 2) ? NEXT : // m_id的用法来自380行, 初始化的部分
                             (out_buffers_num == TO_BYPASS && m_id == 3) ? LAST : -1;

            if (Has_Buffer_Out(out_buffers_num, id_in_out_buffers, 1)) {
                if (!_issued_list[issued_num][id_in_out_buffers]) {
                    (*out_buffers)[id_in_out_buffers].push(_packet);
                    _in_buffers[node_id].pop();
                    _issued_list[issued_num][id_in_out_buffers] = true;
                    reqs++;
                } else
                    conflict_sub++;
            } else {
                if (_issued_list[issued_num][id_in_out_buffers]) conflict_sub++;
            }
        }
    }
    next_node_id = (++next_node_id % connect_buffer_width);
}

std::vector<std::queue<Packet>>* xbar_bypass_router::get_out_buffers(unsigned output_deviceID, int next_gpm_id, int* num) {

    if (_gpm_id == next_gpm_id) { // 到地方了
        if (output_deviceID < _n_shader) {
            *num = TO_SM;
            return (std::vector<std::queue<Packet>>*) &_out_buffers_to_sm;
        }
        else {
            *num = TO_L2;
            return (std::vector<std::queue<Packet>>*) &_out_buffers_to_l2;
        }
    }
    else { // 要转发
        *num = TO_BYPASS;
        return (std::vector<std::queue<Packet>>*) &_out_buffers_bybass;
    }
}

unsigned xbar_bypass_router::get_id_in_out_buffers(unsigned out_buffers_num, unsigned output_deviceID) {
    if (out_buffers_num == 0) { //确定 in / out buffers 的 id
        return output_deviceID % gpm_n_shader;
    }
    if (out_buffers_num == 1) {
        return (output_deviceID - _n_shader) % gpm_n_mem;
    }
    if (out_buffers_num == 2) {
        return output_deviceID;
    }
    else {
        return -1;
    }
}

// assume all the packets are one flit
#define LOCAL_INCT_FLIT_SIZE 40

LocalInterconnect* LocalInterconnect::New(
        const struct inct_config& m_localinct_config) {
    auto* icnt_interface = new LocalInterconnect(m_localinct_config);

    return icnt_interface;
}

LocalInterconnect::LocalInterconnect(
        const struct inct_config& m_localinct_config)
        : m_inct_config(m_localinct_config) {
    n_shader = 0;
    n_mem = 0;
    gpm_num = 0;
    status = 0;
    n_subnets = m_localinct_config.subnets;
}

LocalInterconnect::~LocalInterconnect() {
    for (unsigned i = 0; i < m_inct_config.subnets; ++i) {
        delete net[i];
    }
}

void LocalInterconnect::CreateInterconnect(unsigned m_n_shader,
                                           unsigned m_n_mem, unsigned m_gpm_id, unsigned m_gpm_num) {
    n_shader = m_n_shader;
    n_mem = m_n_mem;
    gpm_num = m_gpm_num;
    gpm_id = m_gpm_id;
    gpm_n_shader = m_n_shader / m_gpm_num;
    gpm_n_mem = m_n_mem / m_gpm_num;
    connect_buffer_width = n_shader + n_mem; // gpm_n_shader + gpm_n_mem;
    CreateBuffers();
    issued_list.clear();
    for (unsigned i = 0; i < 4; ++i) {
        unsigned m_total_nodes = (i == 0) ? n_shader / gpm_num :
                                 (i == 1) ? n_mem / gpm_num :
                                 connect_buffer_width;
        std::vector<bool> (m_issued)(m_total_nodes, false);
        issued_list.push_back(m_issued);
    }
    net.resize(n_subnets);
    net[0] = new xbar_router(0, static_cast<Interconnect_type>(0), m_n_shader,
                             m_n_mem, gpm_id, gpm_num, m_inct_config,
                             in_buffers_list[0],
                             out_buffers_list[1], out_buffers_list[3], out_buffers_list[2],
                             issued_list);
    net[1] = new xbar_router(1, static_cast<Interconnect_type>(1), m_n_shader,
                             m_n_mem, gpm_id, gpm_num, m_inct_config,
                             in_buffers_list[1],
                             out_buffers_list[0], out_buffers_list[3], out_buffers_list[2],
                             issued_list);
    bypass_net.resize(2);
    bypass_net[0] = new xbar_bypass_router(2, static_cast<Interconnect_type>(2),
                             m_n_shader, m_n_mem, gpm_id, gpm_num,
                             m_inct_config, in_buffers_list[2],
                             out_buffers_list[0], out_buffers_list[1], out_buffers_list[3],
                             issued_list);
    bypass_net[1] = new xbar_bypass_router(3, static_cast<Interconnect_type>(2),
                             m_n_shader, m_n_mem, gpm_id, gpm_num,
                             m_inct_config, in_buffers_list[3],
                             out_buffers_list[0], out_buffers_list[1], out_buffers_list[2],
                             issued_list);
}

void LocalInterconnect::CreateBuffers() {
    in_buffers_list.clear();
    std::vector<std::queue<Packet>> in_buffers_0;
    in_buffers_0.resize(n_shader/gpm_num);
    in_buffers_list.push_back(in_buffers_0);
    std::vector<std::queue<Packet>> in_buffers_1;
    in_buffers_1.resize(n_mem/gpm_num);
    in_buffers_list.push_back(in_buffers_1);
    std::vector<std::queue<Packet>> in_buffers_2;
    in_buffers_2.resize(connect_buffer_width);
    in_buffers_list.push_back(in_buffers_2);
    std::vector<std::queue<Packet>> in_buffers_3;
    in_buffers_3.resize(connect_buffer_width);
    in_buffers_list.push_back(in_buffers_3);

    out_buffers_list.clear();
    std::vector<std::queue<Packet>> out_buffers_0;
    out_buffers_0.resize(n_shader/gpm_num);
    out_buffers_list.push_back(out_buffers_0);
    std::vector<std::queue<Packet>> out_buffers_1;
    out_buffers_1.resize(n_mem/gpm_num);
    out_buffers_list.push_back(out_buffers_1);
    std::vector<std::queue<Packet>> out_buffers_2;
    out_buffers_2.resize(connect_buffer_width);
    out_buffers_list.push_back(out_buffers_2);
    std::vector<std::queue<Packet>> out_buffers_3;
    out_buffers_3.resize(connect_buffer_width);
    out_buffers_list.push_back(out_buffers_3);
}

void LocalInterconnect::Init() {
    // empty
    // there is nothing to do
}

void LocalInterconnect::Push(unsigned input_deviceID, unsigned output_deviceID,
                             void* data, unsigned int size) {
    unsigned subnet;
    if (n_subnets == 1) {
        subnet = 0;
    } else {
        if (input_deviceID < n_shader) {
            subnet = 0;
        } else {
            subnet = 1;
        }
    }
    net[subnet]->Push(input_deviceID, output_deviceID, data, size);
}

void* LocalInterconnect::Pop(unsigned output_deviceID) {
    // 0-_n_shader-1 indicates reply(network 1), otherwise request(network 0)
    int subnet = 0;
    if (output_deviceID < n_shader) subnet = 1; //处理模式识别

    return net[subnet]->Pop(output_deviceID);
}

void LocalInterconnect::Advance() {
    for (auto& issued : issued_list) {
        unsigned len = issued.size();
        for (int i = 0; i < len; ++i) {
            issued[i] = false;
        }
    }
    if (status == 0) { // 这里做了小优化, 一轮SM、L2的in_buffers先来, 下一轮就转发的先来。这样普遍会快一点点, 不到百分之一(
        for (unsigned i = 0; i < n_subnets; ++i) {
            net[i]->Advance();
        }
        for (unsigned i = 0; i < 2; ++i) {
            bypass_net[i]->Advance();
        }
    }
    else if (status == 1) {
        for (unsigned i = 0; i < 2; ++i) {
            bypass_net[i]->Advance();
        }
        for (unsigned i = 0; i < n_subnets; ++i) {
            net[i]->Advance();
        }
    }
    status ^= 1;
}

bool LocalInterconnect::Busy() const { // todo 这个没用到就没改（
    for (unsigned i = 0; i < n_subnets; ++i) {
        if (net[i]->Busy()) return true;
    }
    return false;
}

bool LocalInterconnect::HasBuffer(unsigned deviceID, unsigned int size) const {
    bool has_buffer = false;
    if ((n_subnets > 1) && deviceID >= n_shader)  // deviceID is memory node
        has_buffer = net[REPLY_NET]->Has_Buffer_In((deviceID-n_shader) % gpm_n_mem, 1);
    else
        has_buffer = net[REQ_NET]->Has_Buffer_In(deviceID % gpm_n_shader, 1); // todo 为啥不传size却传1 ?
    return has_buffer;
}

bool LocalInterconnect::in_buffers_list_full(int buffer_rank, int node_rank, int buffer_size) //判断是否已满,是返回true
{
    return in_buffers_list[buffer_rank][node_rank].size() >= buffer_size;
}
void LocalInterconnect::in_buffers_list_push(int buffer_rank, int node_rank, const Packet& cnt)
{
    in_buffers_list[buffer_rank][node_rank].push(cnt);
}
void LocalInterconnect::in_buffers_list_pop(int buffer_rank, int node_rank)
{
    return in_buffers_list[buffer_rank][node_rank].pop();
}

Packet LocalInterconnect::in_buffers_list_probe(int buffer_rank, int node_rank)
{
    return in_buffers_list[buffer_rank][node_rank].front();
}

bool LocalInterconnect::in_buffers_list_empty(int buffer_rank, int node_rank) //判断是否为空,空则返回true
{
    return in_buffers_list[buffer_rank][node_rank].empty();
}

bool LocalInterconnect::out_buffers_list_full(int buffer_rank, int node_rank, int buffer_size)
{
    return out_buffers_list[buffer_rank][node_rank].size() >= buffer_size;
}
void LocalInterconnect::out_buffers_list_push(int buffer_rank, int node_rank, const Packet& cnt)
{
    out_buffers_list[buffer_rank][node_rank].push(cnt);
}
void LocalInterconnect::out_buffers_list_pop(int buffer_rank, int node_rank)
{
    return out_buffers_list[buffer_rank][node_rank].pop();
}
Packet LocalInterconnect::out_buffers_list_probe(int buffer_rank, int node_rank)
{
    return out_buffers_list[buffer_rank][node_rank].front();
}
bool LocalInterconnect::out_buffers_list_empty(int buffer_rank, int node_rank)
{
    return out_buffers_list[buffer_rank][node_rank].empty();
}

void MCM_icnt::Push(unsigned int input_deviceID, unsigned int output_deviceID, void *data, unsigned int size) {

    unsigned gpm_rank; //判断在0123哪个GPM中
    if (input_deviceID < n_shader)
    {
        //subnet = 0;
        gpm_rank = (input_deviceID / (n_shader / gpm_num));
    } else {
        //subnet = 1;
        gpm_rank = ((input_deviceID - n_shader) / (n_mem / gpm_num));
    }
    m_icnts[gpm_rank]->Push(input_deviceID, output_deviceID, data, size);
}

void* MCM_icnt::Pop(unsigned int output_deviceID) {
    unsigned gpm_rank;
    if (output_deviceID < n_shader)
    {
        //subnet = 0;
        gpm_rank = (output_deviceID / (n_shader / gpm_num));
    } else {
        //subnet = 1;
        gpm_rank = ((output_deviceID - n_shader) / (n_mem / gpm_num));
    }
    return m_icnts[gpm_rank]->Pop(output_deviceID);
}

bool MCM_icnt::HasBuffer(unsigned int deviceID, unsigned int size) const {
    return m_icnts[get_gpm_num(deviceID)]->HasBuffer(deviceID, size); //是否还有剩余空间
}

unsigned MCM_icnt::get_gpm_num (unsigned int output_deviceID) const {
    unsigned gpm_rank;
    if (output_deviceID < n_shader)
    {
        //subnet = 0;
        gpm_rank = (output_deviceID / gpm_n_shader);
    } else {
        //subnet = 1;
        gpm_rank = ((output_deviceID - n_shader) / gpm_n_mem);
    }
    return gpm_rank;
}

void MCM_icnt::icnt_connect_cycle() {
    for (int i = 0; i < gpm_num; i++)
    {
        m_icnt_latency_queues[0][i]->crossbar_latency_queue_cycle(*m_icnts[i], *m_icnts[(i + 1) % gpm_num], 3, 2);
        m_icnt_latency_queues[1][i]->crossbar_latency_queue_cycle(*m_icnts[(i + 1) % gpm_num], *m_icnts[i], 2, 3);
        // in out 互换,位置互换
        // 0表示顺时针方向,
        // 1表示逆时针方向
    }
}

void MCM_icnt::icnt_cycle() {
    for (int i = 0; i < gpm_num; i++)
    {
        m_icnts[i]->Advance(); //先内部运算
    }
    icnt_connect_cycle(); //再传输数据
}