bigger-dram_smaller-nvram

inc/cache.h

@@ -104,8 +104,16 @@ class CACHE : public MEMORY {
              pf_fill;
 
     // queues
+    // PACKET_QUEUE WQ0{NAME + "_WQ0", WQ_SIZE}, // write0 queue
+    //              WQ1{NAME + "_WQ1", WQ_SIZE}, // write1 queue
+
+    //              RQ0{NAME + "_RQ0", RQ_SIZE}, // read0 queue
+    //              RQ1{NAME + "_RQ1", RQ_SIZE}, // read1 queue
+
     PACKET_QUEUE WQ{NAME + "_WQ", WQ_SIZE}, // write queue
-                 RQ{NAME + "_RQ", RQ_SIZE}, // read queue
+                 RQ{NAME + "_RQ", RQ_SIZE}, // read0 queue
+
+
                  PQ{NAME + "_PQ", PQ_SIZE}, // prefetch queue
                  MSHR{NAME + "_MSHR", MSHR_SIZE}, // MSHR
                  PROCESSED{NAME + "_PROCESSED", ROB_SIZE}; // processed queue
@@ -213,6 +221,10 @@ class CACHE : public MEMORY {
     };
 
     // functions
+    // int CACHE::sub_add_rq(PACKET_QUEUE read_queue, PACKET_QUEUE write_queue, PACKET *packet);
+    // int CACHE::sub_add_wq(PACKET_QUEUE read_queue, PACKET_QUEUE write_queue, PACKET *packet);
+    // int CACHE::sub_add_pq(PACKET_QUEUE read_queue, PACKET_QUEUE write_queue, PACKET *packet);
+
     int  add_rq(PACKET *packet),
          add_wq(PACKET *packet),
          add_pq(PACKET *packet);

inc/champsim.h

@@ -36,6 +36,7 @@
 #define NUM_CPUS 1
 #define CPU_FREQ 4000
 #define DRAM_IO_FREQ 2400 // DDR4-2400
+#define NVRAM_IO_FREQ 1600 // DDR4-2400
 #define PAGE_SIZE 4096
 #define LOG2_PAGE_SIZE 12
 
@@ -53,12 +54,12 @@
 #define FILL_LLC   4
 #define FILL_DRC   8
 #define FILL_DRAM 16
-
+#define FILL_NVRAM 16
 // DRAM
 #define DRAM_CHANNELS 1      // default: assuming one DIMM per one channel 4GB * 1 => 4GB off-chip memory
 #define LOG2_DRAM_CHANNELS 0
-#define DRAM_RANKS 1         // 512MB * 8 ranks => 4GB per DIMM
-#define LOG2_DRAM_RANKS 0
+#define DRAM_RANKS 4         // 512MB * 8 ranks => 4GB per DIMM
+#define LOG2_DRAM_RANKS 2
 #define DRAM_BANKS 8         // 64MB * 8 banks => 512MB per rank
 #define LOG2_DRAM_BANKS 3
 #define DRAM_ROWS 65536      // 2KB * 32K rows => 64MB per bank
@@ -69,6 +70,22 @@
 
 #define DRAM_SIZE (DRAM_CHANNELS*DRAM_RANKS*DRAM_BANKS*DRAM_ROWS*DRAM_ROW_SIZE/1024) 
 #define DRAM_PAGES ((DRAM_SIZE<<10)>>2) 
+
+// NVRAM ///////////////////////////////////////////////
+#define NVRAM_CHANNELS 1      // default: assuming one DIMM per one channel 4GB * 1 => 4GB off-chip memory
+#define LOG2_NVRAM_CHANNELS 0
+#define NVRAM_RANKS 1        // 512MB * 8 ranks => 4GB per DIMM
+#define LOG2_NVRAM_RANKS 0
+#define NVRAM_BANKS 8         // 64MB * 8 banks => 512MB per rank
+#define LOG2_NVRAM_BANKS 3
+#define NVRAM_ROWS 65536      // 2KB * 32K rows => 64MB per bank
+#define LOG2_NVRAM_ROWS 16
+#define NVRAM_COLUMNS 128      // 64B * 32 column chunks (Assuming 1B DRAM cell * 8 chips * 8 transactions = 64B size of column chunks) => 2KB per row
+#define LOG2_NVRAM_COLUMNS 7
+#define NVRAM_ROW_SIZE (BLOCK_SIZE*NVRAM_COLUMNS/1024)
+
+#define NVRAM_SIZE (NVRAM_CHANNELS*NVRAM_RANKS*NVRAM_BANKS*NVRAM_ROWS*NVRAM_ROW_SIZE/1024) 
+#define NVRAM_PAGES ((NVRAM_SIZE<<10)>>2) 
 //#define DRAM_PAGES 10
 
 using namespace std;

inc/champsim.h.bak

@@ -36,6 +36,7 @@
 #define NUM_CPUS 4
 #define CPU_FREQ 4000
 #define DRAM_IO_FREQ 2400 // DDR4-2400
+#define NVRAM_IO_FREQ 1600 // DDR4-2400
 #define PAGE_SIZE 4096
 #define LOG2_PAGE_SIZE 12
 
@@ -53,12 +54,12 @@
 #define FILL_LLC   4
 #define FILL_DRC   8
 #define FILL_DRAM 16
-
+#define FILL_NVRAM 16
 // DRAM
 #define DRAM_CHANNELS 1      // default: assuming one DIMM per one channel 4GB * 1 => 4GB off-chip memory
 #define LOG2_DRAM_CHANNELS 0
-#define DRAM_RANKS 1         // 512MB * 8 ranks => 4GB per DIMM
-#define LOG2_DRAM_RANKS 0
+#define DRAM_RANKS 4         // 512MB * 8 ranks => 4GB per DIMM
+#define LOG2_DRAM_RANKS 2
 #define DRAM_BANKS 8         // 64MB * 8 banks => 512MB per rank
 #define LOG2_DRAM_BANKS 3
 #define DRAM_ROWS 65536      // 2KB * 32K rows => 64MB per bank
@@ -69,6 +70,22 @@
 
 #define DRAM_SIZE (DRAM_CHANNELS*DRAM_RANKS*DRAM_BANKS*DRAM_ROWS*DRAM_ROW_SIZE/1024) 
 #define DRAM_PAGES ((DRAM_SIZE<<10)>>2) 
+
+// NVRAM ///////////////////////////////////////////////
+#define NVRAM_CHANNELS 1      // default: assuming one DIMM per one channel 4GB * 1 => 4GB off-chip memory
+#define LOG2_NVRAM_CHANNELS 0
+#define NVRAM_RANKS 1        // 512MB * 8 ranks => 4GB per DIMM
+#define LOG2_NVRAM_RANKS 0
+#define NVRAM_BANKS 8         // 64MB * 8 banks => 512MB per rank
+#define LOG2_NVRAM_BANKS 3
+#define NVRAM_ROWS 65536      // 2KB * 32K rows => 64MB per bank
+#define LOG2_NVRAM_ROWS 16
+#define NVRAM_COLUMNS 128      // 64B * 32 column chunks (Assuming 1B DRAM cell * 8 chips * 8 transactions = 64B size of column chunks) => 2KB per row
+#define LOG2_NVRAM_COLUMNS 7
+#define NVRAM_ROW_SIZE (BLOCK_SIZE*NVRAM_COLUMNS/1024)
+
+#define NVRAM_SIZE (NVRAM_CHANNELS*NVRAM_RANKS*NVRAM_BANKS*NVRAM_ROWS*NVRAM_ROW_SIZE/1024) 
+#define NVRAM_PAGES ((NVRAM_SIZE<<10)>>2) 
 //#define DRAM_PAGES 10
 
 using namespace std;

inc/dram_controller.h

@@ -12,6 +12,7 @@
 #define tRCD_DRAM_NANOSECONDS 15
 #define tCAS_DRAM_NANOSECONDS 12.5
 
+
 // the data bus must wait this amount of time when switching between reads and writes, and vice versa
 #define DRAM_DBUS_TURN_AROUND_TIME ((15*CPU_FREQ)/2000) // 7.5 ns 
 extern uint32_t DRAM_MTPS, DRAM_DBUS_RETURN_TIME;
@@ -21,7 +22,119 @@ extern uint32_t DRAM_MTPS, DRAM_DBUS_RETURN_TIME;
 #define DRAM_WRITE_LOW_WM     ((DRAM_WQ_SIZE*3)>>2) // 6/8th
 #define MIN_DRAM_WRITES_PER_SWITCH (DRAM_WQ_SIZE*1/4)
 
-void print_dram_config();
+
+
+
+/////////////// NVRAM Config//////////////////////////.
+#define NVRAM_CHANNEL_WIDTH 8 // 8B
+#define NVRAM_WQ_SIZE 128
+#define NVRAM_RQ_SIZE 128
+
+#define tRP_NVRAM_NANOSECONDS 22
+#define tRCD_NVRAM_NANOSECONDS 22
+#define tCAS_NVRAM_NANOSECONDS 15
+
+
+// the data bus must wait this amount of time when switching between reads and writes, and vice versa
+#define NVRAM_DBUS_TURN_AROUND_TIME ((18*CPU_FREQ)/2000) // ?? ns 
+extern uint32_t NVRAM_MTPS, NVRAM_DBUS_RETURN_TIME;
+
+// these values control when to send out a burst of writes
+#define NVRAM_WRITE_HIGH_WM    ((NVRAM_WQ_SIZE*7)>>3) // 7/8th
+#define NVRAM_WRITE_LOW_WM     ((NVRAM_WQ_SIZE*3)>>2) // 6/8th
+#define MIN_NVRAM_WRITES_PER_SWITCH (NVRAM_WQ_SIZE*1/4)
+
+
+void print_dram_config();               
+void print_NVram_config();
+
+
+//////////////////   NVRAM  /////////////////////////////////////
+
+class MEMORY_CONTROLLER_NV : public MEMORY {
+  public:
+    const string NAME;
+
+    DRAM_ARRAY NVram_array[NVRAM_CHANNELS][NVRAM_RANKS][NVRAM_BANKS];
+    uint64_t dbus_cycle_available[NVRAM_CHANNELS], dbus_cycle_congested[NVRAM_CHANNELS], dbus_congested[NUM_TYPES+1][NUM_TYPES+1];
+    uint64_t bank_cycle_available[NVRAM_CHANNELS][NVRAM_RANKS][NVRAM_BANKS];
+    uint8_t  do_write, write_mode[NVRAM_CHANNELS]; 
+    uint32_t processed_writes, scheduled_reads[NVRAM_CHANNELS], scheduled_writes[NVRAM_CHANNELS];
+    int fill_level;
+
+    BANK_REQUEST bank_request[NVRAM_CHANNELS][NVRAM_RANKS][NVRAM_BANKS];
+
+    // queues
+    PACKET_QUEUE WQ[NVRAM_CHANNELS], RQ[NVRAM_CHANNELS];
+
+    // constructor
+    MEMORY_CONTROLLER_NV(string v1) : NAME (v1) {
+        for (uint32_t i=0; i<NUM_TYPES+1; i++) {
+            for (uint32_t j=0; j<NUM_TYPES+1; j++) {
+                dbus_congested[i][j] = 0;
+            }
+        }
+        do_write = 0;
+        processed_writes = 0;
+        for (uint32_t i=0; i<NVRAM_CHANNELS; i++) {
+            dbus_cycle_available[i] = 0;
+            dbus_cycle_congested[i] = 0;
+            write_mode[i] = 0;
+            scheduled_reads[i] = 0;
+            scheduled_writes[i] = 0;
+
+            for (uint32_t j=0; j<NVRAM_RANKS; j++) {
+                for (uint32_t k=0; k<NVRAM_BANKS; k++)
+                    bank_cycle_available[i][j][k] = 0;
+            }
+
+            WQ[i].NAME = "NVRAM_WQ" + to_string(i);
+            WQ[i].SIZE = NVRAM_WQ_SIZE;
+            WQ[i].entry = new PACKET [NVRAM_WQ_SIZE];
+
+            RQ[i].NAME = "NVRAM_RQ" + to_string(i);
+            RQ[i].SIZE = NVRAM_RQ_SIZE;
+            RQ[i].entry = new PACKET [NVRAM_RQ_SIZE];
+        }
+
+        fill_level = FILL_NVRAM;
+    };
+
+    // destructor
+    ~MEMORY_CONTROLLER_NV() {
+
+    };
+
+    // functions
+    int  add_rq(PACKET *packet),
+         add_wq(PACKET *packet),
+         add_pq(PACKET *packet);
+
+    void return_data(PACKET *packet),
+         operate(),
+         increment_WQ_FULL(uint64_t address);
+
+    uint32_t get_occupancy(uint8_t queue_type, uint64_t address),
+             get_size(uint8_t queue_type, uint64_t address);
+
+    void schedule(PACKET_QUEUE *queue), process(PACKET_QUEUE *queue),
+         update_schedule_cycle(PACKET_QUEUE *queue),
+         update_process_cycle(PACKET_QUEUE *queue),
+         reset_remain_requests(PACKET_QUEUE *queue, uint32_t channel);
+
+    uint32_t NVram_get_channel(uint64_t address),
+             NVram_get_rank   (uint64_t address),
+             NVram_get_bank   (uint64_t address),
+             NVram_get_row    (uint64_t address),
+             NVram_get_column (uint64_t address),
+             drc_check_hit (uint64_t address, uint32_t cpu, uint32_t channel, uint32_t rank, uint32_t bank, uint32_t row);
+
+    uint64_t get_bank_earliest_cycle();
+
+    int check_NVram_queue(PACKET_QUEUE *queue, PACKET *packet);
+};
+
+
 
 // DRAM
 class MEMORY_CONTROLLER : public MEMORY {
@@ -106,5 +219,4 @@ class MEMORY_CONTROLLER : public MEMORY {
 
     int check_dram_queue(PACKET_QUEUE *queue, PACKET *packet);
 };
-
 #endif

inc/memory_class.h

@@ -14,13 +14,16 @@
 extern uint32_t tRP,  // Row Precharge (RP) latency
                 tRCD, // Row address to Column address (RCD) latency
                 tCAS; // Column Address Strobe (CAS) latency
+extern uint32_t NV_tRP,  // Row Precharge (RP) latency
+                NV_tRCD, // Row address to Column address (RCD) latency
+                NV_tCAS; // Column Address Strobe (CAS) latency
 
 extern uint64_t l2pf_access;
 
 class MEMORY {
   public:
     // memory interface
-    MEMORY *upper_level_icache[NUM_CPUS], *upper_level_dcache[NUM_CPUS], *lower_level, *extra_interface;
+    MEMORY *upper_level_icache[NUM_CPUS], *upper_level_dcache[NUM_CPUS], *lower_level, *extra_interface, *lower_level_second;
 
     // empty queues
     PACKET_QUEUE WQ{"EMPTY", 1}, RQ{"EMPTY", 1}, PQ{"EMPTY", 1}, MSHR{"EMPTY", 1};
@@ -38,7 +41,7 @@ class MEMORY {
     // stats
     uint64_t ACCESS[NUM_TYPES], HIT[NUM_TYPES], MISS[NUM_TYPES], MSHR_MERGED[NUM_TYPES], STALL[NUM_TYPES];
 
-    MEMORY() {
+    MEMORY(): lower_level(NULL) {
         for (uint32_t i=0; i<NUM_TYPES; i++) {
             ACCESS[i] = 0;
             HIT[i] = 0;

inc/uncore.h

@@ -15,8 +15,10 @@ class UNCORE {
     // LLC
     CACHE LLC{"LLC", LLC_SET, LLC_WAY, LLC_SET*LLC_WAY, LLC_WQ_SIZE, LLC_RQ_SIZE, LLC_PQ_SIZE, LLC_MSHR_SIZE};
 
-    // DRAM
-    MEMORY_CONTROLLER DRAM{"DRAM"}; 
+    // DRAM_0
+    MEMORY_CONTROLLER DRAM0{"DRAM0"}; 
+  // DRAM_1
+    MEMORY_CONTROLLER_NV DRAM1{"DRAM1"}; 
 
     UNCORE(); 
 };

src/dram_controller.cc

@@ -176,19 +176,19 @@ void MEMORY_CONTROLLER_NV::operate()
             // reset scheduled RQ requests
             reset_remain_requests(&RQ[i], i);
             // add data bus turn-around time
-            dbus_cycle_available[i] += DRAM_DBUS_TURN_AROUND_TIME;
+            dbus_cycle_available[i] += NVRAM_DBUS_TURN_AROUND_TIME;
         } else if (write_mode[i]) {
 
             if (WQ[i].occupancy == 0)
                 write_mode[i] = 0;
-            else if (RQ[i].occupancy && (WQ[i].occupancy < DRAM_WRITE_LOW_WM))
+            else if (RQ[i].occupancy && (WQ[i].occupancy < NVRAM_WRITE_LOW_WM))
                 write_mode[i] = 0;
 
             if (write_mode[i] == 0) {
                 // reset scheduled WQ requests
                 reset_remain_requests(&WQ[i], i);
                 // add data bus turnaround time
-                dbus_cycle_available[i] += DRAM_DBUS_TURN_AROUND_TIME;
+                dbus_cycle_available[i] += NVRAM_DBUS_TURN_AROUND_TIME;
             }
         }
 
@@ -199,7 +199,7 @@ void MEMORY_CONTROLLER_NV::operate()
                 schedule(&WQ[i]);
         }
 
-        // process DRAM requests
+        // process NVRAM requests
         if (write_mode[i] && (WQ[i].next_process_index < WQ[i].SIZE)) {
             if (WQ[i].next_process_cycle <= current_core_cycle[WQ[i].entry[WQ[i].next_process_index].cpu])
                 process(&WQ[i]);
@@ -212,7 +212,7 @@ void MEMORY_CONTROLLER_NV::operate()
                 schedule(&RQ[i]);
         }
 
-        // process DRAM requests
+        // process NVRAM requests
         if ((write_mode[i] == 0) && (RQ[i].next_process_index < RQ[i].SIZE)) {
             if (RQ[i].next_process_cycle <= current_core_cycle[RQ[i].entry[RQ[i].next_process_index].cpu])
                 process(&RQ[i]);
@@ -616,22 +616,22 @@ void MEMORY_CONTROLLER_NV::process(PACKET_QUEUE *queue)
         assert(0);
     }
 
-    // paid all DRAM access latency, data is ready to be processed
+    // paid all NVRAM access latency, data is ready to be processed
     if (bank_request[op_channel][op_rank][op_bank].cycle_available <= current_core_cycle[op_cpu]) {
 
         // check if data bus is available
         if (dbus_cycle_available[op_channel] <= current_core_cycle[op_cpu]) {
 
             if (queue->is_WQ) {
                 // update data bus cycle time
-                dbus_cycle_available[op_channel] = current_core_cycle[op_cpu] + DRAM_DBUS_RETURN_TIME;
+                dbus_cycle_available[op_channel] = current_core_cycle[op_cpu] + NVRAM_DBUS_RETURN_TIME;
 
                 if (bank_request[op_channel][op_rank][op_bank].row_buffer_hit)
                     queue->ROW_BUFFER_HIT++;
                 else
                     queue->ROW_BUFFER_MISS++;
 
-                // this bank is ready for another DRAM request
+                // this bank is ready for another NVRAM request
                 bank_request[op_channel][op_rank][op_bank].request_index = -1;
                 bank_request[op_channel][op_rank][op_bank].row_buffer_hit = 0;
                 bank_request[op_channel][op_rank][op_bank].working = false;
@@ -659,7 +659,7 @@ void MEMORY_CONTROLLER_NV::process(PACKET_QUEUE *queue)
                 else
                     queue->ROW_BUFFER_MISS++;
 
-                // this bank is ready for another DRAM request
+                // this bank is ready for another NVRAM request
                 bank_request[op_channel][op_rank][op_bank].request_index = -1;
                 bank_request[op_channel][op_rank][op_bank].row_buffer_hit = 0;
                 bank_request[op_channel][op_rank][op_bank].working = false;

src/main.cc

@@ -643,11 +643,11 @@ int main(int argc, char** argv)
         DRAM_MTPS = DRAM_IO_FREQ/4;
     else
         DRAM_MTPS = DRAM_IO_FREQ;
-   // if (knob_low_bandwidth)
+    if (knob_low_bandwidth)
 
-    NVRAM_MTPS = DRAM_MTPS/4;
-   // else
-    NVRAM_MTPS = DRAM_MTPS;
+        NVRAM_MTPS = NVRAM_IO_FREQ/4;
+    else
+        NVRAM_MTPS = NVRAM_IO_FREQ;
 
     // DRAM access latency
     tRP  = (uint32_t)((1.0 * tRP_DRAM_NANOSECONDS  * CPU_FREQ) / 1000); 
@@ -671,6 +671,7 @@ int main(int argc, char** argv)
     printf("Off-chip DRAM Size: %u MB Channels: %u Width: %u-bit Data Rate: %u MT/s\n",
             DRAM_SIZE, DRAM_CHANNELS, 8*DRAM_CHANNEL_WIDTH, DRAM_MTPS);
 
+
     printf("Off-chip NVRAM Size: %u MB Channels: %u Width: %u-bit Data Rate: %u MT/s\n",
             NVRAM_SIZE, NVRAM_CHANNELS, 8*NVRAM_CHANNEL_WIDTH, NVRAM_MTPS);