double_sided_rowhammer.cc

// Copyright 2015, Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Small test program to systematically check through the memory to find bit
// flips by double-sided row hammering.
//
// Compilation instructions:
//   g++ -std=c++11 [filename]
//
// ./test_double_sided_hammering [-t nsecs] [-p percentage] [-f outputfile]
//
// Hammers for nsecs seconds, acquires the described fraction of memory (0.0
// to 0.9 or so). Normally writes to stdout, but if -f is provided, will fork
// and write to the specified file instead.

#include <asm/unistd.h>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <linux/kernel-page-flags.h>
#include <linux/perf_event.h>
#include <map>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mount.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
#include <vector>

// The fraction of physical memory that should be mapped for testing.
double fraction_of_physical_memory = 0.3;

// The time to hammer before aborting. Defaults to one hour.
uint64_t number_of_seconds_to_hammer = 3600;

// The number of memory reads to try.
uint64_t number_of_reads = 1000*1024;

// Obtain the size of the physical memory of the system.
uint64_t GetPhysicalMemorySize() {
  struct sysinfo info;
  sysinfo( &info );
  return (size_t)info.totalram * (size_t)info.mem_unit;
}

// If physical_address is in the range, put (physical_address, virtual_address)
// into the map.
bool PutPointerIfInAddressRange(const std::pair<uint64_t, uint64_t>& range,
    uint64_t physical_address, uint8_t* virtual_address,
    std::map<uint64_t, uint8_t*>& pointers) {
  if (physical_address >= range.first && physical_address <= range.second) {
    printf("[!] Found desired physical address %lx at virtual %lx\n", 
        (uint64_t)physical_address, (uint64_t)virtual_address);
    pointers[physical_address] = virtual_address;
    return true;
  }
  return false;
}

bool IsRangeInMap(const std::pair<uint64_t, uint64_t>& range,
    const std::map<uint64_t, uint8_t*>& mapping) {
  for (uint64_t check = range.first; check <= range.second; check += 0x1000) {
    if (mapping.find(check) == mapping.end()) {
      printf("[!] Failed to find physical memory at %lx\n", check);
      return false;
    }
  }
  return true;
}

uint64_t GetPageFrameNumber(int pagemap, uint8_t* virtual_address) {
    // Read the entry in the pagemap.
    off_t pos = lseek(pagemap,
        (reinterpret_cast<uintptr_t>(virtual_address) / 0x1000) * 8, SEEK_SET);
    uint64_t value;
    int got = read(pagemap, &value, 8);
    assert(got == 8);
    uint64_t page_frame_number = value & ((1ULL << 54)-1);
    return page_frame_number; 
}

void SetupMapping(uint64_t* mapping_size, void** mapping) {
  *mapping_size = 
    static_cast<uint64_t>((static_cast<double>(GetPhysicalMemorySize()) * 
          fraction_of_physical_memory));

  *mapping = mmap(NULL, *mapping_size, PROT_READ | PROT_WRITE,
      MAP_POPULATE | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
  assert(*mapping != (void*)-1);

  // Initialize the mapping so that the pages are non-empty.
  printf("[!] Initializing large memory mapping ...");
  for (uint64_t index = 0; index < *mapping_size; index += 0x1000) {
    uint64_t* temporary = reinterpret_cast<uint64_t*>(
        static_cast<uint8_t*>(*mapping) + index);
    temporary[0] = index;
  }
  printf("done\n");
}

// Build a memory mapping that is big enough to cover all of physical memory.
bool GetMappingsForPhysicalRanges(
    const std::pair<uint64_t, uint64_t>& physical_range_A_to_hammer, 
    std::map<uint64_t, uint8_t*>& pointers_to_hammer_A,
    const std::pair<uint64_t, uint64_t>& physical_range_B_to_hammer,
    std::map<uint64_t, uint8_t*>& pointers_to_hammer_B,
    const std::pair<uint64_t, uint64_t>& physical_range_to_check,
    std::map<uint64_t, uint8_t*>& pointers_to_range_to_check,
    void** out_mapping) {

  uint64_t mapping_size;
  void* mapping;
  SetupMapping(&mapping_size, &mapping);

  int pagemap = open("/proc/self/pagemap", O_RDONLY);
  assert(pagemap >= 0);

  // Don't assert if opening this fails, the code needs to run under usermode.
  int kpageflags = open("/proc/kpageflags", O_RDONLY);

  // Iterate over the entire mapping, identifying the physical addresses for 
  // each 4k-page.
  for (uint64_t offset = 0; offset < mapping_size; offset += 0x1000) {
    uint8_t* virtual_address = static_cast<uint8_t*>(mapping) + offset;
    uint64_t page_frame_number = GetPageFrameNumber(pagemap, virtual_address);
    // Read the flags for this page if we have access to kpageflags.
    uint64_t page_flags = 0;
    if (kpageflags >= 0) {
      int pos = lseek(kpageflags, page_frame_number * 8, SEEK_SET);
      int got = read(kpageflags, &page_flags, 8);
      assert(got == 8);
    }

    uint64_t physical_address;
    if (page_flags & KPF_HUGE) {
      printf("[!] %lx is on huge page\n", (uint64_t)virtual_address);
      physical_address = (page_frame_number * 0x1000) + 
        (reinterpret_cast<uintptr_t>(virtual_address) & (0x200000-1));
    } else {
      physical_address = (page_frame_number * 0x1000) + 
       (reinterpret_cast<uintptr_t>(virtual_address) & 0xFFF);
    }

    //printf("[!] %lx is %lx\n", (uint64_t)virtual_address, 
    //    (uint64_t)physical_address);
    PutPointerIfInAddressRange(physical_range_A_to_hammer, physical_address,
       virtual_address, pointers_to_hammer_A);
    PutPointerIfInAddressRange(physical_range_B_to_hammer, physical_address,
       virtual_address, pointers_to_hammer_B);
    PutPointerIfInAddressRange(physical_range_to_check, physical_address,
       virtual_address, pointers_to_range_to_check);
  }
  // Check if all physical addresses the caller asked for are in the resulting
  // map.
  if (IsRangeInMap(physical_range_A_to_hammer, pointers_to_hammer_A)
      && IsRangeInMap(physical_range_B_to_hammer, pointers_to_hammer_B)
      && IsRangeInMap(physical_range_to_check, pointers_to_range_to_check)) {
    return true;
  }
  return false;
}

//
// Example code for perf_event_open from the perf_event_open manpage.
static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid,
  int cpu, int group_fd, unsigned long flags) {
  int ret;
  ret = syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags);
  return ret;
}

uint64_t HammerAddressesStandard(
    const std::pair<uint64_t, uint64_t>& first_range,
    const std::pair<uint64_t, uint64_t>& second_range,
    uint64_t number_of_reads) {
  uint64_t* first_pointer = reinterpret_cast<uint64_t*>(first_range.first);
  uint64_t* second_pointer = reinterpret_cast<uint64_t*>(second_range.first);
  volatile uint64_t sum = 0;

  while (number_of_reads-- > 0) {
    sum += first_pointer[0];
    sum += second_pointer[0];
    asm volatile(
        "clflush (%0);\n\t"
        "clflush (%1);\n\t"
        : : "r" (first_pointer), "r" (second_pointer) : "memory");
  }
  return sum;
}

typedef uint64_t(HammerFunction)(
    const std::pair<uint64_t, uint64_t>& first_range,
    const std::pair<uint64_t, uint64_t>& second_range,
    uint64_t number_of_reads);

// A comprehensive test that attempts to hammer adjacent rows for a given 
// assumed row size (and assumptions of sequential physical addresses for 
// various rows.
uint64_t HammerAllReachablePages(uint64_t presumed_row_size, 
    void* memory_mapping, uint64_t memory_mapping_size, HammerFunction* hammer,
    uint64_t number_of_reads) {
  // This vector will be filled with all the pages we can get access to for a
  // given row size.
  std::vector<std::vector<uint8_t*>> pages_per_row;
  uint64_t total_bitflips = 0;

  pages_per_row.resize(memory_mapping_size / presumed_row_size);
  int pagemap = open("/proc/self/pagemap", O_RDONLY);
  assert(pagemap >= 0);

  printf("[!] Identifying rows for accessible pages ... ");
  for (uint64_t offset = 0; offset < memory_mapping_size; offset += 0x1000) {
    uint8_t* virtual_address = static_cast<uint8_t*>(memory_mapping) + offset;
    uint64_t page_frame_number = GetPageFrameNumber(pagemap, virtual_address);
    uint64_t physical_address = page_frame_number * 0x1000;
    uint64_t presumed_row_index = physical_address / presumed_row_size;
    //printf("[!] put va %lx pa %lx into row %ld\n", (uint64_t)virtual_address,
    //    physical_address, presumed_row_index);
    if (presumed_row_index > pages_per_row.size()) {
      pages_per_row.resize(presumed_row_index);
    }
    pages_per_row[presumed_row_index].push_back(virtual_address);
    //printf("[!] done\n");
  }
  printf("Done\n");

  // We should have some pages for most rows now.
  for (uint64_t row_index = 0; row_index + 2 < pages_per_row.size(); 
      ++row_index) {
    if ((pages_per_row[row_index].size() != 64) || 
        (pages_per_row[row_index+2].size() != 64)) {
      printf("[!] Can't hammer row %ld - only got %ld/%ld pages "
          "in the rows above/below\n",
          row_index+1, pages_per_row[row_index].size(), 
          pages_per_row[row_index+2].size());
      continue;
    } else if (pages_per_row[row_index+1].size() == 0) {
      printf("[!] Can't hammer row %ld, got no pages from that row\n", 
          row_index+1);
      continue;
    }
    printf("[!] Hammering rows %ld/%ld/%ld of %ld (got %ld/%ld/%ld pages)\n", 
        row_index, row_index+1, row_index+2, pages_per_row.size(), 
        pages_per_row[row_index].size(), pages_per_row[row_index+1].size(), 
        pages_per_row[row_index+2].size());
    // Iterate over all pages we have for the first row.
    for (uint8_t* first_row_page : pages_per_row[row_index]) {
      // Iterate over all pages we have for the second row.
      for (uint8_t* second_row_page : pages_per_row[row_index+2]) {
        // Set all the target pages to 0xFF.
        for (uint8_t* target_page : pages_per_row[row_index+1]) {
          memset(target_page, 0xFF, 0x1000);
        }
        // Test sleep code to see how this affects the distribution.
        sleep(1);
        // Now hammer the two pages we care about.
        std::pair<uint64_t, uint64_t> first_page_range(
            reinterpret_cast<uint64_t>(first_row_page), 
            reinterpret_cast<uint64_t>(first_row_page+0x1000));
        std::pair<uint64_t, uint64_t> second_page_range(
            reinterpret_cast<uint64_t>(second_row_page),
            reinterpret_cast<uint64_t>(second_row_page+0x1000));
        hammer(first_page_range, second_page_range, number_of_reads);
        // Now check the target pages.
        uint64_t number_of_bitflips_in_target = 0;
        for (const uint8_t* target_page : pages_per_row[row_index+1]) {
          for (uint32_t index = 0; index < 0x1000; ++index) {
            if (target_page[index] != 0xFF) {
              ++number_of_bitflips_in_target;
            }
          }
        }
        if (number_of_bitflips_in_target > 0) {
          printf("[!] Found %ld flips in row %ld (%lx to %lx) when hammering "
              "%lx and %lx\n", number_of_bitflips_in_target, row_index+1,
              ((row_index+1)*presumed_row_size), 
              ((row_index+2)*presumed_row_size)-1,
              GetPageFrameNumber(pagemap, first_row_page)*0x1000, 
              GetPageFrameNumber(pagemap, second_row_page)*0x1000);
          total_bitflips += number_of_bitflips_in_target;
        }
      }
    }
  }
  return total_bitflips;
}

uint64_t HammerAllReachableRows(HammerFunction* hammer, 
    uint64_t number_of_reads) {
  uint64_t mapping_size;
  void* mapping;
  SetupMapping(&mapping_size, &mapping);

  uint64_t result = HammerAllReachablePages(1024*256, mapping, mapping_size,
    hammer, number_of_reads);
}

void HammeredEnough(int sig) {
  printf("[!] Spent %ld seconds hammering, exiting now.\n",
      number_of_seconds_to_hammer);
  fflush(stdout);
  fflush(stderr);
  exit(0);
}

int main(int argc, char** argv) {
  // Turn off stdout buffering when it is a pipe.
  setvbuf(stdout, NULL, _IONBF, 0);
  bool should_fork = false;
  char outputfilename[1024];

  int opt;
  while ((opt = getopt(argc, argv, "t:p:f:r:")) != -1) {
    switch (opt) {
      case 't':
        number_of_seconds_to_hammer = atoi(optarg);
        break;
      case 'p':
        fraction_of_physical_memory = atof(optarg);
        break;
      case 'f':
        memset(outputfilename, 0, sizeof(outputfilename));
        strncpy(outputfilename, optarg, 1023);
        should_fork = true;
        break;
      default:
        fprintf(stderr, "Usage: %s [-t nsecs] [-p percent] [-f outfile]\n", 
            argv[0]);
        exit(EXIT_FAILURE);
    }
  }

  signal(SIGALRM, HammeredEnough);

  if (should_fork) {
    pid_t pid = fork();
    if (pid == 0) {
      fpos_t pos;
      int fd;
      fflush(stdout);
      fgetpos(stdout, &pos);
      fd = dup(fileno(stdout));
      freopen(outputfilename, "w", stdout);
      alarm(number_of_seconds_to_hammer);
      HammerAllReachableRows(&HammerAddressesStandard, number_of_reads);
    }
  } else {
    printf("[!] Starting the testing process...\n");
    alarm(number_of_seconds_to_hammer);
    HammerAllReachableRows(&HammerAddressesStandard, number_of_reads);
  }
}