kev-report.qmd

---
title: "KEV Report"
output:
  html_document:
    self_contained: yes
editor_options: 
  chunk_output_type: console
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(
  echo = FALSE,
  dev = "svg",
  warning = FALSE,
  message = FALSE
)
options(
  tidyverse.quiet = TRUE,
  future.rng.onMisuse = FALSE
)
```

### Generated on: `r Sys.Date()`

```{r libs, cache=FALSE}
suppressPackageStartupMessages({
  library(furrr)
  library(hrbrthemes)
  library(gt)
  library(gtExtras)
  library(stringi)
  library(ggbeeswarm)
  library(tidyverse)
})

extrafont::loadfonts(quiet=TRUE)
```

```{r data}
# READ IN KEV
kev <- jsonlite::fromJSON("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json")

kvuln <- as_tibble(janitor::clean_names(kev$vulnerabilities))
kvuln$date_added <- as.Date(kvuln$date_added)
kvuln$grp <- as.numeric(factor(kvuln$date_added))
kvuln$vendor_project <- stri_trim_both(kvuln$vendor_project)

# SAVE LOCALLY
write_csv(kvuln, "~/Data/kvuln.csv")

.nvd_cvd <- function(cve_id) {
  httr::GET(
    url = sprintf(
      "https://services.nvd.nist.gov/rest/json/cve/1.0/%s", as.character(cve_id[1])
    ),
    httr::accept_json()
  ) -> res
  httr::warn_for_status(res)
  out <- httr::content(res, as = "text", encoding = "UTF-8")
	if (grepl("<html", out)) {
		message(cve_id)
		out <- NULL
	} else {
    out <- jsonlite::fromJSON(out)
	}
  if (is.null(out)) out <- list()
  out
}

# INDIVIDUAL CVE DETAILS RECORDS ARE CACHED LOCALLY 
nvd_cve <- memoise::memoise(.nvd_cvd, cache = cachem::cache_disk(dir = "~/Data/nvd"))

# IN THE EVENT WE ARE RE-BOOTSTRAPPING THE CACHE OR THERE ARE ALOT OF CVES IN A CATALOG UPDATE
plan(multisession)

kvuln$cve_details <- future_map(kvuln$cve_id, nvd_cve, .progress = TRUE)

# IT'S A MORE COMPLEX DATAFRAME NOW SO USE RDS
saveRDS(kvuln, sprintf("~/Data/%s-kev.rds", Sys.Date()))

# DO SOME TRANSORMATIONS
kvuln$date_added <- as.Date(kvuln$date_added)
kvuln$vendor_project <- stri_trans_tolower(stri_trim_both(kvuln$vendor_project))

# ALL THE CVE DETAILS ARE IN A ******* AWFUL NESTED JSON B/C NVD IS AN IDIOT
map_chr(kvuln$cve_details, ~{
  pub <- .x$result$CVE_Items$publishedDate
  if (is.null(pub)) NA else substr(pub, 1, 10)
}) |> 
  as.Date() -> kvuln$pub_date

map_chr(kvuln$cve_details, ~{
  cvss <- .x$result$CVE_Items$impact$baseMetricV3$cvssV3$baseScore
  if (is.null(cvss)) NA else cvss
}) |> as.numeric() -> kvuln$cvss

map_chr(kvuln$cve_details, ~{
  cwe <- .x$result$CVE_Items$cve$problemtype$problemtype_data[[1]]$description[[1]]$value
  cwe <- if (is.null(cwe)) NA else cwe
  if (length(cwe)>1) cwe <- paste0(cwe, collapse=",")
  cwe
}) -> kvuln$cwe

map_chr(kvuln$cve_details, ~{
  vec <- .x$result$CVE_Items$impact$baseMetricV3$cvssV3$attackVector
  if (is.null(vec)) NA else vec
}) -> kvuln$vector

map_chr(kvuln$cve_details, ~{
  complexity <- .x$result$CVE_Items$impact$baseMetricV3$cvssV3$attackComplexity
  if (is.null(complexity)) NA else complexity
}) -> kvuln$complexity

map_chr(kvuln$cve_details, ~{
  severity <- .x$result$CVE_Items$impact$baseMetricV3$cvssV3$baseSeverity
  if (is.null(severity)) NA else severity
}) -> kvuln$severity

write_csv(
  kvuln |> select(-cve_details), sprintf("~/Development/cisa-known-exploited-vulns/data/%s-enriched.csv", Sys.Date()))
```

```{r kev-label}
# MAKE A RELEASE TAG LIKE CISA SHOULD HAVE
count(kvuln, date_added) |> 
  mutate(rls = sprintf("kev-%02d", 1:n())) |> 
  rename(n_vuln = n) -> krls

kvuln |> 
  left_join(
    krls, 
    by = "date_added"
) -> kvuln
```

NOTE: KEV entries without a CVE, or CVE's without all necessary details will not render in the group they belong in.

```{r kev-dist-plot, fig.width=13}
suppressWarnings({
  ggplot() +
    geom_quasirandom(
      data = kvuln |> 
        filter(!is.na(pub_date)) |> 
        arrange(cvss),
      aes(
        x = sprintf("%s\nn=%s", gsub("-", "\n", rls), n_vuln), 
        y = pub_date, 
        group = date_added,
        fill = cvss
      ),
      direction = -1,
      alpha = 1/1.1,
      size = 3, 
      shape = 21,
      color = "#b2b2b2",
      width = 0.2
    ) +
    scale_x_discrete(position = "top") +
    scale_fill_viridis_c(
      breaks = 1:10,
      limits = c(1, 10),
      direction = 1,
      option = "magma"
    ) +
    labs(
      x = NULL, y = "CVE Publication Date",
      title = "CVE Publication Date Distribution by KEV Release",
      subtitle = "Each distribution group is one KEV release; dots are the CVE; posiiton on Y axis is the CVE age; color is CVSS score "
    ) +
    theme_ipsum_es(grid="Y") +
    theme(
      axis.text.x = element_text(color = "black", size = 9),
      axis.text.y = element_text(color = "black"),
      legend.position = "bottom",
      legend.direction = "horizontal"
    )
})
```

```{r kev-rls-history}
kvuln |> 
  count(kev_rls = sprintf("%s (n=%s)", rls, n_vuln)) |> 
  arrange(desc(kev_rls)) |> 
  gt::gt() |> 
  gt_plt_bar(column = n) |> 
  gt_theme_nytimes() |>  
  tab_header(title = "KEV Release History")
```
NOTE: KEV releases with fewer than two elements will be dropped from this view

```{r cve-date-distribution, fig.width=11}
suppressWarnings({
  ggplot() +
    geom_density(
      data = kvuln |> 
        filter(
          !is.na(pub_date),
          n_vuln > 2
        ) |>
        mutate(rls = sprintf("%s (n=%s)", rls, n_vuln)),
      aes(
        x = pub_date, 
        group = rls,
        fill = rls
      ),
      alpha = 1/2
    ) +
    labs(
      x = NULL, y = NULL,
      title = "CVE Publication Date Distribution by KEV Release"
    ) +
    theme_ipsum_es(grid="Y") +
    theme(legend.position = "none")
})
```

```{r kev-release-year-table}
kvuln |> 
  filter(!is.na(pub_date)) |> 
  count(year = lubridate::year(pub_date)) |> 
  arrange(year) |>
  mutate(
    prop = proportions(n),
    cprop = cumsum(prop)
  ) |> 
  gt::gt() |> 
  gt::fmt_percent(c("prop", "cprop")) |> 
  gt_theme_nytimes() |>  
  tab_header(title = "KEV CVE Release Year")
```

```{r kev-pub-date-dist, fig.width=11}
ggplot() +
  geom_histogram(
    data = kvuln |> 
      filter(!is.na(pub_date)),
    aes(
      x = pub_date
    ),
    fill = "steelblue",
    bins = 30
  ) +
  scale_y_comma() +
  labs(
    x = NULL, y = NULL,
    title = "KEV CVE Publication Date Distribution"
  ) +
  theme_ipsum_es(grid="Y")
```

```{r kev-cwe, fig.width=11, fig.height=11}
kvuln |> 
  separate_rows(cwe, sep=",") |> 
  count(rls, cwe) |> 
  complete(rls, cwe) |> 
  mutate(
    rls = gsub("-", "\n", rls),
    cwe = factor(cwe, levels = unique(stri_sort(cwe, na_last = TRUE, numeric = TRUE)))
  ) |> 
  ggplot() +
  geom_tile(
    aes(rls, cwe, fill = n),
    color = "white",
    size = 1
  ) +
  scale_x_discrete(position = "top") +
  scale_fill_viridis_c(
    trans = "log10",
    na.value = "gray80"  
  ) +
  labs(
    x = NULL, y = NULL,
    title = "CWE's Per KEV Release"
  ) +
  theme_ipsum_es(grid="")
```

```{r cwe-by-year, fig.height=9}
count(kvuln, cwe) |> 
  filter(n>2) |> 
  select(-n) |> 
  left_join(kvuln) |> 
  mutate(
    year = lubridate::year(pub_date),
  ) |> 
  filter(!is.na(year)) |> 
  filter(!grepl("NVD|Unkn", cwe)) |> 
  filter(!(
    cwe %in% c(
      "CWE-918",
      "CWE-863",
      "CWE-862",
      "CWE-611",
      "CWE-59",
      "CWE-404",
      "CWE-400",
      "CWE-290"
    )
  )) |> 
  count(cwe, year) |> 
  mutate(
    cwe = fct_reorder(cwe, n, sum)
  ) |> 
  complete(cwe, year) |> 
  ggplot() +
  geom_tile(
    aes(year, y=cwe, fill=n),
    color = "white", size = 1
  ) + 
  scale_x_continuous(
    position = "top",
    breaks = 2007:2022,
    expand = c(0, 0, 0, 0)
  ) +
  scale_fill_viridis_c(
    name = NULL,
    option = "magma",
    na.value = "gray90"
  ) +
  labs(
    x = NULL, y = NULL,
    title = "CISA KEV CWE Release Year Density"
  ) +
  theme_ipsum_es(grid="", axis_text_size = 8)
```

```{r kev-vendors-01, fig.width=11, fig.height=10}
count(kvuln, vendor_project, sort=TRUE) |> 
  filter(n>1) |> 
  mutate(
    vendor_project = factor(vendor_project, levels = vendor_project) |> fct_rev()
  ) |>
  ggplot() +
  geom_col(
    aes(n, vendor_project),
    fill = "steelblue", color = NA
  ) +
  scale_x_comma(trans = "log10") +
  labs(
    x = NULL, y = NULL,
    title = "Vendors Appearing More Than Once",
    subtitle = "log10 scale"
  ) +
  theme_ipsum_es(grid="X")
```

```{r kev-vendors-02, fig.width=11, fig.height=10}
count(kvuln, vendor_project, sort=TRUE) |> 
  filter(n>1) |> 
  select(vendor_project) |> 
  left_join(
    count(kvuln, rls, vendor_project, sort=TRUE),
    by = "vendor_project"
  ) |> 
  mutate(
    rls = gsub("-", "\n", rls),
    vendor_project = fct_reorder(vendor_project, n, sum)
  ) |> 
  complete(rls, vendor_project) |> 
  ggplot() +
  geom_tile(
    aes(rls, vendor_project, fill = n),
    color = "white",
    size = 1
  ) +
  scale_x_discrete(position = "top") +
  scale_fill_viridis_c(
    na.value = "gray80",
    trans = "log10"
  ) +
  labs(
    x = NULL, y = NULL,
    title = "Vendors by KEV Release (Only Vendors Appearing More Than Once)",
    subtitle = "log10 scale"
  ) +
  theme_ipsum_es(grid="")
```

```{r kev-severity, fig.height=7}
count(kvuln, rls, severity) |>
  mutate(
    severity = replace_na(severity, "?") |> 
      factor(levels = c("MEDIUM", "HIGH", "CRITICAL", "?"), ordered = TRUE)
  ) |> 
  complete(rls, severity) |> 
  ggplot() +
  geom_tile(
    aes(
      severity, rls, fill = n
    ),
    color = "white", size = 1
  ) +
  scale_x_discrete(position = "top") +
  scale_fill_viridis_c(
    trans = "log10",
    na.value = "gray80"  
  ) +
  labs(
    x = NULL, y = NULL,
    title = "KEV Release Severity"
  ) +
  theme_ipsum_es(grid="")
```

```{r kev-release-vector, fig.height=7}
count(kvuln, rls, vector) |>
  mutate(
    vector = replace_na(vector, "?") |> 
      factor(levels = c("LOCAL", "ADJACENT_NETWORK", "NETWORK", "?"), ordered = TRUE)
  ) |> 
  complete(rls, vector) |> 
  ggplot() +
  geom_tile(
    aes(
      vector, rls, fill = n
    ),
    color = "white", size = 1
  ) +
  scale_x_discrete(position = "top") +
  scale_fill_viridis_c(
    trans = "log10",
    na.value = "gray80"  
  ) +
  labs(
    x = NULL, y = NULL,
    title = "KEV Release Vector"
  ) +
  theme_ipsum_es(grid="")
```

```{r kev-table}
kvuln |> 
  select(-cve_details, -required_action, -due_date, -grp, -n_vuln) |> 
  select(rls, everything()) |> 
  DT::datatable(
    options = list(
      order = list( list(1, 'desc' ))
    )
  )
```