diff --git a/llm-top-10-governance-doc/LLM_AI_Security_and_Governance_Checklist.pdf b/llm-top-10-governance-doc/LLM_AI_Security_and_Governance_Checklist.pdf new file mode 100644 index 00000000..6ab9a3d8 Binary files /dev/null and b/llm-top-10-governance-doc/LLM_AI_Security_and_Governance_Checklist.pdf differ diff --git a/llm-top-10-governance-doc/LLM_AI_Security_and_Governance_Checklist.tex b/llm-top-10-governance-doc/LLM_AI_Security_and_Governance_Checklist.tex new file mode 100755 index 00000000..792cc4bd --- /dev/null +++ b/llm-top-10-governance-doc/LLM_AI_Security_and_Governance_Checklist.tex @@ -0,0 +1,77 @@ +%%% ================================================ +%%% OWASP Top 10 For LLM Applications Template +%%% Version: 0.0.1 +%%% Date: 2023-11-28 +%%% Template Authors: +%%% - Jason Ross +%%% ================================================ + +%%% ================================================ +%%% How to use this +%%% ================================================ +%%% Update the variables below +%%% If you want to change text color on the front +%%% cover, the areas required are commented below. +%%% If you want to modify text and border colors +%%% for your chapter headers go into the file +%%% `structure.tex` and replace the name of the +%%% colour (set to ) with a new colour name +%%% (find and replace ctrl+f will do this for you). +%%% ================================================ + +%%% ================================================ +%%% VARIABLES +%%% ================================================ + +%%% Project Name +\def\projectName{LLM AI Security \& Governance Checklist} +\def\projectSubName{From the OWASP Top 10 \\ for LLM Applications Team} +\def\docVersion{0.5} + +%%% Project Type +\def\projectType{OWASP Project Document} + +%%% Report Date (defaults to Today's date) +\def\date{\today} + + +%%% ================================================ +%%% DOCUMENT CONFIGURATION +%%% ================================================ + +\documentclass[ + 11pt, % Default font size, select one of 10pt, 11pt or 12pt + fleqn, % Left align equations + letterpaper, % Paper size, use either 'a4paper' for A4 size or 'letterpaper' for US letter size + % landscape, % Uncomment for for a landscape layout (useful for wide tables or figures) + oneside, % Uncomment for oneside mode: this doesn't start new chapters and parts on odd pages (adding an empty page if required) + % this mode is more suitable if the book is to be read on a screen instead of printed +]{owasp-doc} + + +%%% ================================================ +%%% DOCUMENT BEGINS HERE +%%% ================================================ +% \tracingmacros=1 % turn on tracing +\begin{document} + +\pagestyle{fancy} + +%%% COVER PAGE +\input{sections/coverpage} + +%%% CHANGELOG & DISCLAIMER +\input{sections/changelog-disclaimer} + +%%% TABLE OF CONTENTS +\input{sections/toc} + +%%% ALL OTHER CONTENT +\input{sections/main} + +%%% APPENDICES +\appendix +\input{apx/team} + +\end{document} +% \tracingmacros=0 % turn off tracing \ No newline at end of file diff --git a/llm-top-10-governance-doc/Makefile b/llm-top-10-governance-doc/Makefile new file mode 100755 index 00000000..b3fb59dd --- /dev/null +++ b/llm-top-10-governance-doc/Makefile @@ -0,0 +1,48 @@ +# simple Makefile to compile the latex + +PROJ=`cd ../; basename \`pwd\`` +DOC=LLM_AI_Security_and_Governance_Checklist +FILES=$(DOC).tex + +# create a random password +# requires pwgen +PW!=pwgen -n1 + +# create a PDF +pdf: $(FILES) distclean + +# lualtex needs to run iteratively so it can process all references + pdflatex $(DOC).tex + pdflatex $(DOC).tex + pdflatex $(DOC).tex + +# now clean up the intermediate steps + make clean + +# create an encrypted PDF +# requires pdftk (apt-get install pdftk-java) +crypt: pdf + pdftk $(DOC).pdf output $(DOC).encrypted.pdf user_pw $(PW) allow allfeatures + echo "Encryption password" is $(PW) + +# create an RTF file for folks that want "Word" versions +rtf: $(FILES) + latex2rtf $(DOC) + +# clear out all the intermediate cruft from a build, but leave the PDFs +clean: + rm -f *.tmp *.out *.aux *.idx *.log *.ilg *.xref *.lg \ + *.toc *.ind *.4og *.4ct *.4tc *.idv *.lot *.lov *.lltr *.lstr \ + *.ptc *.dvi *.tv *.vrb *.hv *.hst *.ver *.mv *.lv *.odt $(DOC).synctex.gz \ + *.rtf *.cg *.cp *.cpc *.dd *.sc *.xml *.bcf *.loc *.tdo *.4o* $(DOC).txt tex4ht.env \ + *.fdb_latexmk *.fls $(DOC).gz > /dev/null 2>&1 + -find . -name "*.aux" -exec rm {} \; + +# clear out all the intermediate cruft and also any PDFs +distclean: + -rm -f $(DOC).pdf $(DOC).encrypted.pdf *.tmp *.out *.aux *.idx *.log *.ilg *.xref *.lg \ + *.toc *.ind *.4og *.4ct *.4tc *.idv *.lot *.lov *.lltr *.lstr \ + *.ptc *.dvi *.tv *.vrb *.hv *.hst *.ver *.mv *.lv *.odt $(DOC).synctex.gz \ + *.rtf *.cg *.cp *.cpc *.dd *.sc *.xml *.bcf *.loc *.tdo *.4o* $(DOC).txt tex4ht.env \ + *.fdb_latexmk *.fls $(DOC).gz > /dev/null 2>&1 + -find . -name "*.aux" -exec rm {} \; diff --git a/llm-top-10-governance-doc/apx/team.tex b/llm-top-10-governance-doc/apx/team.tex new file mode 100644 index 00000000..ff7ad11d --- /dev/null +++ b/llm-top-10-governance-doc/apx/team.tex @@ -0,0 +1,45 @@ +% !TEX root = report.tex + +% ------------------------------------------------ +% TEAM +% ------------------------------------------------ + +\headerimage +\chapter{Team} + +Thank you to the OWASP Top 10 for LLM Applications Cybersecurity and Governance +Checklist Contributors. + +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.33\textwidth}|p{0.33\textwidth}|p{0.33\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \multicolumn{3}{|c|}{ + \textbf{Checklist Contributors} + } \\ + \hline + \endfirsthead + \hline + \multicolumn{3}{|c|}{ + \textbf{Checklist Contributors} + } \\ + \hline + \endhead + \endfoot + %%% TABLE DATA GOES HERE +\hline + Sandy Dunn & Heather Linn & John Sotiropoulos \\ + \hline + Steve Wilson & Fabrizio Cilli & Aubrey King \\ + \hline + Bob Simonoff & David Rowe & Rob Vanderveer \\ + \hline + Emmanual Guilherme Junior & Andrea Succi & Jason Ross \\ + \hline + %%% TABLE DATA ENDS HERE + \caption{OWASP LLM AI Security \& Governance Checklist v.0.5 Team} + \label{tab:team} +\end{longtable} diff --git a/llm-top-10-governance-doc/assets/FrontCover.jpg b/llm-top-10-governance-doc/assets/FrontCover.jpg new file mode 100755 index 00000000..d27f9141 Binary files /dev/null and b/llm-top-10-governance-doc/assets/FrontCover.jpg differ diff --git a/llm-top-10-governance-doc/assets/Header_Background.png b/llm-top-10-governance-doc/assets/Header_Background.png new file mode 100644 index 00000000..59d7562f Binary files /dev/null and b/llm-top-10-governance-doc/assets/Header_Background.png differ diff --git a/llm-top-10-governance-doc/assets/Header_Black_Granite.jpg b/llm-top-10-governance-doc/assets/Header_Black_Granite.jpg new file mode 100755 index 00000000..fd843d86 Binary files /dev/null and b/llm-top-10-governance-doc/assets/Header_Black_Granite.jpg differ diff --git a/llm-top-10-governance-doc/assets/Header_Black_Rocks.jpg b/llm-top-10-governance-doc/assets/Header_Black_Rocks.jpg new file mode 100755 index 00000000..87fab4af Binary files /dev/null and b/llm-top-10-governance-doc/assets/Header_Black_Rocks.jpg differ diff --git a/llm-top-10-governance-doc/assets/Header_Blue_Net.jpg b/llm-top-10-governance-doc/assets/Header_Blue_Net.jpg new file mode 100755 index 00000000..011c042c Binary files /dev/null and b/llm-top-10-governance-doc/assets/Header_Blue_Net.jpg differ diff --git a/llm-top-10-governance-doc/assets/Page_Header.png b/llm-top-10-governance-doc/assets/Page_Header.png new file mode 100644 index 00000000..cc09dd59 Binary files /dev/null and b/llm-top-10-governance-doc/assets/Page_Header.png differ diff --git a/llm-top-10-governance-doc/assets/ai_deployment_strategy.jpg b/llm-top-10-governance-doc/assets/ai_deployment_strategy.jpg new file mode 100644 index 00000000..a673ae83 Binary files /dev/null and b/llm-top-10-governance-doc/assets/ai_deployment_strategy.jpg differ diff --git a/llm-top-10-governance-doc/assets/ai_deployment_strategy.png b/llm-top-10-governance-doc/assets/ai_deployment_strategy.png new file mode 100644 index 00000000..5e53ae75 Binary files /dev/null and b/llm-top-10-governance-doc/assets/ai_deployment_strategy.png differ diff --git a/llm-top-10-governance-doc/assets/ai_implementation_strategy.png b/llm-top-10-governance-doc/assets/ai_implementation_strategy.png new file mode 100644 index 00000000..a5cbb036 Binary files /dev/null and b/llm-top-10-governance-doc/assets/ai_implementation_strategy.png differ diff --git a/llm-top-10-governance-doc/assets/ai_llm_relationship.png b/llm-top-10-governance-doc/assets/ai_llm_relationship.png new file mode 100644 index 00000000..78a4e9f6 Binary files /dev/null and b/llm-top-10-governance-doc/assets/ai_llm_relationship.png differ diff --git a/llm-top-10-governance-doc/assets/ai_threat_map.png b/llm-top-10-governance-doc/assets/ai_threat_map.png new file mode 100644 index 00000000..10566799 Binary files /dev/null and b/llm-top-10-governance-doc/assets/ai_threat_map.png differ diff --git a/llm-top-10-governance-doc/assets/cat.jpg b/llm-top-10-governance-doc/assets/cat.jpg new file mode 100755 index 00000000..754c68fa Binary files /dev/null and b/llm-top-10-governance-doc/assets/cat.jpg differ diff --git a/llm-top-10-governance-doc/assets/llm_attack_surface.png b/llm-top-10-governance-doc/assets/llm_attack_surface.png new file mode 100644 index 00000000..1746443a Binary files /dev/null and b/llm-top-10-governance-doc/assets/llm_attack_surface.png differ diff --git a/llm-top-10-governance-doc/assets/owasp_logo.png b/llm-top-10-governance-doc/assets/owasp_logo.png new file mode 100644 index 00000000..6fea322d Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_logo.png differ diff --git a/llm-top-10-governance-doc/assets/owasp_top_10_llm_app_arch.png b/llm-top-10-governance-doc/assets/owasp_top_10_llm_app_arch.png new file mode 100644 index 00000000..892ec433 Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_top_10_llm_app_arch.png differ diff --git a/llm-top-10-governance-doc/assets/owasp_top_10_llm_highlevel.jpeg b/llm-top-10-governance-doc/assets/owasp_top_10_llm_highlevel.jpeg new file mode 100644 index 00000000..cc50487d Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_top_10_llm_highlevel.jpeg differ diff --git a/llm-top-10-governance-doc/assets/owasp_wasp.png b/llm-top-10-governance-doc/assets/owasp_wasp.png new file mode 100644 index 00000000..b8ce1559 Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_wasp.png differ diff --git a/llm-top-10-governance-doc/assets/owasp_wasp.svg b/llm-top-10-governance-doc/assets/owasp_wasp.svg new file mode 100644 index 00000000..a617682a --- /dev/null +++ b/llm-top-10-governance-doc/assets/owasp_wasp.svg @@ -0,0 +1,61 @@ + +image/svg+xml™ + \ No newline at end of file diff --git a/llm-top-10-governance-doc/assets/owasp_wasp1.png b/llm-top-10-governance-doc/assets/owasp_wasp1.png new file mode 100644 index 00000000..04b9c908 Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_wasp1.png differ diff --git a/llm-top-10-governance-doc/assets/owasp_wasp_nobg.png b/llm-top-10-governance-doc/assets/owasp_wasp_nobg.png new file mode 100644 index 00000000..b5f67570 Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_wasp_nobg.png differ diff --git a/llm-top-10-governance-doc/assets/owasp_wasp_nobg_black.png b/llm-top-10-governance-doc/assets/owasp_wasp_nobg_black.png new file mode 100644 index 00000000..758dae2d Binary files /dev/null and b/llm-top-10-governance-doc/assets/owasp_wasp_nobg_black.png differ diff --git a/llm-top-10-governance-doc/assets/trustworthy_ai.png b/llm-top-10-governance-doc/assets/trustworthy_ai.png new file mode 100644 index 00000000..73fefe30 Binary files /dev/null and b/llm-top-10-governance-doc/assets/trustworthy_ai.png differ diff --git a/llm-top-10-governance-doc/fragments/changelog.tex b/llm-top-10-governance-doc/fragments/changelog.tex new file mode 100644 index 00000000..ffd1429f --- /dev/null +++ b/llm-top-10-governance-doc/fragments/changelog.tex @@ -0,0 +1,11 @@ +% !TEX root = owasp-doc.tex +%%% Changelog +\begin{figure}[t!] +\fontsize{14}{14} +\owaspbf{Revision History} +\fontsize{11}{11} + \begin{versionhistory} + \vhEntry{0.1}{2023-11-01}{Sandy Dunn}{initial draft} + \vhEntry{0.5}{2023-12-06}{Sandy Dunn, OWASP LLM Apps Team}{public draft} +\end{versionhistory} +\end{figure} diff --git a/llm-top-10-governance-doc/fragments/disclaimer.tex b/llm-top-10-governance-doc/fragments/disclaimer.tex new file mode 100644 index 00000000..4707987b --- /dev/null +++ b/llm-top-10-governance-doc/fragments/disclaimer.tex @@ -0,0 +1,10 @@ +% !TEX root = owasp-doc.tex +%%% Disclaimer +\textit{The information provided in this document does not, and is not intended +to, constitute legal advice. All information is for general informational +purposes only.\\ +\\ +This document contains links to other third-party websites. Such links are only +for convenience and OWASP does not recommend or endorse the contents of the +third-party sites. +} \ No newline at end of file diff --git a/llm-top-10-governance-doc/fragments/resources/__tpl_2col.tex b/llm-top-10-governance-doc/fragments/resources/__tpl_2col.tex new file mode 100644 index 00000000..f204df35 --- /dev/null +++ b/llm-top-10-governance-doc/fragments/resources/__tpl_2col.tex @@ -0,0 +1,29 @@ +% !TEX root = owasp-doc.tex +\clearpage +\section{TKTKSectionName} +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.45\textwidth}|p{0.55\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \textbf{Name} & + \textbf{Description}\\ + \hline + \endfirsthead + \hline + \rowcolor{owasplightpurple} + \textbf{Name} & + \textbf{Description} \\ + \hline + \endhead + \endfoot + %%% TABLE DATA STARTS HERE + \href{TKTKlink}{TKTKtext} & + TKTKdescription + \hline + %%% TABLE DATA ENDS HERE + \caption{TKTKcaption} + \label{tab:TKTKlabel} +\end{longtable} \ No newline at end of file diff --git a/llm-top-10-governance-doc/fragments/resources/__tpl_3col.tex b/llm-top-10-governance-doc/fragments/resources/__tpl_3col.tex new file mode 100644 index 00000000..15991b85 --- /dev/null +++ b/llm-top-10-governance-doc/fragments/resources/__tpl_3col.tex @@ -0,0 +1,32 @@ +% !TEX root = owasp-doc.tex +\clearpage +\section{TKTKSectionName} +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.25\textwidth}|p{0.25\textwidth}|p{0.35\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \textbf{TKTKName Resource} & + \textbf{Description} & + \textbf{Why It Is Recommended \& Where To Use It} \\ + \hline + \endfirsthead + \hline + \rowcolor{owasplightpurple} + \textbf{TKTKName Resource} & + \textbf{Description} & + \textbf{Why It Is Recommended \& Where To Use It} \\ + \hline + \endhead + \endfoot + %%% TABLE DATA GOES HERE + \href{TKTKlink}{TKTKtext} & + TKTKDescription & + TKTKReason \\ + \hline + %%% TABLE DATA ENDS HERE + \caption{TKTKCaption} + \label{tab:TKTKlabel} +\end{longtable} \ No newline at end of file diff --git a/llm-top-10-governance-doc/fragments/resources/ai-procurement-guidance.tex b/llm-top-10-governance-doc/fragments/resources/ai-procurement-guidance.tex new file mode 100644 index 00000000..bb02cf8a --- /dev/null +++ b/llm-top-10-governance-doc/fragments/resources/ai-procurement-guidance.tex @@ -0,0 +1,35 @@ +% !TEX root = owasp-doc.tex +\clearpage +\textbf{AI Procurement Guidance} +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.45\textwidth}|p{0.55\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \textbf{Name} & + \textbf{Description} \\ + \hline + \endfirsthead + \hline + \rowcolor{owasplightpurple} + \textbf{Name} & + \textbf{Description} \\ + \hline + \endhead + \endfoot + %%% TABLE DATA STARTS HERE + \href{https://www3.weforum.org/docs/WEF_Adopting_AI_Responsibly_Guidelines_for_Procurement_of_AI_Solutions_by_the_Private_Sector_2023.pdf}{World Economic Forum: Adopting AI Responsibly: Guidelines for Procurement of AI Solutions by the Private Sector: Insight Report June 2023} & + The standard benchmarks and assessment criteria for procuring Artificial + systems are in early development. The procurement guidelines provide + organizations with a baseline of considerations for the end-to-end + procurement process. + + Use this guidance to augment an organization's existing Third Party Risk + Supplier and Vendor procurement process. \\ + \hline + %%% TABLE DATA ENDS HERE + \caption{AI Procurement Guidance} + \label{tab:ai-procurement-guidance} +\end{longtable} \ No newline at end of file diff --git a/llm-top-10-governance-doc/fragments/resources/ai-vuln-repos.tex b/llm-top-10-governance-doc/fragments/resources/ai-vuln-repos.tex new file mode 100644 index 00000000..fc3888fa --- /dev/null +++ b/llm-top-10-governance-doc/fragments/resources/ai-vuln-repos.tex @@ -0,0 +1,47 @@ +% !TEX root = owasp-doc.tex +\clearpage +\textbf{AI Vulnerability Repositories} +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.45\textwidth}|p{0.55\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \textbf{Name} & + \textbf{Description}\\ + \hline + \endfirsthead + \hline + \rowcolor{owasplightpurple} + \textbf{Name} & + \textbf{Description} \\ + \hline + \endhead + \endfoot + %%% TABLE DATA STARTS HERE + \href{https://incidentdatabase.ai/}{AI Incident Database} & + A repository of articles about different times AI has failed in real-world + applications and is maintained by a college research group and crowds sourced. \\ + \hline + \href{https://oecd.ai/en/incidents}{OECD AI Incidents Monitor (AIM)} & + Offers an accessible starting point for comprehending the landscape of AI-related challenges. \\ + \hline + \rowcolor{owasplightpurple} + \multicolumn{2}{|c|}{ + \textbf{Three of the leading companies tracking AI Model vulnerabilities} + } \\ + \hline + \href{https://huntr.com/}{Huntr Bug Bounty : ProtectAI} & + Bug bounty platform for AI/ML \\ + \hline + \href{https://avidml.gitbook.io/}{AI Vulnerability Database (AVID) : \href{https://garak.ai/}{Garak}} & + Database of model vulnerabilities \\ + \hline + \href{https://airisk.io/}{AI Risk Database: Robust Intelligence} & + Database of model vulnerabilities \\ + \hline + %%% TABLE DATA ENDS HERE + \caption{AI Vulnerability Repositories} + \label{tab:ai-vulnerability-repositories} +\end{longtable} \ No newline at end of file diff --git a/llm-top-10-governance-doc/fragments/resources/mitre.tex b/llm-top-10-governance-doc/fragments/resources/mitre.tex new file mode 100644 index 00000000..76f18874 --- /dev/null +++ b/llm-top-10-governance-doc/fragments/resources/mitre.tex @@ -0,0 +1,124 @@ +% !TEX root = owasp-doc.tex +\clearpage +\textbf{MITRE Resources} +The increased frequency of LLM threats emphasizes the value of a +resilience-first approach to defending an organization's attack surface. +Existing TTPS are combined with new attack surfaces and capabilities in LLM +Adversary threats and mitigations. MITRE maintains a well-established and +widely accepted mechanism for coordinating opponent tactics and procedures +based on real-world observations. + +Coordination and mapping of an organization's LLM Security Strategy to MITRE +ATT\&CK and MITRE ATLAS allows an organization to determine where LLM Security +is covered by current processes such as API Security Standards or where +security holes exists. + +MITRE ATT\&CK (Adversarial Tactics, Techniques, and Common Knowledge) is a +framework, collection of data matrices, and assessment tool that was made by +the MITRE Corporation to help organizations figure out how well their +cybersecurity works across their entire digital attack surface and find holes +that had not been found before. It is a knowledge repository that is used all +over the world. The MITRE ATT\&CK matrix contains a collection of strategies +used by adversaries to achieve a certain goal. In the ATT\&CK Matrix, these +objectives are classified as tactics. The objectives are outlined in attack +order, beginning with reconnaissance and progressing to the eventual goal of +exfiltration or impact. + +MITRE ATLAS, which stands for "Adversarial Threat Landscape for Artificial +Intelligence Systems," is a knowledge base that is based on real-life examples +of attacks on machine learning (ML) systems by bad actors. ATLAS is based on the +MITRE ATT\&CK architecture, and its tactics and procedures complement those +found in ATT\&CK. +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.25\textwidth}|p{0.25\textwidth}|p{0.35\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \textbf{MITRE Resource} & + \textbf{Description} & + \textbf{Why It Is Recommended \& Where To Use It} \\ + \hline + \endfirsthead + \hline + \rowcolor{owasplightpurple} + \textbf{MITRE Resource} & + \textbf{Description} & + \textbf{Why It Is Recommended \& Where To Use It} \\ + \hline + \endhead + \endfoot + %%% TABLE DATA STARTS HERE + \href{https://attack.mitre.org/}{MITRE ATT\&CK} & + Knowledge base of adversary tactics and techniques based on real-world observations & + The ATT\&CK knowledge base is used as a foundation for the development of + specific threat models and methodologies. Map existing controls within the + organization to adversary tactics and techniques to identify gaps or areas to + test. \\ + \hline + \href{https://medium.com/mitre-engenuity/att-ck-workbench-2-0-your-bench-your-team-your-most-relevant-ttps-5b9620457ef4}{MITRE AT\&CK Workbench} & + Create or extend ATT\&CK data in a local knowledge base & + Host and manage a customized copy of the ATT\&CK knowledge base. This local + copy of the ATT\&CK knowledge base can be extended with new or updated + techniques, tactics, mitigation groups, and software that is specific to your + organization. \\ + \hline + \href{https://atlas.mitre.org/}{MITRE ATLAS} & + MITRE ATLAS (Adversarial Threat Landscape for Artificial-Intelligence Systems) + is a knowledge base of adversary tactics, techniques, and case studies for + machine learning (ML) systems based on real-world observations, demonstrations + from ML red teams and security groups, and the state of the possible from + academic research & + Use it to map known ML vulnerabilities and map checks and controls for + proposed projects or existing systems. \\ + \hline + \href{https://mitre-engenuity.org/cybersecurity/center-for-threat-informed-defense/attack-powered-suit/}{MITRE ATT\&CK Powered Suit} & + ATT\&CK Powered Suit is a browser extension that puts the MITRE ATT\&CK + knowledge base at your fingertips. & + Add to your browser to quickly search for tactics, techniques, and more + without disrupting your workflow. \\ + \hline + \href{https://mitre-engenuity.org/cybersecurity/center-for-threat-informed-defense/our-work/threat-report-attck-mapper-tram/}{The Threat Report ATT\&CK Mapper (TRAM)} & + Automates TTP Identification in CTI Reports & + Mapping TTPs found in CTI reports to MITRE ATT\&CK is difficult, error prone, + and time-consuming. TRAM uses LLMs to automate this process for the 50 most + common techniques. Supports Juypter notebooks. \\ + \hline + \href{https://center-for-threat-informed-defense.github.io/attack-flow/}{Attack Flow v2.1.0} & + Attack Flow is a language for describing how cyber adversaries combine and + sequence various offensive techniques to achieve their goals. & + Attack Flow helps visualize how an attacker uses a technique, so defenders + and leaders understand how adversaries operate and improve their own + defensive posture. \\ + \hline + \href{https://caldera.mitre.org/}{MITRE Caldera} & + A cyber security platform (framework) designed to easily automate adversary + emulation, assist manual red-teams, and automate incident response. & + \href{https://caldera.readthedocs.io/en/latest/Plugin-library.html}{Plugins} are available for Caldera that help to expand the core capabilities + of the framework and provide additional functionality, including agents, + reporting, collections of TTPs and others + \\ + \hline + \href{https://github.com/mitre-atlas/arsenal}{CALDERA plugin: Arsenal} & + A plugin developed for adversary emulation of AI-enabled systems. & + This plugin provides TTPs defined in MITRE ATLAS to interface with CALDERA. \\ + \hline + \href{https://github.com/redcanaryco/atomic-red-team}{Atomic Red Team} & + Library of tests mapped to the MITRE ATT\&CK framework. & + Use to validate and test controls in an environment. Security teams can use + Atomic Red Team to quickly, portably, and reproducibly test their environments. + You can execute atomic tests directly from the command line; no installation + is required. + \\ + \hline + \href{https://mitre-engenuity.org/cybersecurity/center-for-threat-informed-defense/our-work/cti-blueprints/}{MITRE CTI Blueprints} & + Automates Cyber Threat Intelligence reporting. & + CTI Blueprints helps Cyber Threat Intelligence (CTI) analysts create + high-quality, actionable reports more consistently and efficiently. + \\ + \hline + %%% TABLE DATA ENDS HERE + \caption{MITRE Resources} + \label{tab:mitre-resources} +\end{longtable} \ No newline at end of file diff --git a/llm-top-10-governance-doc/fragments/resources/owasp.tex b/llm-top-10-governance-doc/fragments/resources/owasp.tex new file mode 100644 index 00000000..bca0a9aa --- /dev/null +++ b/llm-top-10-governance-doc/fragments/resources/owasp.tex @@ -0,0 +1,147 @@ +% !TEX root = owasp-doc.tex +% \clearpage %%% Since this is the first section in the resources chapter, we don't clear the page +\textbf{OWASP Resources} +Using LLM solutions expands an organization's attack surface and presents new +challenges, requiring special tactics and defenses. It also poses problems that +are similar to known issues, and there are already established cybersecurity +procedures and mitigations. Integrating LLM cybersecurity with an organization's +established cybersecurity controls, processes, and procedures allows an +organization to reduce its vulnerability to threats. How they integrate with +each other is available at the +\href{https://owasp.org/www-project-integration-standards/}{OWASP Integration Standards}. +%%% TABLE FORMATTING +\setlength\LTleft{0pt} +\setlength\LTright{0pt} +\begin{longtable}[c]{|p{0.25\textwidth}|p{0.25\textwidth}|p{0.35\textwidth}|} + %%% Header and footer information + \hline + \rowcolor{owasplightpurple} + \textbf{OWASP Resource} & + \textbf{Description} & + \textbf{Why It Is Recommended \& Where To Use It} \\ + \hline + \endfirsthead + \hline + \rowcolor{owasplightpurple} + \textbf{OWASP Resource} & + \textbf{Description} & + \textbf{Why It Is Recommended \& Where To Use It} \\ + \hline + \endhead + \endfoot + %%% TABLE DATA GOES HERE + \href{https://owasp.org/www-project-samm/}{OWASP SAMM}& + Software Assurance Maturity Model & + Provides an effective and measurable way to analyze and improve an + organization's secure development lifecycle. SAMM supports the complete + software lifecycle. It is interative and risk-driven, enabling organizations + to identify and prioritize gaps in secure software development so resources + for improving the process can be dedicated where efforts have the greatest + improvement impact. \\ + \hline + \href{https://owasp.org/www-project-ai-security-and-privacy-guide/}{OWASP AI Security and Privacy Guide} & + OWASP Project with a goal of connecting worldwide for an exchange on AI + security, fostering standards alignment, and driving collaboration. & + The OWASP AI Security and Privacy Guide is a comprehensive list of the most + important AI security and privacy considerations. It is meant to be a + comprehensive resource for developers, security researchers, and security + consultants to verify the security and privacy of AI systems. \\ + \hline + \href{https://owasp.org/www-project-ai-security/}{OWASP AI Exchange} & + OWASP AI Exchange is the intake method for the OWASP AI Security and Privacy Guide. & + The AI Exchange is the primary intake method used by OWASP to drive the direction of + the OWASP AI Security and Privacy Guide. \\ + \hline + \href{https://mltop10.info/}{OWASP Machine Learning Security Top 10} & + OWASP Machine Learning Security Top 10 security issues of machine learning systems. & + The OWASP Machine Learning Security Top 10 is a community-driven effort to + collect and present the most important security issues of machine learning + systems in a format that is easy to understand by both a security expert and + a data scientist. This project includes the ML Top 10 and is a live working + document that provides clear and actionable insights on designing, creating, + testing, and procuring secure and privacy-preserving AI systems. It is the + best OWASP resource for AI global regulatory and privacy information.\\ + \hline + \href{https://www.opencre.org/}{OpenCRE} & + OpenCRE (Common Requirement Enumeration) is the interactive content-linking + platform for uniting security standards and guidelines into one overview. & + Use this site to search for standards. You can search by standard name or by + control type. \\ + \hline + \href{https://owasp.org/www-community/Threat_Modeling}{OWASP Threat Modeling} & + A structured, formal process for threat modeling of an application & + Learn everything about Threat Modeling which is a structured representation + of all the information that affects the security of an application. \\ + \hline + \href{https://owasp.org/www-project-cyclonedx/}{OWASP CycloneDX} & + OWASP CycloneDX is a full-stack Bill of Materials (BOM) standard that + provides advanced supply chain capabilities for cyber risk reduction. & + Modern software is assembled using third-party and open source components. + They are glued together in complex and unique ways and integrated with + original code to achieve the desired functionality. An SBOM provides an + accurate inventory of all components which enables organizations to identify + risk, allows for greater transparency, and enables rapid impact analysis. + \href{https://www.nist.gov/itl/executive-order-14028-improving-nations-cybersecurity/software-security-supply-chains-software-1}{EO 14028} + provided minimum requirements for SBOM for federal systems. \\ + \hline + \href{https://scvs.owasp.org/}{OWASP Software Component Verification Standard (SCVS) } & + A community-driven effort to establish a framework for identifying activities, + controls, and best practices can help in identifying and reducing risk in a + software supply chain. & + Use SCVS to develop a common set of activities, controls, and best-practices + that can reduce risk in a software supply chain and identify a baseline and + path to mature software supply chain vigilance. \\ + \hline + \href{https://owasp.org/www-project-api-security/}{OWASP API Security Project } & + API Security focuses on strategies and solutions to understand and mitigate + the unique vulnerabilities and security risks of Application Programming + Interfaces (APIs) & + APIs are a foundational element of connecting applications, and mitigating + misconfigurations or vulnerabilities is mandatory to protect users and + organizations. Use for security testing and red teaming the build and + production environments. \\ + \hline + \href{https://owasp.org/www-project-application-security-verification-standard/}{OWASP Application Security Verification Standard ASVS} & + Application Security Verification Standard (ASVS) Project provides a basis + for testing web application technical security controls and also provides + developers with a list of requirements for secure development. & + Cookbook for web application security requirements, security testing, and + metrics. Use to establish security user stories and security use case release + testing. \\ + \hline + \href{https://owasp.org/www-project-threat-and-safeguard-matrix/}{OWASP Threat and Safeguard Matrix (TaSM)} & + An action oriented view to safeguard and enable the business & + This matrix allows a company to overlay its major threats with the NIST Cyber + Security Framework Functions (Identify, Protect, Detect, Respond, \& Recover) + to build a robust security plan. Use it as a dashboard to track and report on + security across the organization. \\ + \hline + \href{https://www.defectdojo.com/}{Defect Dojo} & + An open source vulnerability management tool that streamlines the testing + process by offering templating, report generation, metrics, and baseline + self-service tools. & + Use Defect Dojo to reduce the time for logging vulnerabilities with templates + for vulnerabilities, imports for common vulnerability scanners, report + generation, and metrics. \\ + \hline + %%% TABLE DATA ENDS HERE + \caption{OWASP Resources} + \label{tab:owasp-resources} +\end{longtable} + +\textbf{OWASP Top 10 for Large Language Model Applications} +\begin{figure}[ht] + \centering + \includegraphics[width=0.8\textwidth]{owasp_top_10_llm_highlevel} + \caption{Image of OWASP Top 10 for Large Language Model Applications} + \label{fig:owasp-top-10-llm-highlevel} +\end{figure} + +\clearpage +\textbf{OWASP Top 10 for Large Language Model Applications Visualized} +\begin{figure}[ht] + \centering + \includegraphics[width=0.8\textwidth]{owasp_top_10_llm_app_arch} + \caption{Image of OWASP Top 10 for Large Language Model Applications Visualized} + \label{fig:owasp-top-10-llm-visualized} +\end{figure} \ No newline at end of file diff --git a/llm-top-10-governance-doc/owasp-doc.cls b/llm-top-10-governance-doc/owasp-doc.cls new file mode 100755 index 00000000..b882643a --- /dev/null +++ b/llm-top-10-governance-doc/owasp-doc.cls @@ -0,0 +1,508 @@ +%%% ============================================================================ +%%% This LaTeX document class is based on the LaTeX Templates port of the +%%% Legrand Orange Book Template by Mathias Legrand. It has been modified +%%% to meet the OWASP Documentation Project's style guide. +%%% The original template is available at: +%%% https://www.latextemplates.com/template/legrand-orange-book +%%% +%%% This template is licensed under a CC BY-NC-SA 4.0 license. +%%% To see what this license means for your use of the template, visit: +%%% https://creativecommons.org/licenses/by-nc-sa/4.0/ +%%% ============================================================================ + +\def\headerimagefile{Header_Blue_Net} + +\NeedsTeXFormat{LaTeX2e} +\ProvidesClass{owasp-doc}[2023-11-29 The OWASP Doc 0.1] +\DeclareOption*{\PassOptionsToClass{\CurrentOption}{book}} % Pass through any options to the base class +\ProcessOptions\relax % Process given options +\LoadClass{book} % Load the base class + +%------------------------------------------------------------------------------- +% REQUIRED PACKAGES +%------------------------------------------------------------------------------- +\usepackage{graphicx} % Required for including pictures +\graphicspath{{assets/}} % Specifies the root path for images + +\usepackage{float} % Required for controlling float positions +\usepackage[usenames, svgnames, table]{xcolor} % Required for defining and using custom colors +\usepackage{calc} % Used for spacing the index letter headings correctly +\usepackage{changepage} % Required for temporarily indenting text blocks +\usepackage[toc]{appendix} % Required for the appendices chapters +\usepackage{etoolbox} % Required for conditional logic and easily changing commands +\usepackage{eso-pic} % Required for specifying an image background in the cover page +\usepackage[nochapter, tablegrid]{vhistory} % Required for changelog +\usepackage{enumitem} % Customize lists +\setlist{nolistsep} % Reduce spacing between bullet points and numbered lists +% \usepackage{booktabs} % Required for nicer horizontal rules in tables +\usepackage{array} +% \usepackage{lscape} % Required for landscape tables +\usepackage{longtable} % Tables which may stretch over more than 1 page +\usepackage{ltxtable} +\usepackage{tabularx} % Required for tables with adjustable-width columns +\usepackage[none]{hyphenat} % Hyphenates words to split over two lines +\usepackage{listings} % Source code formatting and highlighting +\usepackage{color} % Required for syntax highlighting +\usepackage{colortbl} % Required for coloring table cells +\usepackage[skip=\baselineskip]{parskip} % Required for sane paragraph spacing +\usepackage{lmodern} +\usepackage{kpfonts} +\usepackage[export]{adjustbox} + +% =================================================== +% Color Definitions +% =================================================== + +% branding -- uses hex values from libre office colors: +% https://github.com/LibreOffice/core/blob/master/extras/source/palettes/standard.soc +\definecolor{urlblue}{HTML}{3465a4} % Dark Blue 1 +\definecolor{owaspdarkpurple}{HTML}{5b277d} % Dark Indigo 1 +\definecolor{owasplightpurple}{HTML}{b7b3ca} % Light Indigo 3 +\definecolor{owasplightgrey}{HTML}{eeeeee} % Light Grey 4 +\definecolor{owaspdarkgrey}{HTML}{666666} % Dark Grey 1 +\definecolor{owaspred}{HTML}{f10d0c} % Dark Red 1 +\definecolor{owasporange}{HTML}{ea7500} % Dark Orange 1 +\definecolor{owaspyellow}{HTML}{ffff38} % Light Yellow 1 +\definecolor{owaspgold}{HTML}{ffbf00} % Gold +\definecolor{owasplightgreen}{HTML}{5eb91e} % Dark Lime 1 +\definecolor{owaspdarkgreen}{HTML}{069a2e} % Dark Green 1 + +% Syntax highlighting color scheme +% -- this is based on pygments defaults +% https://github.com/richleland/pygments-css/blob/master/default.css +\definecolor{Brown}{HTML}{773714} +\definecolor{OliveGreen}{HTML}{3d6624} +\definecolor{CadetBlue}{HTML}{244166} +\definecolor{nsclass}{HTML}{0000ff} +\definecolor{atnotation}{HTML}{008000} +\definecolor{comment}{HTML}{408080} +\definecolor{import}{HTML}{008000} +\definecolor{string}{HTML}{ba2121} +\definecolor{class}{HTML}{0000ff} +\definecolor{custommethod}{HTML}{008000} + +% Misc. colors +\definecolor{Apricot}{HTML}{ffb786} + +% Vulnerability header colors +% \definecolor{vcrit}{HTML}{e50000} +% \definecolor{vhigh}{HTML}{e52d00} +% \definecolor{vmed}{HTML}{dde500} +% \definecolor{vlow}{HTML}{7ee500} +% \definecolor{vinfo}{HTML}{aaaaaa} + +% allow bold font to be colored +% example: \colorbf{owaspdarkpurple}{some purple bold text} +\newcommand\owaspbf[1]{\textcolor{owaspdarkpurple}{\textbf{#1}}} + + +%------------------------------------------------------------------------------- +% MARGINS +%------------------------------------------------------------------------------- +\usepackage{geometry} % Required for adjusting page dimensions and margins +\geometry{ + top=2.5cm, % Top margin + bottom=2.5cm, % Bottom margin + inner=2.5cm, % Inner margin (left on odd pages, right on even or left in oneside mode) + outer=2.5cm, % Outer margin (right on odd pages, left on even or right in oneside mode) + headsep=10pt, % Space from the top margin to the baseline of the header + headheight=0.5cm, % Header height + footskip=1.4cm, % Space from the bottom margin to the baseline of the footer + columnsep=1cm, % Horizontal space between columns when in two column mode + %showframe, % Uncomment to show how the type block is set on the page +} + +\setlength{\parindent}{0pt} % Disable paragraph indentation + + + +%------------------------------------------------------------------------------- +% FONTS +%------------------------------------------------------------------------------- +% \usepackage{avant} % Use the Avantgarde font for headings +\usepackage[sfdefault]{roboto} % Use the Roboto font by default +\usepackage{microtype} % Improve typography +\usepackage[utf8]{inputenc} % Required for inputting international characters +\usepackage[T1]{fontenc} % Output font encoding for international characters +\usepackage{mathptmx} % Use the Adobe Times Roman as the default text font + % together with math symbols from the Symbol, Chancery + % and Computer Modern fonts + +% adds the OWASP logo to the bottom corner of the page +\newcommand{\owasplogobottomright}{% + \AddToShipoutPicture*{\put(375,15){\includegraphics[width=80mm]{owasp_logo.png}}} +} + +% creates the header image on new sections +\newcommand{\headerimage}{% + \clearpage + \AddToShipoutPicture*{% + \AtPageUpperLeft{% + \raisebox{-\height}{% + \includegraphics[width=\paperwidth, height=1in]{\headerimagefile}% + }% + }% + }% +} + +%---------------------------------------------------------------------------------------- +% HEADERS AND FOOTERS +%---------------------------------------------------------------------------------------- + +\usepackage{fancyhdr} % Required for customizing headers and footers +\pagestyle{fancy} % Enable the custom headers and footers + +% Top horizontal rule thickness +\renewcommand{\headrulewidth}{0.5pt} + +% Redefine the styling of the current chapter name in the header +\renewcommand{\chaptermark}[1]{\markboth{\sffamily\normalsize\bfseries\chaptername\ \thechapter.~#1}{}} + +% Redefine the styling of the current section name in the header +\renewcommand{\sectionmark}[1]{\markright{\sffamily\normalsize\thesection~#1}{}} + +% Clear default headers and footers +\fancyhf{} + +\fancyhead[LE, RO]{\sffamily\thepage} % Header for left even pages and right odd pages +\fancyhead[LO]{\rightmark} % Header for left odd pages +\fancyhead[RE]{\leftmark} % Header for right even pages + +% Style for when a plain pagestyle is specified +\fancypagestyle{plain}{ + \fancyhead{} % Clear headers + \renewcommand{\headrulewidth}{0pt} % Remove header rule +} + +\usepackage{emptypage} % removes headers and footers on empty pages between chapters + + +%---------------------------------------------------------------------------------------- +% SECTION STYLING +%---------------------------------------------------------------------------------------- + +\setcounter{secnumdepth}{3} % Number sections down to subsubsections, a value of 2 will number to subsections only and 1 to sections only + +\usepackage[ + explicit, % Specify section names explicitly in \titleformat with #1 + newparttoc, % Required for writing the part toc entry number in a standard way (overriding default LaTeX behavior) so the titletoc package can style it +]{titlesec} % Required for modifying sections + +%------------------------------------------- + +% % Command to create and output the parts pages +% \newcommand{\outputpartpage}[2]{ +% \begin{tikzpicture}[remember picture, overlay] +% \node[anchor=center, rectangle, fill=owaspdarkpurple!20, inner sep=0pt, minimum height=\paperheight, minimum width=\paperwidth] at (current page.center) {}; % Background color +% \node[anchor=north west, inner sep=0pt, xshift=\Gm@lmargin, yshift=-\Gm@tmargin] at (current page.north west) {\fontsize{220pt}{220pt}\selectfont\sffamily\bfseries\textcolor{owaspdarkpurple!40}{#1}}; % Part number +% \node[anchor=north east, inner sep=0pt, xshift=-\Gm@rmargin, yshift=-\Gm@tmargin] at (current page.north east) {\fontsize{30pt}{30pt}\selectfont\sffamily\bfseries\textcolor{white}{\strut #2}}; % Part title +% \node[anchor=south east, inner sep=0pt, xshift=-\Gm@rmargin, yshift=\Gm@bmargin] at (current page.south east) { % Mini table of contents +% \parbox[t][][t]{8.5cm}{ % Width of box holding the mini ToC +% \printcontents[part]{l}{0}{\setcounter{tocdepth}{1}} % Display the mini table of contents showing chapters and sections, change tocdepth to 2 to also show subsections or 0 to only show chapters +% } +% }; +% \end{tikzpicture} +% } + +\titleformat + {\chapter} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {} % Format of the whole section + {\filright\huge \textcolor{owaspdarkpurple}{#1}} % Format of the section label + {0em} % Space between the title and label + {} % Code before the label + [] % Code after the label + +%------------------------------------------- + +\titleformat + {\section} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily\Large} % Format of the whole section + {\llap{{}\hspace{1em}} #1} % Format of the section label + {0cm} % Space between the title and label + {} % Code before the label + [] % Code after the label + +\titleformat + {name=\section, numberless} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily\Large} % Format of the whole section + {#1} % Format of the section label + {0cm} % Space between the title and label + {} % Code before the label + [] % Code after the label + +\titlespacing*{\section}{0pt}{12pt}{6pt} % Spacing around section titles, the order is: left, before and after + +%------------------------------------------- + +\titleformat + {\subsection} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily\large} % Format of the whole section + {\llap{\textcolor{owaspdarkpurple}{\thesubsection}\hspace{1em}}#1} % Format of the section label + {0cm} % Space between the title and label + {} % Code before the label + [] % Code after the label + +\titleformat + {name=\subsection, numberless} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily\large} % Format of the whole section + {#1} % Format of the section label + {0cm} % Space between the title and label + {} % Code before the label + [] % Code after the label + +\titlespacing*{\subsection}{0pt}{12pt}{4pt} % Spacing around section titles, the order is: left, before and after + +%------------------------------------------- + +\titleformat + {\subsubsection} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily} % Format of the whole section + {\llap{{\thesubsubsection}\hspace{1em}}#1} % Format of the section label + {0cm} % Space between the title and label + {} % Code before the label + [] % Code after the label + +\titleformat + {name=\subsubsection, numberless} % Section type being modified + [block] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily} % Format of the whole section + {#1} % Format of the section label + {0cm} % Space between the title and label + {} % Code before the label + [] % Code after the label + +\titlespacing*{\subsubsection}{0pt}{8pt}{4pt} % Spacing around section titles, the order is: left, before and after + +%------------------------------------------- + +\titleformat + {\paragraph} % Section type being modified + [runin] % Shape type, can be: hang, block, display, runin, leftmargin, rightmargin, drop, wrap, frame + {\bfseries\sffamily} % Format of the whole section + {} % Format of the section label + {0cm} % Space between the title and label + {#1} % Code before the label + [] % Code after the label + +\titlespacing*{\paragraph}{0pt}{0pt}{6pt} % Spacing around section titles, the order is: left, before and after + +%---------------------------------------------------------------------------------------- +% TABLE OF CONTENTS +%---------------------------------------------------------------------------------------- +\usepackage{titletoc} % Required for manipulating the table of contents +\setcounter{tocdepth}{2} % Display down to subsections in the table of contents +\contentsmargin{0cm} % Removes the default right margin + +%------------------------------------------------ + +% Styling of numbered parts in the table of contents +\newcommand{\tocentrypartnumbered}[1]{% + \setlength\fboxsep{0pt}% Remove box padding + \contentslabel[% + % Part number box + \colorbox{owaspdarkpurple!20}{% Background color + \strut\parbox[c][.7cm]{1.1cm}{% Box size + \color{owaspdarkpurple!70}\Large\sffamily\bfseries\centering\thecontentslabel% Part number + }% + }% + \hspace{4pt}% + % Part title box + \colorbox{owaspdarkpurple!40}{% Background color + \strut\parbox[c][.7cm]{\linewidth-1.25cm}{% Box size + \centering\Large\sffamily #1% Part title + }% + }% + ]{1.25cm} +} + +% Styling of unnumbered parts in the table of contents +\newcommand{\tocentrypartunnumbered}[1]{% + \setlength\fboxsep{0pt}% Remove box padding + \contentslabel[% + % Part title box + \colorbox{owaspdarkpurple!40}{% Background color + \strut\parbox[c][.7cm]{\linewidth}{% Box size + \centering\Large\sffamily #1% Part title + }% + }% + ]{1.25cm} +} + +\titlecontents{part} % Section type being modified + [1.25cm] % Left indentation + {\addvspace{20pt}\Large\sffamily\bfseries\hypersetup{linkcolor=owaspdarkpurple}} % Before code + {\tocentrypartnumbered} % Formatting of numbered sections of this type + {\tocentrypartunnumbered} % Formatting of numberless sections of this type + {} % Formatting of the filler to the right of the heading and the page number + [] % After code + +%------------------------------------------------ + +\titlecontents{chapter} % Section type being modified + [1.25cm] % Left indentation + {\addvspace{12pt}\large\sffamily\bfseries\hypersetup{linkcolor=owaspdarkpurple}} % Before code + {\color{owaspdarkpurple}\contentslabel[\Large\thecontentslabel]{1.25cm}} % Formatting of numbered sections of this type + {} % Formatting of numberless sections of this type + {\color{owaspdarkpurple!60}\normalsize\;\titlerule*[6pt]{.}\;\color{owaspdarkpurple}\thecontentspage} % Formatting of the filler to the right of the heading and the page number + [] % After code + +%------------------------------------------------ + +\titlecontents{section} % Section type being modified + [1.25cm] % Left indentation + {\addvspace{3pt}\sffamily\bfseries} % Before code + {\contentslabel[\thecontentslabel]{1.25cm}} % Formatting of numbered sections of this type + {} % Formatting of numberless sections of this type + {~{\normalfont\titlerule*[6pt]{.}}~\thecontentspage} % Formatting of the filler to the right of the heading and the page number + [] % After code + +%------------------------------------------------ + +\titlecontents{subsection} % Section type being modified + [1.25cm] % Left indentation + {\addvspace{1pt}\sffamily\small} % Before code + {\contentslabel[\thecontentslabel]{1.25cm}} % Formatting of numbered sections of this type + {} % Formatting of numberless sections of this type + {~\titlerule*[6pt]{.}~\thecontentspage} % Formatting of the filler to the right of the heading and the page number + [] % After code + +%------------------------------------------------ + +\titlecontents{subsubsection} % Section type being modified + [1.75cm] % Left indentation + {\sffamily\small} % Before code + {\contentslabel[\thecontentslabel]{1.75cm}} % Formatting of numbered sections of this type + {} % Formatting of numberless sections of this type + {~\titlerule*[6pt]{.}~\thecontentspage} % Formatting of the filler to the right of the heading and the page number + [] % After code + +%------------------------------------------------ + +% List of figures entries +\titlecontents{figure} % Section type being modified + [0cm] % Left indentation + {\addvspace{2pt}\sffamily\small} % Before code + {\thecontentslabel\hspace*{1em}} % Formatting of numbered sections of this type + {} % Formatting of numberless sections of this type + {~\titlerule*[6pt]{.}~\thecontentspage} % Formatting of the filler to the right of the heading and the page number + [] % After code + +%------------------------------------------------ + +% List of tables entries +\titlecontents{table} % Section type being modified + [0cm] % Left indentation + {\addvspace{2pt}\sffamily\small} % Before code + {\thecontentslabel\hspace*{1em}} % Formatting of numbered sections of this type + {} % Formatting of numberless sections of this type + {~\titlerule*[6pt]{.}~\thecontentspage} % Formatting of the filler to the right of the heading and the page number + [] % After code + + +%---------------------------------------------------------------------------------------- +% INDEX +%---------------------------------------------------------------------------------------- + +\usepackage{makeidx} % Required to make an index + +\makeindex % Tells LaTeX to create the files required for indexing + +% Modify the command to output the index to output the correctly styled Index name to the page headers +\patchcmd{\theindex} + {\MakeUppercase\indexname} % Find + {\sffamily\normalsize\bfseries\indexname} % Replace + {} + {} + +%---------------------------------------------------------------------------------------- +% TABLES +%---------------------------------------------------------------------------------------- + +% \usepackage{array} % Required for manipulating table elements + +% \newcolumntype{R}[1]{>{\raggedleft\arraybackslash}p{#1}} % Define a new right-aligned paragraph column type +% \newcolumntype{L}[1]{>{\raggedright\arraybackslash}p{#1}} % Define a new left-aligned (no justification) paragraph column type +% \newcolumntype{C}[1]{>{\centering\arraybackslash}p{#1}} % Define a new centered paragraph column type + +%---------------------------------------------------------------------------------------- +% LISTS +%---------------------------------------------------------------------------------------- + +\usepackage{enumitem} % Required for list customization +\setlist{} % Customize spacing inside and outside lists + +\renewcommand{\labelenumi}{\arabic{enumi}.} % Customize the first level numbered list characters +\renewcommand{\labelenumii}{\alph{enumii}.} % Customize the second level numbered list characters +\renewcommand{\labelenumiii}{\roman{enumiii}.} % Customize the third level numbered list characters + +%------------------------------------------------ + +\renewcommand{\labelitemi}{\raisebox{-2pt}{\Large \textbullet}} % Customize the first level bullet point character +\renewcommand{\labelitemii}{--} % Customize the second level bullet point character +\renewcommand{\labelitemiii}{$\circ$} % Customize the third level bullet point character + + +\newlist{checklist}{itemize}{2} +\setlist[checklist]{label=$\square$} + +%---------------------------------------------------------------------------------------- +% LINKS +%---------------------------------------------------------------------------------------- + +\usepackage{hyperref} % Required for links +\hypersetup{ + colorlinks, % Whether to color links (a thin box is output around links if this is false) + hidelinks, % Hide the default boxes around links + urlcolor=urlblue, % Color for \url and \href links + linkcolor=urlblue, % Color for \ref/\nameref links + citecolor=owaspdarkpurple, % Color for reference citations like \cite{} + hyperindex=true, % Adds links from the page numbers in the index to the relevant page + linktoc=all, % Link from section names and page numbers in the table of contents +} + +%---------------------------------------------------------------------------------------- +% BOOKMARKS +%---------------------------------------------------------------------------------------- + +\usepackage{bookmark} % Required for customizing PDF bookmarks + +\bookmarksetup{ + open, % Open all bookmark drawers automatically + numbered, % Add section numbers to bookmarks + addtohook={% + \ifnum\bookmarkget{level}=-1 % Customize part bookmarks + \bookmarksetup{color=owaspdarkpurple, bold}% Bold and colored + \fi + \ifnum\bookmarkget{level}=0 % Customize chapter bookmarks + \bookmarksetup{bold}% Bold + \fi + } +} + +%---------------------------------------------------------------------------------------- +% TITLE PAGE +%---------------------------------------------------------------------------------------- + +\renewcommand{\titlepage}[2]{ + \thispagestyle{empty} % Suppress headers and footers on the title page + + \begin{tikzpicture}[remember picture, overlay] + \node [inner sep=0pt] at (current page.center) {#1}; % Background image + \node [anchor=center, inner sep=1.25cm, rectangle, fill=owaspdarkpurple!30!white, fill opacity=0.6, text opacity=1, minimum height=0.2\paperheight, minimum width=\paperwidth, text width=0.8\paperwidth] at (current page.center) {#2}; % Title highlight box with title(s) and author(s) + \end{tikzpicture} + + \newpage +} + + +%---------------------------------------------------------------------------------------- +% LOAD PACKAGES THAT CLASH WITH OTHER PACKAGES +%---------------------------------------------------------------------------------------- + +\usepackage{tikz} % Required for drawing custom shapes and positioning elements absolutely on the page \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/changelog-disclaimer.tex b/llm-top-10-governance-doc/sections/changelog-disclaimer.tex new file mode 100755 index 00000000..873b5e5a --- /dev/null +++ b/llm-top-10-governance-doc/sections/changelog-disclaimer.tex @@ -0,0 +1,12 @@ +% !TEX root = owasp-doc.tex +%%% ================================================ +%%% CHANGELOG & DISCLAIMER +%%% ================================================ +\vfill +% changelog +\input{fragments/changelog} +\vfill +% disclaimer +\input{fragments/disclaimer} +\vspace{2cm} +\owasplogobottomright \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/checklist.tex b/llm-top-10-governance-doc/sections/checklist.tex new file mode 100644 index 00000000..667c822a --- /dev/null +++ b/llm-top-10-governance-doc/sections/checklist.tex @@ -0,0 +1,154 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% CHECK LIST +% ================================================ +\headerimage +\chapter{Check List} +\section{Adversarial Risk} +Adversarial Risk includes competitors and attackers. + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Scrutinize how competitors are investing in artificial intelligence. Although there are risks in AI adoption, there are also business benefits that may impact future market positions. + \item Threat Model: how attackers may accelerate exploit attacks against the organization, employees, executives, or users. + \item Threat models potential attacks on customers or clients through spoofing and generative AI. + \item Investigate the impact of current controls, such as password resets, which use voice recognition. + \item Update the Incident Response Plan and playbooks for LLM incidents. +\end{checklist} +\end{minipage} + +\section{AI Asset Inventory} +An AI asset inventory should apply to both internally developed and external or +third-party solutions. + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Catalog existing AI services, tools, and owners. Designate a tag in asset management for specific inventory. + \item Include AI components in the Software Bill of Material (SBOM), a comprehensive list of all the software components, dependencies, and metadata associated with applications. + \item Catalog AI data sources and the sensitivity of the data (protected, confidential, public) + \item Establish if pen testing or red teaming of deployed AI solutions is required to determine the current attack surface risk. + \item Create an AI solution onboarding process. + \item Ensure skilled IT admin staff is available either internally or externally, in accordance to the SBoM +\end{checklist} +\end{minipage} + +\section{AI Security and Privacy Training} + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Train all users on ethics, responsibility, and legal issues such as warranty, license, and copyright. + \item Update security awareness training to include GenAI related threats. Voice cloning and image cloning, as well as in anticipation of increased spear phishing attacks + \item Any adopted GenAI solutions should include training for both DevOps and cybersecurity for the deployment pipeline to ensure AI safety and security assurances. +\end{checklist} +\end{minipage} + +\section{Establish Business Cases} +Solid business cases are essential to determining the business value of any +proposed AI solution,balancing risk and benefits, and evaluating and testing +return on investment. There are an enormous number of potential use cases; a +few examples are provided. + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Enhance customer experience + \item Better operational efficiency + \item Better knowledge management + \item Enhanced innovation + \item Market Research and Competitor Analysis + \item Document creation, translation, summarization, and analysis +\end{checklist} +\end{minipage} + +\section{Governance} +Corporate governance in LLM is needed to provide organizations with transparency +and accountability. Identifying AI platform or process owners who are +potentially familiar with the technology or the selected use cases for the +business is not only advised but also necessary to ensure adequate reaction +speed that prevents collateral damages to well established enterprise digital +processes. + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Establish the organization\'s AI RACI chart (who is responsible, who is accountable, who should be consulted, and who should be informed) + \item Document and assign AI risk, risk assessments, and governance responsibility within the organization. + \item Establish data management policies, including technical enforcement, regarding data classification and usage limitations. Models should only leverage data classified for the minimum access level of any user of the system. For example, update the data protection policy to emphasize not to input protected or confidential data into nonbusiness-managed tools. + \item Create an AI Policy supported by established policy (e.g., standard of good conduct, data protection, software use) + \item Publish an acceptable use matrix for various generative AI tools for employees to use. + \item Document the sources and management of any data that the organization uses from the generative LLM models. +\end{checklist} +\end{minipage} + +\clearpage +\section{Legal} +Many of the legal implications of AI are undefined and potentially very costly. +An IT, security, and legal partnership is critical to identifying gaps and +addressing obscure decisions. + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Confirm product warranties are clear in the product development stream to assign who is responsible for product warranties with AI. + \item Review and update existing terms and conditions for any GenAI considerations. + \item Review AI EULA agreements. End-user license agreements for GenAI platforms are very different in how they handle user prompts, output rights and ownership, data privacy, compliance and liability, privacy, and limits on how output can be used. + \item Review existing AI-assisted tools used for code development. A chatbot\'s ability to write code can threaten a company\'s ownership rights to its own product if a chatbot is used to generate code for the product. For example, it could call into question the status and protection of the generated content and who holds the right to use the generated content. + \item Review any risks to intellectual property. Intellectual property generated by a chatbot could be in jeopardy if improperly obtained data was used during the generative process, which is subject to copyright, trademark, or patent protection. If AI products use infringing material, it creates a risk for the outputs of the AI, which may result in intellectual property infringement. + \item Review any contracts with indemnification provisions. Indemnification clauses try to put the responsibility for an event that leads to liability on the person who was more at fault for it or who had the best chance of stopping it. Establish guardrails to determine whether the provider of the AI or its user caused the event, giving rise to liability. + \item Review liability for potential injury and property damage caused by AI systems. + \item Review insurance coverage. Traditional (D\&O) liability and commercial general liability insurance policies are likely insufficient to fully protect AI use. + \item Identify any copyright issues. Human authorship is required for copyright. An organization may also be liable for plagiarism, propagation of bias, or intellectual property infringement if LLM tools are misused. + \item Ensure agreements are in place for contractors and appropriate use of AI for any development or provided services. + \item Restrict or prohibit the use of generative AI tools for employees or contractors where enforceable rights may be an issue or where there are IP infringement concerns. + \item Assess and AI solutions used for employee management or hiring could result in disparate treatment claims or disparate impact claims. + \item Make sure the AI solutions do not collect or share sensitive information without proper consent or authorization. +\end{checklist} +\end{minipage} + +\clearpage +\section{Regulatory} +The EU AI Act is anticipated to be the first comprehensive AI law but will +apply in 2025 at the earliest. The EU\'s General Data Protection Regulation +(GDPR) does not specifically address AI but includes rules for data collection, +data security, fairness and transparency, accuracy and reliability, and +accountability, which can impact GenAI use. In the United States, AI regulation +is included within broader consumer privacy laws. Ten US states have passed +laws or have laws that will go into effect by the end of 2023. + +Federal organizations such as the US Equal Employment Opportunity Commission +(EEOC), the Consumer Financial Protection Bureau (CFPB), the Federal Trade +Commission (FTC), and the US Department of Justice\'s Civil Rights Division +(DOJ) are closely monitoring hiring fairness. + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Determine State specific compliance requirements. + \item Determine compliance requirements for restricting electronic monitoring of employees and employment-related automated decision systems (Vermont) + \item Determine compliance requirements for consent for facial recognition and the AI video analysis required (Illinois, Maryland) + \item Review any AI tools in use or being considered for employee hiring or management. + \item Confirm the vendor\'s compliance with applicable AI laws and best practices. + \item Ask and document any products using AI during the hiring process. Ask how the model was trained, how it is monitored, and track any corrections made to avoid discrimination and bias. + \item Ask and document what accommodation options are included. + \item Ask and document whether the vendor collects confidential data. + \item Ask how the vendor or tool stores and deletes data and regulates the use of facial recognition and video analysis tools during pre-employment. + \item Review other organization-specific regulatory requirements with AI that may raise compliance issues. The Employee Retirement Income Security Act of 1974, for instance, has fiduciary duty requirements for retirement plans that a chatbot might not be able to meet. +\end{checklist} +\end{minipage} + +\section{Using or Implementing Large Language Model Solutions} + +\begin{minipage}{\linewidth} +\begin{checklist} + \item Threat Model: LLM components and architecture trust boundaries. + \item Data Security: Verify how data is classified and protected based on sensitivity, including personal and proprietary business data. (How are user permissions managed, and what safeguards are in place?) + \item Access Control: Implement least privilege access controls and implement defense-in-depth measures + \item Training Pipeline Security: Require rigorous control around training data governance, pipelines, models, and algorithms. + \item Input and Output Security: Evaluate input validation methods, as well as how outputs are filtered, sanitized, and approved. + \item Monitoring and Response: Map workflows, monitoring, and responses to understand automation, logging, and auditing. Confirm audit records are secure. + \item Include application testing, source code review, vulnerability assessments, and red teaming in the production release process. + \item Consider vulnerabilities in the LLM model solutions (Rezilion OSFF Scorecard). + \item Look into the effects of threats and attacks on LLM solutions, such as prompt injection, the release of sensitive information, and process manipulation. + \item Investigate the impact of attacks and threats to LLM models, including model poisoning, improper data handling, supply chain attacks, and model theft. + \item Supply Chain Security: Request third-party audits, penetration testing, and code reviews for third-party providers. (both initially and on an ongoing basis) + \item Infrastructure Security: How often does the vendor perform resilience testing? What are their SLAs in terms of availability, scalability, and performance? + \item Update incident response playbooks and include an LLM incident in tabletop exercises. + \item Identify or expand metrics to benchmark generative cybersecurity AI against other approaches to measure expected productivity improvements. +\end{checklist} +\end{minipage} \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/coverpage.tex b/llm-top-10-governance-doc/sections/coverpage.tex new file mode 100755 index 00000000..18525acb --- /dev/null +++ b/llm-top-10-governance-doc/sections/coverpage.tex @@ -0,0 +1,32 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% COVER PAGE +% ================================================ +\pagestyle{empty} +%%% Background images +\AddToShipoutPicture*{\put(0,0){\includegraphics{FrontCover.jpg}}} +\AddToShipoutPicture*{\put(0,0){\includegraphics[width=8.5in, height=11in]{owasp_wasp1.png}}} + +\vfill +\begingroup +%%% OWASP logo +\begin{figure}[t] +\includegraphics[width=0.6\textwidth,left]{owasp_logo.png} +\end{figure} +%%% Title +\begin{center} + \par\normalfont\fontsize{42}{42}\sffamily\selectfont + \vspace*{2.75cm} + \textbf{\projectName} +%%% Subtitle + \par\normalfont\fontsize{24}{24}\sffamily\selectfont + \vspace*{0.65cm} + \projectSubName +\end{center} +%%% Version and Publication Date +\vspace*{10cm} + \par\normalfont\fontsize{22}{22}\sffamily\selectfont + \textbf{\color{white}Version: \docVersion}\par + \par\normalfont\fontsize{14}{14}\sffamily\selectfont + \textit{\color{white}\textbf{Published}: \date}\par +\endgroup \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/llm-challenges.tex b/llm-top-10-governance-doc/sections/llm-challenges.tex new file mode 100644 index 00000000..e27bcb48 --- /dev/null +++ b/llm-top-10-governance-doc/sections/llm-challenges.tex @@ -0,0 +1,102 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% LLM Challenges +% ================================================ +\headerimage +\chapter{Large Language Model Challenges} +Large Language models face a number of serious and unique issues. One of the +most important is that while working with LLMs, the control and data planes +cannot be strictly isolated or separable. Another significant challenge is +that LLMs are nondeterministic by design, yielding a different outcome when +prompted or requested. It is not always a challenge, but LLMs employ semantic +search rather than keyword search. The key distinction between the two is that +the model's algorithm prioritizes the terms in its response. This is a +significant departure from how consumers have traditionally used technology, +and it has an impact on the consistency and reliability of the findings. +Hallucinations, emerging from the gaps and training flaws in the data the model +is trained on, are the result of this method. + +There are methods to improve reliability and reduce the attack surface for +jailbreaking, model tricking, and hallucinations, but there is a trade-off +between restrictions and utility in both cost and functionality. + +LLM use and applications increase an organization's attack surface. Some risks +associated with LLMs are unique, but many are familiar issues, such as the +known software bill of materials (SBOM), supply chain, data loss protection +(DLP), and authorized access. There are also increased risks not directly +related to GenAI, but GenAI increases the efficiency, capability, and +effectiveness of attacks. + +Adversaries are increasingly harnessing LLM and Generative AI tools to refine +and expedite traditional methods. These enhanced techniques allow them to +effortlessly craft new malware, potentially embedded with novel zero-day +vulnerabilities or designed to evade detection. They can also generate +sophisticated, unique, or tailored phishing schemes. The creation of convincing +deep fakes, whether video or audio, further facilitates their social +engineering ploys. Additionally, these tools enable them to execute intrusions +and develop innovative hacking utilities. It is very likely that in the future, +more “tailored” and compound use of AI technology by criminal actors will demand +specific responses and dedicated solutions for appropriate defense schemas. + +\clearpage +\section{LLM Threat Categories} +\begin{figure}[h] + \centering + \includegraphics[width=\textwidth]{ai_threat_map} + \caption{Image of types of AI threats} + \label{fig:ai-threat-map} +\end{figure} + +\section{Artificial Intelligence Security and Privacy Training} +Employees throughout organizations benefit from training to understand +artificial intelligence, generative artificial intelligence, and the future +potential consequences of building, buying, or utilizing LLMs. Training for +permissible use and security awareness should target all employees as well as +be more specialized for certain positions such as human resources, legal, +developers, data teams, and security teams. + +Fair use policies and healthy interaction are key aspects that, if incorporated +from the very start, will be a cornerstone to the success of future AI +cybersecurity awareness campaigns. This will necessarily imply the user's +knowledge of the basic rules for interaction as well as the ability to separate +good behavior from bad or unethical behavior. + +\section{Incorporate LLM Security and governance with Existing, Established Practices and Controls} +While AI and generated AI add a new dimension to cybersecurity, resilience, +privacy, and meeting legal and regulatory requirements, the best practices that +have been around for a long time are still the best way to find risks, test +them, fix them, and lower them. + +\begin{itemize} + \item The management of artificial intelligence systems is integrated with + existing organizational practices. + \item Apply existing privacy, governance, and security practices. +\end{itemize} + +\clearpage +\section{Fundamental Security Principles} +LLM capabilities introduce a different type of attack and attack surface. LLMs +are vulnerable to complex business logic bugs, such as prompt injection, +insecure plugin design, and remote code execution. Existing best practices are +the best way to solve these issues. An internal product security team that +understands secure software review, architecture, data governance, and +third-party assessments The cybersecurity team should also check how strong +the current controls are to find problems that could be made worse by LLM, +like voice cloning, impersonation, or getting around captchas. + +Accounting for the specific skills and competences developed in the last few +years around machine learning, NLP and NLU, deep Learning and lately, LLMs and +GenAI, it is advised to have skilled professionals with practice, knowledge, or +experience in these fields to side with security teams in adopting, at best, +and even shaping new potential analyses and responses to those issues. + +\section{Risk} +Reference to risk uses the ISO 31000 definition: Risk = "effect of uncertainty on objectives." +LLM risks included in the checklist include a targeted list of LLM risks that +address adversarial, safety, legal, regulatory, reputation, financial, and competitive risks. + +\section{Vulnerability and Mitigation Taxonomy} +Established methods of vulnerability classification and threat sharing are in +early development, such as Oval, STIX, threat sharing, and vulnerability +classification. The checklist anticipates calibrating with existing, +established, and accepted standards, such as CVE classification. \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/llm-strategy.tex b/llm-top-10-governance-doc/sections/llm-strategy.tex new file mode 100644 index 00000000..55da5f2c --- /dev/null +++ b/llm-top-10-governance-doc/sections/llm-strategy.tex @@ -0,0 +1,35 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% LLM Strategy +% ================================================ +\headerimage +\chapter{Determining LLM Strategy} +The acceleration of LLM applications has raised the visibility of all +artificial intelligence applications' organizational use. Recommendations for +policy, governance, and accountability should be considered holistically. + +The immediate LLM threats are the use of online tools, browser plugins, +third-party applications, the extended attack surface, and ways attackers can +leverage LLM tools to facilitate attacks. + +\begin{figure}[h] + \centering + \includegraphics[width=\textwidth]{ai_implementation_strategy} + \caption{Image of steps of LLM implementation} + \label{fig:llm-implementation-strategy} +\end{figure} + +\clearpage +\section{Deployment Strategy} +The scopes range from leveraging public consumer applications to training +proprietary models on private data. Factors like use case sensitivity, +capabilities needed, and resources available help determine the right balance +of convenience vs. control. But understanding these five model types provides a +framework for evaluating options. + +\begin{figure}[h] + \centering + \includegraphics[width=\textwidth]{ai_deployment_strategy} + \caption{Image of options for deployment strategy} + \label{fig:llm-deployment-strategy} +\end{figure} \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/main.tex b/llm-top-10-governance-doc/sections/main.tex new file mode 100644 index 00000000..8bf80422 --- /dev/null +++ b/llm-top-10-governance-doc/sections/main.tex @@ -0,0 +1,9 @@ +% !TEX root = owasp-doc.tex +%%% ================================================ +%%% MAIN SECTIONS +%%% ================================================ +\input{sections/overview} +\input{sections/llm-challenges} +\input{sections/llm-strategy} +\input{sections/checklist} +\input{sections/resources} \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/overview.tex b/llm-top-10-governance-doc/sections/overview.tex new file mode 100755 index 00000000..e6df7bab --- /dev/null +++ b/llm-top-10-governance-doc/sections/overview.tex @@ -0,0 +1,125 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% OVERVIEW +% ================================================ +\headerimage +\chapter{Overview} +Every internet user and business should prepare for the impact of a surge in +powerful generative artificial intelligence (GenAI) applications. GenAI holds +enormous promise and opportunities for discovery, efficiency, and driving +corporate growth across many industries and disciplines. However, as with any +strong new technology, it introduces new challenges to security and privacy. + +Artificial Intelligence, Machine Learning, Large Language Models, and Diffusion +Models have been in development and the focus of academic research for many +years. Recent improvements in training data availability, computer power, +GenAI capacity, and the release of solutions such as ChatGPT, ElevenLabs, +Midjourney, along with their broader availability outside of what previously +was a relatively isolated and specialized field, have led to its eruptive +growth. These advances in artificial intelligence (AI) emphasize the importance +of organizations developing plans to manage their engagement and use of AI +within their organization. + +\begin{itemize} + \item \textbf{Artificial intelligence} is a broad term that encompasses all fields of + computer science that enable machines to accomplish tasks that would normally + require human intelligence. Machine learning and generative AI are two + subcategories of AI. + \item \textbf{Machine learning} is a subset of AI that focuses on creating algorithms + that can learn from data. Machine learning algorithms are trained on a set of + data, and then they can use that data to make predictions or decisions about + new data. + \item \textbf{Generative AI} is a type of machine learning that focuses on creating + new data. Often, GenAI relies on the use of large language models to perform + the tasks needed to create the new data. + \item A \textbf{large language model (LLM)} is a type of AI program that uses machine + learning to perform natural language processing (NLP) tasks. LLMs are trained + on large data sets to understand, summarize, generate, and predict new + content. +\end{itemize} + +\clearpage +The diagram below shows the relationship of LLM to the field of AI generally: + +\begin{figure}[h] + \centering + \includegraphics[width=\textwidth]{ai_llm_relationship} + \caption{Image of LLM relationship within the field of Artificial Intelligence} + \label{fig:ai-llm-relationship} +\end{figure} + +Organizations will face new challenges defending and managing GenAI solutions. +Additionally, there is significant potential for accelerated threats from threat +actors who will use GenAI to augment attack techniques. + +Many applications within a business employ artificial intelligence applications, +such as human resource hiring, SPAM detection for email, behavioral analytics +for SIEM, and MDR apps. The primary focus of this document is on Large Language +Model applications, which can produce content. + +\clearpage +\section{Responsible and Trustworthy Artificial Intelligence} +As challenges and benefits of Artificial Intelligence emerge - and regulations +and laws are passed - the principles and pillars of responsible and trustworthy +AI usage are evolving from idealistic objects and concerns to established +standards. + +The \href{https://owasp.org/www-project-ai-security-and-privacy-guide/}{OWASP AI Security and Privacy Guide} +working group is monitoring these changes and addressing the broader and more +challenging considerations for all aspects of artificial intelligence. + +\begin{figure}[h] + \centering + \includegraphics[width=\textwidth]{trustworthy_ai} + \caption{Image credit \href{https://montrealethics.ai/}{Montreal AI Ethics Institute}} + \label{fig:trustworthy-ai} +\end{figure} + +\clearpage +\section{Who is This For?} +Executive, technology, cybersecurity, privacy, compliance, and legal leaders +must pay close attention to the fast GenAI technological transformation and +devise a strategy to benefit from opportunities while fighting against threats +and managing risks. + +This checklist is designed to assist these technology and business leaders in +quickly understanding the risks and benefits of using LLM, allowing them to +focus on developing a comprehensive list of essential areas and tasks required +to defend and protect the organization as they create a Large Language Model +strategy. + +Scenarios presented here include those that pertain to internal use of models +released commercially or those that are open sourced, as well as scenarios for +organizations that consume LLM services provided by third-parties. Resources +from MITRE Engenuity, OWASP, and others are referenced. + +The diagram below shows how these resources can be used to create a threat +informed defense strategy. + +\begin{figure}[h] + \centering + \includegraphics[width=\textwidth]{llm_attack_surface} + \caption{Image of integrating LLM Security with OWASP and MITRE resources} + \label{fig:llm-attack-surface} +\end{figure} + +It is the hope of the OWASP Top 10 for LLM Applications team that this list will +help organizations improve their existing defensive techniques and develop +techniques to address the new threats that come from using this exciting technology. + +\clearpage +\section{Why a Checklist?} + +Checklists can help with strategy development by ensuring thoroughness, +clarifying goals, fostering consistency, and allowing for focused, deliberate +effort, all of which may result in fewer oversights. Following the list can +build confidence in a path to secure adoption while sparking ideas for future +business cases moving forward. It\'s a very forward and very practical way to +achieve continuous improvement. + +\textbf{Not Comprehensive} +While this document is intended to support organizations in developing an +initial LLM strategy in a rapidly changing technical, legal, and regulatory +environment, it does not cover every use case or obligation. Organizations +should extend assessments and practices beyond the scope of the provided +checklist as required for their use case or jurisdiction. diff --git a/llm-top-10-governance-doc/sections/resources.tex b/llm-top-10-governance-doc/sections/resources.tex new file mode 100644 index 00000000..2f377b15 --- /dev/null +++ b/llm-top-10-governance-doc/sections/resources.tex @@ -0,0 +1,10 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% Resources +% ================================================ +\headerimage +\chapter{Resources} +\input{fragments/resources/owasp} +\input{fragments/resources/mitre} +\input{fragments/resources/ai-vuln-repos} +\input{fragments/resources/ai-procurement-guidance} \ No newline at end of file diff --git a/llm-top-10-governance-doc/sections/toc.tex b/llm-top-10-governance-doc/sections/toc.tex new file mode 100755 index 00000000..942e73d1 --- /dev/null +++ b/llm-top-10-governance-doc/sections/toc.tex @@ -0,0 +1,9 @@ +% !TEX root = owasp-doc.tex +% ================================================ +% Table of Contents +% ================================================ +\headerimage +\tableofcontents +% if you want a list of tables included at the +% start of the document uncomment the next line +%\listoftables \ No newline at end of file