convert_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9299da0e-a932-418f-b240-0a180e7f62b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pypdf import PdfReader\n",
    "import re\n",
    "from pdfminer.high_level import extract_text\n",
    "\n",
    "pd.set_option('display.max_colwidth', 500)\n",
    "pd.set_option('display.max_rows', 1000)\n",
    "pd.set_option('display.max_columns', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2f83fcf7-1fa1-4680-abe2-4f0e31bef876",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract document using different readers\n",
    "def extract_pdf_pypdf(pdf_path):\n",
    "    reader = PdfReader(pdf_path)\n",
    "    full_text = \"\"\n",
    "    for page in reader.pages:\n",
    "        full_text += page.extract_text()\n",
    "    return full_text\n",
    "\n",
    "def extract_pdf_pdfminer(pdf_path):\n",
    "    return extract_text(pdf_path)\n",
    "\n",
    "def save_to_txt(text, output_path):\n",
    "    with open(output_path, 'w', encoding='utf-8') as file:\n",
    "        file.write(text)\n",
    "\n",
    "# Extract and save files\n",
    "pdf_path = \"ssm.pubcon230622_guide.en.pdf\"\n",
    "text = extract_pdf_pypdf(pdf_path)\n",
    "text_pdfminer = extract_pdf_pdfminer(pdf_path)\n",
    "\n",
    "save_to_txt(text, 'trim_guide_pypdf.txt')\n",
    "save_to_txt(text_pdfminer, 'trim_guide_pdfminer.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8e68716b-7622-4c4a-86da-0e41714488a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Words with more than 3 letters in trim_guide_pypdf.txt: 64221\n",
      "Words with more than 3 letters in trim_guide_pdfminer.txt: 64181\n",
      "Words with more than 3 letters in ssm.pubcon230622_guide.en.txt: 62748\n"
     ]
    }
   ],
   "source": [
    "def count_words(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        text = file.read()\n",
    "        \n",
    "    # Use regex to tokenize the text and find words with more than 3 letters\n",
    "    words = re.findall(r'\\b\\w{4,}\\b', text)\n",
    "    return len(words)\n",
    "\n",
    "# Paths to the three text files\n",
    "file1 = 'trim_guide_pypdf.txt'\n",
    "file2 = 'trim_guide_pdfminer.txt'\n",
    "file3 = 'ssm.pubcon230622_guide.en.txt'\n",
    "\n",
    "# Get word counts for each file\n",
    "count1 = count_words(file1)\n",
    "count2 = count_words(file2)\n",
    "count3 = count_words(file3)\n",
    "\n",
    "# Print comparison\n",
    "print(f\"Words with more than 3 letters in {file1}: {count1}\")\n",
    "print(f\"Words with more than 3 letters in {file2}: {count2}\")\n",
    "print(f\"Words with more than 3 letters in {file3}: {count3}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "05e193dd-8e34-4c5b-9946-b265c1da383e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove header and footer\n",
    "def remove_ecb_guide_from_file(filename):\n",
    "    # Pattern to match the unwanted lines\n",
    "    pattern = r'^ECB guide to internal models\\s*–.*?\\d+'\n",
    "    \n",
    "    # Read the file line by line and clean each line\n",
    "    with open(filename, 'r', encoding='utf-8') as file:\n",
    "        lines = file.readlines()\n",
    "        cleaned_lines = [re.sub(pattern, '', line) for line in lines]\n",
    "    \n",
    "    # Write the cleaned lines back to the file\n",
    "    with open('trim_guide_pypdf_no_footer.txt', 'w', encoding='utf-8') as file:\n",
    "        file.writelines(cleaned_lines)\n",
    "\n",
    "# Call the function\n",
    "filename = \"trim_guide_pypdf.txt\"\n",
    "remove_ecb_guide_from_file(filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "435000ae-da0f-4cb3-a123-ee8ea02854fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_text_file(filename):\n",
    "    with open(filename, 'r', encoding='utf-8') as f:\n",
    "        lines = f.readlines()\n",
    "\n",
    "    # Add a sentinel line at the end to make sure we process the last line correctly\n",
    "    lines.append('Sentinel line')\n",
    "\n",
    "    processed_lines = []\n",
    "    i = 0\n",
    "    while i < len(lines) - 1:\n",
    "        line = lines[i].strip()\n",
    "        \n",
    "        # New Rule: Appending # to a sentence if the next non-empty line starts with 1 and matches the criteria of Rule 2\n",
    "        j = i + 1\n",
    "        while j < len(lines) and not lines[j].strip():  # Skip empty lines\n",
    "            j += 1\n",
    "        \n",
    "        if j < len(lines) and lines[j].startswith('1') and re.match(r'^\\d+ [A-Z]', lines[j]) and re.match(r'^[A-Z]', line):\n",
    "            processed_lines.append('# ' + line + '\\n')\n",
    "        # Rule 1\n",
    "        elif re.match(r'^\\d+\\.\\d+ [A-Z]', line):\n",
    "            processed_lines.append('\\n')\n",
    "            processed_lines.append('### ' + line + '\\n')\n",
    "            processed_lines.append('\\n')\n",
    "        # Rule 3\n",
    "        elif re.match(r'^\\d+\\.\\d+\\.\\d+ [A-Z]', line):\n",
    "            processed_lines.append('\\n')\n",
    "            processed_lines.append('#### ' + line + '\\n')\n",
    "            processed_lines.append('\\n')\n",
    "        # Rule 2 todo\n",
    "        elif re.match(r'^\\d+ [A-Z]', line) and (re.match(r'^\\d+\\.\\d+ [A-Z]', lines[i + 1]) or re.match(r'^\\d+\\. [A-Z]', lines[i + 1])):\n",
    "            processed_lines.append('\\n')\n",
    "            processed_lines.append('## ' + line + '\\n')\n",
    "            processed_lines.append('\\n')\n",
    "        # Rule 4\n",
    "        elif re.match(r'^\\d+\\.', line):\n",
    "            processed_lines.append('\\n')\n",
    "            processed_lines.append(line + '\\n')\n",
    "        else:\n",
    "            processed_lines.append(line + '\\n')\n",
    "\n",
    "        i += 1\n",
    "\n",
    "    # Write to the modified file\n",
    "    with open('trim_guide_pypdf_mod.txt', 'w', encoding='utf-8') as f:\n",
    "        f.writelines(processed_lines)\n",
    "\n",
    "# Call the function\n",
    "filename = 'trim_guide_pypdf_no_footer.txt'\n",
    "process_text_file(filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "924a1347-ad0f-4d16-8840-3d357da800f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_data(text):\n",
    "    lines = text.split('\\n')\n",
    "    data = []\n",
    "    level_0_label, level_1_label, level_2_label, level_3_label, level_4_label = None, None, None, None, None\n",
    "    body_buffer = []\n",
    "    special = False\n",
    "    in_special_mode = False\n",
    "\n",
    "    for i, line in enumerate(lines):\n",
    "        stripped_line = line.strip()\n",
    "\n",
    "        # Detect start of special paragraph\n",
    "        if stripped_line.startswith(\"&&\"):\n",
    "            in_special_mode = True\n",
    "            stripped_line = stripped_line[2:].strip()  # remove the \"&&\"\n",
    "\n",
    "        # Detect end of special paragraph\n",
    "        if stripped_line.endswith(\"^^\") and in_special_mode:\n",
    "            in_special_mode = False\n",
    "            stripped_line = stripped_line[:-2].strip()  # remove the \"^^\"\n",
    "            special = True\n",
    "\n",
    "        # If in special mode, just append to body buffer and continue\n",
    "        if in_special_mode:\n",
    "            body_buffer.append(stripped_line)\n",
    "            continue\n",
    "\n",
    "        # Detect headings based on our markers\n",
    "        if stripped_line.startswith(\"######\"):\n",
    "            level_5_label = stripped_line.replace(\"######\", \"\").strip()\n",
    "        elif stripped_line.startswith(\"#####\"):\n",
    "            level_4_label = stripped_line.replace(\"#####\", \"\").strip()\n",
    "            level_5_label = None\n",
    "        elif stripped_line.startswith(\"####\"):\n",
    "            level_3_label = stripped_line.replace(\"####\", \"\").strip()\n",
    "            level_4_label, level_5_label = None, None\n",
    "        elif stripped_line.startswith(\"###\"):\n",
    "            level_2_label = stripped_line.replace(\"###\", \"\").strip()\n",
    "            level_3_label, level_4_label, level_5_label = None, None, None\n",
    "        elif stripped_line.startswith(\"##\"):\n",
    "            level_1_label = stripped_line.replace(\"##\", \"\").strip()\n",
    "            level_2_label, level_3_label, level_4_label, level_5_label = None, None, None, None\n",
    "        elif stripped_line.startswith(\"#\"):\n",
    "            level_0_label = stripped_line.replace(\"#\", \"\").strip()\n",
    "            level_1_label, level_2_label, level_3_label, level_4_label, level_5_label = None, None, None, None, None\n",
    "        else:\n",
    "            body_buffer.append(stripped_line)\n",
    "\n",
    "        # Empty line or end of special paragraph indicates end of a paragraph\n",
    "        if (not stripped_line or special) and body_buffer:\n",
    "            # Exclude blank lines from body buffer\n",
    "            cleaned_body_buffer = [line for line in body_buffer if line]\n",
    "            if cleaned_body_buffer:\n",
    "                data.append([level_0_label, level_1_label, level_2_label, level_3_label, level_4_label, ' '.join(body_buffer), special])\n",
    "            body_buffer = []\n",
    "            special = False\n",
    "\n",
    "    # Add the last buffered body text if present\n",
    "    if body_buffer:\n",
    "        data.append([level_0_label, level_1_label, level_2_label, level_3_label, level_4_label, ' '.join(body_buffer), special])\n",
    "\n",
    "    # Convert data to Pandas DataFrame\n",
    "    df = pd.DataFrame(data, columns=['Level_0_Label', 'Level_1_Label', 'Level_2_Label', 'Level_3_Label', 'Level_4_Label', 'Body', 'Special'])\n",
    "\n",
    "    def starts_with_integer(s):\n",
    "        return 1 if re.match(r'^\\d+\\.', s) else 0\n",
    "\n",
    "    # Count words with 3 or more letters\n",
    "    df['num_of_words'] = df['Body'].apply(lambda x: sum(1 for word in x.split() if len(word) >= 3))\n",
    "    \n",
    "    # Add a new column based on the condition\n",
    "    df['In scope'] = df['Body'].apply(starts_with_integer)\n",
    "    \n",
    "    # Check if the string ends with a .\n",
    "    df['Body'] = df['Body'].str.strip()\n",
    "    df['ends_with_period'] = df['Body'].str.endswith('.')\n",
    "\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8c85a117-dcd0-4b30-b556-874733c07fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Cleans text columns based on specified patterns.\n",
    "    \n",
    "    Parameters:\n",
    "    - df: DataFrame containing the columns to clean.\n",
    "    - columns: List of column names to apply the cleaning on.\n",
    "    \n",
    "    Returns:\n",
    "    - DataFrame with cleaned columns.\n",
    "    \"\"\"\n",
    "    for col in columns:\n",
    "        df[col] = df[col].str.replace('  ', ' ', regex=True) \\\n",
    "                         .str.replace(' -', '-', regex=True) \\\n",
    "                         .str.replace('regulato ry', 'regulatory', regex=False) \\\n",
    "                         .str.replace('B anking', 'Banking', regex=False) \\\n",
    "                         .str.replace('t he', 'the', regex=False) \\\n",
    "                         .str.replace('fo llow', 'follow', regex=False) \\\n",
    "                         .str.replace('\\(  ', '(', regex=True) \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4621cb6a-e6ff-4896-a417-af1583559336",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Level_0_Label</th>\n",
       "      <th>Level_1_Label</th>\n",
       "      <th>Level_2_Label</th>\n",
       "      <th>Level_3_Label</th>\n",
       "      <th>Level_4_Label</th>\n",
       "      <th>Body</th>\n",
       "      <th>Special</th>\n",
       "      <th>num_of_words</th>\n",
       "      <th>In scope</th>\n",
       "      <th>ends_with_period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Counterparty credit risk 215</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ECB guide to internal models June 2023 Document releases Release Date of issue Release number Sections/Chapters modified Rationale underlying the release Guide for the Targeted Review of Internal Models (TRIM) – First version 17/02/2017 1.0- Public Consultation – ECB guide to internal models – general topics chapter 28/03/2018 2.0a General topics Incorporation of institutions’ feedback on version 1.0, outcomes of the supervisory review on general topics, and related horizontal analyses Publi...</td>\n",
       "      <td>False</td>\n",
       "      <td>379</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Counterparty credit risk 215</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Foreword</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Counterparty credit risk 215</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1. Articles 143, 283 and 363 of Regulation (EU) No 575/2013 (CRR)1 require the European Central Bank (ECB) to grant permission to use internal models for credit risk, counterparty credit risk and market risk where the requirements set out in the corresponding chapters of the CRR are met by the institution(s) concerned. Based on the current applicable European Union (EU) and national law, the ECB guide to internal models provides transparency on how the ECB understands those rules and how it ...</td>\n",
       "      <td>False</td>\n",
       "      <td>79</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Counterparty credit risk 215</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2. The guide is also intended as a document for the internal use of the different supervisory teams, with the aim of ensuring a common and consistent approach to matters related to internal models. When applying the relevant regulatory framework in specific cases, the ECB will take into due consideration the particular circumstances of the institution concerned.</td>\n",
       "      <td>False</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Counterparty credit risk 215</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>3. This guide should not be construed as going beyond the current existing applicable EU law including, am ong others, adopted regulatory technical standards (RTS), and national law and therefore is not intended to replace or overrule applicable EU and national law. In accordance with the requirements set out in the CRR, the European Banking Authority (EBA) has drafted RTS. These include the Final draft RTS on the specification of the assessment meth odology for competent authorities regardi...</td>\n",
       "      <td>False</td>\n",
       "      <td>153</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Counterparty credit risk 215</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1 Regulation (EU) No 575/2013 of the European Parliament and of the Council of 26 June 2013 on prudential requirements for credit institutions and investment firms and amending Regulation (EU) No 648/2012 (OJ L 176, 27.6.2013 , p. 1), referred to in this guide as the “CRR” . 2 Final draft Regulatory Technical Standards on the specification of the assessment methodology for competent authorities regarding compliance of an institution with the requirements to use internal models for market ris...</td>\n",
       "      <td>False</td>\n",
       "      <td>246</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>General topics</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>3 Commission Delegated Regulation (EU) 2021/930 of 1 March 2021 supplementing the CR R with regard to regulatory technical standards specifying the nature, severity and duration of an economic downturn referred to in Article 181(1), point (b), and Article 182(1), point (b), of that Regulation (OJ L 204, 10.6.2021, p. 1). 4 Commission Dele gated Regulation (EU) 2022/439 of 20 October 2021 supplementing Regulation (EU) No 575/2013 of the European Parliament and of the Council with regard to re...</td>\n",
       "      <td>False</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.1 Relevant regulatory references</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Table 1 Date of issue Article Paragraph/Point Legal background Commission Delegated Regulation (EU) No 2022/439 20/10/2021 3, 9, 10, 12, 13, 14, 16, 17, 30, 31, 32 CRD IV,1 as implemented in the relevant national law 26/06/2013 3, 76, 85 Paragraph 1, sub- paragraphs 7, 9 and 11 CRR 26/06/2013 175, 179, 185, 189, 190, 191, 287, 288, 292, 293, 368 Other references Final draft RTS on assessment methodology for IMA and significant share2 22/11/2016 7-34 EBA Guidelines on SREP3 18/03/2022 235 EBA...</td>\n",
       "      <td>False</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.1 Relevant regulatory references</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Currently the RTS on assessment methodology for IMA and significant share only exist in the final draft version. Once adopted, these RTS will become an additional relevant regulatory reference for this guide.</td>\n",
       "      <td>False</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.1 Relevant regulatory references</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1. The principles listed in this section relate to internal models that are subject to supervisory approval for the calculation of own funds requirements for credit, market and counterparty credit risk (Pillar 1 models), unless stated otherwi se.</td>\n",
       "      <td>False</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.2 Guidelines at consolidated and subsidiary levels</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2. An inconsistent implementation of internal model-related tasks within a banking group bears the risk of an inappropriate coverage of the risks measured by</td>\n",
       "      <td>False</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.2 Guidelines at consolidated and subsidiary levels</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1 Directive 2013/36/EU of the European Parliament and the C ouncil of 26 June 2013 on access to the activity of credit institutions and the prudential supervision of credit institutions and investment firms, amending Directive 2002/87/EC and repealing Directives 2006/48/EC and 2006/49/EC (OJ L 176,</td>\n",
       "      <td>False</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.2 Guidelines at consolidated and subsidiary levels</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>27.6.2013, p. 338), referred to in this guide as “CRD”. 2 Final draft Regulatory Technical Standards on the specification of the assessment methodology for competent authorities regarding compliance of an institution with the requirements to use internal models for market risk and assessment of significant share under points (b) and (c) of Article 363(4) of Regulation (EU) No 575/2013 (EBA/RTS/2016/07), referred to in this guide as the “Final draft RTS on assessment methodology for IMA and s...</td>\n",
       "      <td>False</td>\n",
       "      <td>147</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.2 Guidelines at consolidated and subsidiary levels</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>3. A consistent and integrated level of application of the group principles and guidelines in the relevant entities is expected. This could be ensured by applying controls that verify that these princi ples are implemented correctly in all relevant entities. Examples of such controls include periodic monitoring procedures by the parent entity on implementation at local level or a gap analysis between group-wide principles and local application, including local guidelines. These controls can ...</td>\n",
       "      <td>False</td>\n",
       "      <td>104</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.3 Documentation of internal models</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>4. All internal models should be documented to allow a qualified third party to independently understand the methodology, assumptions, limitations and use of the model and to replicate its development and implementation.</td>\n",
       "      <td>False</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.3 Documentation of internal models</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>5. The institution should therefore defin e principles and guidelines for model documentation. These should encompass guidance for the governance of the model documentation itself. The scope of the model documentation should be defined by type of model. In-scope areas should include at least the t echnical aspects of the model (methodology and assumptions), data (processes), instructions for model users and performance/validation (including the results of implementation testing).</td>\n",
       "      <td>False</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.3 Documentation of internal models</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>6. The institution should be able to demonstrate how its documentation an d the register of its internal models facilitate the internal and external understanding of the models. The institution can choose to have one or more registers for models in use, according to the different risk types (credit, market or counterparty credit risk). The register(s) should contain at least the following information for each model:7 the model owner(s), range of application, materiality, approval date, poten...</td>\n",
       "      <td>False</td>\n",
       "      <td>78</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.3 Documentation of internal models</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>5 The model life cycle generally includes the following steps: development (including data preparation), calibration (incl uding data preparation), validation, supervisory approval (if necessary), implementation in internal processes, application and review of estimates . 6 In the case of credit risk, “internal models” should be read hereinafter as “IRB rating systems”. 7 Regarding credit risk, Article 32 of Commission Delegated Regulation (EU) No 2022/439 specifies the contents of a registe...</td>\n",
       "      <td>False</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.3 Documentation of internal models</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>7. Adequate controls of the register of the institution’s internal models, together with an inventory of the documentation, including a n annual review, should be in place. This includes a policy for document management that clearly states the roles and responsibilities involved in approving documents, as well as how changes in documentation are applied and communicated internally. In addi tion, the institution should have a policy regarding the adequate archiving and maintenance of informat...</td>\n",
       "      <td>False</td>\n",
       "      <td>69</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.3 Documentation of internal models</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>8. Documentation should be kept up to date and the i nstitution should keep documents for an appropriate period of time, taking into account legal or regulatory retention periods.</td>\n",
       "      <td>False</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.4 Implementation of a model risk management framework</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>9. Effective model risk management allows institutions to reduce the risk of potential losses and underestimation of own funds requirements as a result of flaws in the development, implementation or use of the models. To mitigate these risks, institutions should have a model risk management framework8 in place that allows them to ide ntify, understand and manage their model risk for internal models across the group. This framework should comprise, at least, the following. (a) A written model...</td>\n",
       "      <td>False</td>\n",
       "      <td>176</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.4 Implementation of a model risk management framework</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>8 Institutions are expected to implement an effective model risk management framework for all models in use. However, Section 1 focuses only on internal models approved for use for the calculation of own funds requirements for credit, market and counterparty credit risk and the respective requirements for a model risk manage ment framework for these models. 9 “Model risk” as defined in Article 3(1) sub-paragraph (11) of CRD. errors) should be considered. This method ology should be applied c...</td>\n",
       "      <td>False</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.5 Identification of management body and senior management</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>10. Institutions should clearly define the roles and responsibi lities of their management body and senior management as defined in Article 3(1) sub- paragraphs (7) and (9) of CRD, as implemented in the relevant national law with regard to internal models and in relation to each risk type. The institution’s internal doc umentation should also clearly describe the composition of the management body and the senior management.</td>\n",
       "      <td>False</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.5 Identification of management body and senior management</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>11. The term “management body” could refer to the single board, in a one-tier system, or to the function of the management and supervisory boards in a two- tier corporate governance system. The institution should document the roles and responsibilities of each individual in the management body. For the purposes of this guide, the management body refers to the members of the management body in its management function (executive members).</td>\n",
       "      <td>False</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.5 Identification of management body and senior management</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>12. The institution should assess the appropriateness of designated committees of the management body (if applicable) in order to ensure that they provide an adequate support function for effective decision-making procedures. Thi s holds, in particular, for decisions concerning material aspects of the institution’s internal models. The institution should clearly document the composition, mandate and reporting lines of committees responsible for internal model governance and oversig ht, as we...</td>\n",
       "      <td>False</td>\n",
       "      <td>94</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.5 Identification of management body and senior management</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>10 Article 85(1) of CRD refers to model risk in operational risk. However, and specifically with regard to Pillar 1 models, the EBA Guidelines on SREP st ate that competent authorities should consider model risk as part of the assessment of specific risks to capital (e.g. IRB model deficiency is considered as part of the credit risk assessment) and for the capital adequacy assessment. In particular for IRB models, the expected range of estimation errors should be reflected in the margin of c...</td>\n",
       "      <td>False</td>\n",
       "      <td>87</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.5 Identification of management body and senior management</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>13. The institution should also be able to identify which individuals constitute its senior management with respect to the credit, market and counterparty credit risk Pillar I model frameworks. In addition to the specifica tions of Article 3(1) sub-paragraphs (7) and (9) of CRD, senior management can be deemed to constitute the highest hierarchical level(s) below the management body with a clearly defined responsibility for internal models.11 The senior management should either report direct...</td>\n",
       "      <td>False</td>\n",
       "      <td>98</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.6 General principles for internal validation</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>14. All internal models and internal estimates should be subject to an initial, and subsequently to an annual, internal validation. For the avoidance of doubt, the term “initial validation” in the guide refers to the validation of new models as well as the validation of material changes and extensions to approved models.</td>\n",
       "      <td>False</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.6 General principles for internal validation</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>15. To ensure the effective independence of the internal validation function from the model development pr ocess (i.e. model design, development, implementation and monitoring), institutions should have appropriate organisational arrangements in place. The ECB understands that the possible organisational arrangements are as follows:12 (a) separation into two different units reporting to different members of the senior management; (b) separation into two different units reporting to the same ...</td>\n",
       "      <td>False</td>\n",
       "      <td>96</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>General topics</td>\n",
       "      <td>1 Overarching principles for internal models</td>\n",
       "      <td>1.6 General principles for internal validation</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>16. Consequently, the ECB understands that large and complex institutions should implement the most ro bust independence option.</td>\n",
       "      <td>False</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Level_0_Label  \\\n",
       "0   Counterparty credit risk 215   \n",
       "1   Counterparty credit risk 215   \n",
       "2   Counterparty credit risk 215   \n",
       "3   Counterparty credit risk 215   \n",
       "4   Counterparty credit risk 215   \n",
       "5   Counterparty credit risk 215   \n",
       "6                 General topics   \n",
       "7                 General topics   \n",
       "8                 General topics   \n",
       "9                 General topics   \n",
       "10                General topics   \n",
       "11                General topics   \n",
       "12                General topics   \n",
       "13                General topics   \n",
       "14                General topics   \n",
       "15                General topics   \n",
       "16                General topics   \n",
       "17                General topics   \n",
       "18                General topics   \n",
       "19                General topics   \n",
       "20                General topics   \n",
       "21                General topics   \n",
       "22                General topics   \n",
       "23                General topics   \n",
       "24                General topics   \n",
       "25                General topics   \n",
       "26                General topics   \n",
       "27                General topics   \n",
       "28                General topics   \n",
       "29                General topics   \n",
       "\n",
       "                                   Level_1_Label  \\\n",
       "0                                           None   \n",
       "1                                           None   \n",
       "2                                           None   \n",
       "3                                           None   \n",
       "4                                           None   \n",
       "5                                           None   \n",
       "6                                           None   \n",
       "7   1 Overarching principles for internal models   \n",
       "8   1 Overarching principles for internal models   \n",
       "9   1 Overarching principles for internal models   \n",
       "10  1 Overarching principles for internal models   \n",
       "11  1 Overarching principles for internal models   \n",
       "12  1 Overarching principles for internal models   \n",
       "13  1 Overarching principles for internal models   \n",
       "14  1 Overarching principles for internal models   \n",
       "15  1 Overarching principles for internal models   \n",
       "16  1 Overarching principles for internal models   \n",
       "17  1 Overarching principles for internal models   \n",
       "18  1 Overarching principles for internal models   \n",
       "19  1 Overarching principles for internal models   \n",
       "20  1 Overarching principles for internal models   \n",
       "21  1 Overarching principles for internal models   \n",
       "22  1 Overarching principles for internal models   \n",
       "23  1 Overarching principles for internal models   \n",
       "24  1 Overarching principles for internal models   \n",
       "25  1 Overarching principles for internal models   \n",
       "26  1 Overarching principles for internal models   \n",
       "27  1 Overarching principles for internal models   \n",
       "28  1 Overarching principles for internal models   \n",
       "29  1 Overarching principles for internal models   \n",
       "\n",
       "                                                  Level_2_Label Level_3_Label  \\\n",
       "0                                                          None          None   \n",
       "1                                                          None          None   \n",
       "2                                                          None          None   \n",
       "3                                                          None          None   \n",
       "4                                                          None          None   \n",
       "5                                                          None          None   \n",
       "6                                                          None          None   \n",
       "7                            1.1 Relevant regulatory references          None   \n",
       "8                            1.1 Relevant regulatory references          None   \n",
       "9                            1.1 Relevant regulatory references          None   \n",
       "10         1.2 Guidelines at consolidated and subsidiary levels          None   \n",
       "11         1.2 Guidelines at consolidated and subsidiary levels          None   \n",
       "12         1.2 Guidelines at consolidated and subsidiary levels          None   \n",
       "13         1.2 Guidelines at consolidated and subsidiary levels          None   \n",
       "14                         1.3 Documentation of internal models          None   \n",
       "15                         1.3 Documentation of internal models          None   \n",
       "16                         1.3 Documentation of internal models          None   \n",
       "17                         1.3 Documentation of internal models          None   \n",
       "18                         1.3 Documentation of internal models          None   \n",
       "19                         1.3 Documentation of internal models          None   \n",
       "20      1.4 Implementation of a model risk management framework          None   \n",
       "21      1.4 Implementation of a model risk management framework          None   \n",
       "22  1.5 Identification of management body and senior management          None   \n",
       "23  1.5 Identification of management body and senior management          None   \n",
       "24  1.5 Identification of management body and senior management          None   \n",
       "25  1.5 Identification of management body and senior management          None   \n",
       "26  1.5 Identification of management body and senior management          None   \n",
       "27               1.6 General principles for internal validation          None   \n",
       "28               1.6 General principles for internal validation          None   \n",
       "29               1.6 General principles for internal validation          None   \n",
       "\n",
       "   Level_4_Label  \\\n",
       "0           None   \n",
       "1           None   \n",
       "2           None   \n",
       "3           None   \n",
       "4           None   \n",
       "5           None   \n",
       "6           None   \n",
       "7           None   \n",
       "8           None   \n",
       "9           None   \n",
       "10          None   \n",
       "11          None   \n",
       "12          None   \n",
       "13          None   \n",
       "14          None   \n",
       "15          None   \n",
       "16          None   \n",
       "17          None   \n",
       "18          None   \n",
       "19          None   \n",
       "20          None   \n",
       "21          None   \n",
       "22          None   \n",
       "23          None   \n",
       "24          None   \n",
       "25          None   \n",
       "26          None   \n",
       "27          None   \n",
       "28          None   \n",
       "29          None   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   Body  \\\n",
       "0   ECB guide to internal models June 2023 Document releases Release Date of issue Release number Sections/Chapters modified Rationale underlying the release Guide for the Targeted Review of Internal Models (TRIM) – First version 17/02/2017 1.0- Public Consultation – ECB guide to internal models – general topics chapter 28/03/2018 2.0a General topics Incorporation of institutions’ feedback on version 1.0, outcomes of the supervisory review on general topics, and related horizontal analyses Publi...   \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              Foreword   \n",
       "2   1. Articles 143, 283 and 363 of Regulation (EU) No 575/2013 (CRR)1 require the European Central Bank (ECB) to grant permission to use internal models for credit risk, counterparty credit risk and market risk where the requirements set out in the corresponding chapters of the CRR are met by the institution(s) concerned. Based on the current applicable European Union (EU) and national law, the ECB guide to internal models provides transparency on how the ECB understands those rules and how it ...   \n",
       "3                                                                                                                                          2. The guide is also intended as a document for the internal use of the different supervisory teams, with the aim of ensuring a common and consistent approach to matters related to internal models. When applying the relevant regulatory framework in specific cases, the ECB will take into due consideration the particular circumstances of the institution concerned.   \n",
       "4   3. This guide should not be construed as going beyond the current existing applicable EU law including, am ong others, adopted regulatory technical standards (RTS), and national law and therefore is not intended to replace or overrule applicable EU and national law. In accordance with the requirements set out in the CRR, the European Banking Authority (EBA) has drafted RTS. These include the Final draft RTS on the specification of the assessment meth odology for competent authorities regardi...   \n",
       "5   1 Regulation (EU) No 575/2013 of the European Parliament and of the Council of 26 June 2013 on prudential requirements for credit institutions and investment firms and amending Regulation (EU) No 648/2012 (OJ L 176, 27.6.2013 , p. 1), referred to in this guide as the “CRR” . 2 Final draft Regulatory Technical Standards on the specification of the assessment methodology for competent authorities regarding compliance of an institution with the requirements to use internal models for market ris...   \n",
       "6   3 Commission Delegated Regulation (EU) 2021/930 of 1 March 2021 supplementing the CR R with regard to regulatory technical standards specifying the nature, severity and duration of an economic downturn referred to in Article 181(1), point (b), and Article 182(1), point (b), of that Regulation (OJ L 204, 10.6.2021, p. 1). 4 Commission Dele gated Regulation (EU) 2022/439 of 20 October 2021 supplementing Regulation (EU) No 575/2013 of the European Parliament and of the Council with regard to re...   \n",
       "7   Table 1 Date of issue Article Paragraph/Point Legal background Commission Delegated Regulation (EU) No 2022/439 20/10/2021 3, 9, 10, 12, 13, 14, 16, 17, 30, 31, 32 CRD IV,1 as implemented in the relevant national law 26/06/2013 3, 76, 85 Paragraph 1, sub- paragraphs 7, 9 and 11 CRR 26/06/2013 175, 179, 185, 189, 190, 191, 287, 288, 292, 293, 368 Other references Final draft RTS on assessment methodology for IMA and significant share2 22/11/2016 7-34 EBA Guidelines on SREP3 18/03/2022 235 EBA...   \n",
       "8                                                                                                                                                                                                                                                                                                      Currently the RTS on assessment methodology for IMA and significant share only exist in the final draft version. Once adopted, these RTS will become an additional relevant regulatory reference for this guide.   \n",
       "9                                                                                                                                                                                                                                                                1. The principles listed in this section relate to internal models that are subject to supervisory approval for the calculation of own funds requirements for credit, market and counterparty credit risk (Pillar 1 models), unless stated otherwi se.   \n",
       "10                                                                                                                                                                                                                                                                                                                                                        2. An inconsistent implementation of internal model-related tasks within a banking group bears the risk of an inappropriate coverage of the risks measured by   \n",
       "11                                                                                                                                                                                                          1 Directive 2013/36/EU of the European Parliament and the C ouncil of 26 June 2013 on access to the activity of credit institutions and the prudential supervision of credit institutions and investment firms, amending Directive 2002/87/EC and repealing Directives 2006/48/EC and 2006/49/EC (OJ L 176,   \n",
       "12  27.6.2013, p. 338), referred to in this guide as “CRD”. 2 Final draft Regulatory Technical Standards on the specification of the assessment methodology for competent authorities regarding compliance of an institution with the requirements to use internal models for market risk and assessment of significant share under points (b) and (c) of Article 363(4) of Regulation (EU) No 575/2013 (EBA/RTS/2016/07), referred to in this guide as the “Final draft RTS on assessment methodology for IMA and s...   \n",
       "13  3. A consistent and integrated level of application of the group principles and guidelines in the relevant entities is expected. This could be ensured by applying controls that verify that these princi ples are implemented correctly in all relevant entities. Examples of such controls include periodic monitoring procedures by the parent entity on implementation at local level or a gap analysis between group-wide principles and local application, including local guidelines. These controls can ...   \n",
       "14                                                                                                                                                                                                                                                                                         4. All internal models should be documented to allow a qualified third party to independently understand the methodology, assumptions, limitations and use of the model and to replicate its development and implementation.   \n",
       "15                 5. The institution should therefore defin e principles and guidelines for model documentation. These should encompass guidance for the governance of the model documentation itself. The scope of the model documentation should be defined by type of model. In-scope areas should include at least the t echnical aspects of the model (methodology and assumptions), data (processes), instructions for model users and performance/validation (including the results of implementation testing).   \n",
       "16  6. The institution should be able to demonstrate how its documentation an d the register of its internal models facilitate the internal and external understanding of the models. The institution can choose to have one or more registers for models in use, according to the different risk types (credit, market or counterparty credit risk). The register(s) should contain at least the following information for each model:7 the model owner(s), range of application, materiality, approval date, poten...   \n",
       "17  5 The model life cycle generally includes the following steps: development (including data preparation), calibration (incl uding data preparation), validation, supervisory approval (if necessary), implementation in internal processes, application and review of estimates . 6 In the case of credit risk, “internal models” should be read hereinafter as “IRB rating systems”. 7 Regarding credit risk, Article 32 of Commission Delegated Regulation (EU) No 2022/439 specifies the contents of a registe...   \n",
       "18  7. Adequate controls of the register of the institution’s internal models, together with an inventory of the documentation, including a n annual review, should be in place. This includes a policy for document management that clearly states the roles and responsibilities involved in approving documents, as well as how changes in documentation are applied and communicated internally. In addi tion, the institution should have a policy regarding the adequate archiving and maintenance of informat...   \n",
       "19                                                                                                                                                                                                                                                                                                                                  8. Documentation should be kept up to date and the i nstitution should keep documents for an appropriate period of time, taking into account legal or regulatory retention periods.   \n",
       "20  9. Effective model risk management allows institutions to reduce the risk of potential losses and underestimation of own funds requirements as a result of flaws in the development, implementation or use of the models. To mitigate these risks, institutions should have a model risk management framework8 in place that allows them to ide ntify, understand and manage their model risk for internal models across the group. This framework should comprise, at least, the following. (a) A written model...   \n",
       "21  8 Institutions are expected to implement an effective model risk management framework for all models in use. However, Section 1 focuses only on internal models approved for use for the calculation of own funds requirements for credit, market and counterparty credit risk and the respective requirements for a model risk manage ment framework for these models. 9 “Model risk” as defined in Article 3(1) sub-paragraph (11) of CRD. errors) should be considered. This method ology should be applied c...   \n",
       "22                                                                          10. Institutions should clearly define the roles and responsibi lities of their management body and senior management as defined in Article 3(1) sub- paragraphs (7) and (9) of CRD, as implemented in the relevant national law with regard to internal models and in relation to each risk type. The institution’s internal doc umentation should also clearly describe the composition of the management body and the senior management.   \n",
       "23                                                             11. The term “management body” could refer to the single board, in a one-tier system, or to the function of the management and supervisory boards in a two- tier corporate governance system. The institution should document the roles and responsibilities of each individual in the management body. For the purposes of this guide, the management body refers to the members of the management body in its management function (executive members).   \n",
       "24  12. The institution should assess the appropriateness of designated committees of the management body (if applicable) in order to ensure that they provide an adequate support function for effective decision-making procedures. Thi s holds, in particular, for decisions concerning material aspects of the institution’s internal models. The institution should clearly document the composition, mandate and reporting lines of committees responsible for internal model governance and oversig ht, as we...   \n",
       "25  10 Article 85(1) of CRD refers to model risk in operational risk. However, and specifically with regard to Pillar 1 models, the EBA Guidelines on SREP st ate that competent authorities should consider model risk as part of the assessment of specific risks to capital (e.g. IRB model deficiency is considered as part of the credit risk assessment) and for the capital adequacy assessment. In particular for IRB models, the expected range of estimation errors should be reflected in the margin of c...   \n",
       "26  13. The institution should also be able to identify which individuals constitute its senior management with respect to the credit, market and counterparty credit risk Pillar I model frameworks. In addition to the specifica tions of Article 3(1) sub-paragraphs (7) and (9) of CRD, senior management can be deemed to constitute the highest hierarchical level(s) below the management body with a clearly defined responsibility for internal models.11 The senior management should either report direct...   \n",
       "27                                                                                                                                                                                   14. All internal models and internal estimates should be subject to an initial, and subsequently to an annual, internal validation. For the avoidance of doubt, the term “initial validation” in the guide refers to the validation of new models as well as the validation of material changes and extensions to approved models.   \n",
       "28  15. To ensure the effective independence of the internal validation function from the model development pr ocess (i.e. model design, development, implementation and monitoring), institutions should have appropriate organisational arrangements in place. The ECB understands that the possible organisational arrangements are as follows:12 (a) separation into two different units reporting to different members of the senior management; (b) separation into two different units reporting to the same ...   \n",
       "29                                                                                                                                                                                                                                                                                                                                                                                     16. Consequently, the ECB understands that large and complex institutions should implement the most ro bust independence option.   \n",
       "\n",
       "    Special  num_of_words  In scope  ends_with_period  \n",
       "0     False           379         0             False  \n",
       "1     False             1         0             False  \n",
       "2     False            79         1              True  \n",
       "3     False            46         1              True  \n",
       "4     False           153         1              True  \n",
       "5     False           246         0              True  \n",
       "6     False           150         0              True  \n",
       "7     False            72         0             False  \n",
       "8     False            29         0              True  \n",
       "9     False            32         1              True  \n",
       "10    False            17         1             False  \n",
       "11    False            35         0             False  \n",
       "12    False           147         1              True  \n",
       "13    False           104         1              True  \n",
       "14    False            25         1              True  \n",
       "15    False            57         1              True  \n",
       "16    False            78         1             False  \n",
       "17    False            69         0              True  \n",
       "18    False            69         1              True  \n",
       "19    False            21         1              True  \n",
       "20    False           176         1             False  \n",
       "21    False           132         0              True  \n",
       "22    False            57         1              True  \n",
       "23    False            57         1              True  \n",
       "24    False            94         1              True  \n",
       "25    False            87         0              True  \n",
       "26    False            98         1              True  \n",
       "27    False            39         1              True  \n",
       "28    False            96         1              True  \n",
       "29    False            17         1              True  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('trim_guide_pypdf_mod.txt', 'r', encoding='utf-8') as file:\n",
    "    text = file.read()\n",
    "\n",
    "df = extract_data(text)\n",
    "columns_to_clean = ['Body', 'Level_0_Label', 'Level_1_Label', 'Level_2_Label']\n",
    "df = clean_text_columns(df, columns_to_clean)\n",
    "\n",
    "df.to_excel('trim_guide_v2.xlsx', index=False)\n",
    "\n",
    "df.head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e98988d7-c0ef-4201-a539-7ab8ba29607a",
   "metadata": {},
   "source": [
    "1. **Rule 1:** For lines that have a format of \"x.y [text]\" (like \"1.1 Relevant regulatory references\"):\n",
    "   - Insert an empty line before and after this line.\n",
    "   - Prefix the line with \"###\".\n",
    "   \n",
    "2. **Rule 2:** For lines that have a format of \"x [text]\" (like \"1 Overarching principles for internal models\") AND the immediate next line follows the format of Rule 1:\n",
    "   - Insert an empty line before and after this line.\n",
    "   - Prefix the line with \"##\".\n",
    "   \n",
    "3. **Rule 3:** For lines that have a format of \"x.x.x [text]\" (like \"2.1.1 Some other text\"):\n",
    "   - Insert an empty line before and after this line.\n",
    "   - Prefix the line with \"####\".\n",
    "\n",
    "4. **Rule 4:** For lines that have a format of \"x. [text]\" (like \"5. Add an empty line in front of it\"):\n",
    "   - Insert an empty line before this line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6c0b22a-0074-448c-bee6-f300f1048a3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([None, '1 Overarching principles for internal models',\n",
       "       '2 Roll-out and permanent partial use', '3 Internal governance',\n",
       "       '4 Internal validation', '5 Internal audit', '6 Model use',\n",
       "       '7 Management of changes to the IRB approach',\n",
       "       '8 Third-party involvement', '1 Scope of the credit risk chapter',\n",
       "       '2 Data maintenance for the IRB approach', '3 Use of data',\n",
       "       '4 Definition of default', '5 Probability of default',\n",
       "       '6 Loss given default', '7 Conversion factors',\n",
       "       '8 Model-related MoC', '9 Review of estimates',\n",
       "       '10 Calculation of m aturity for non-retail exposures',\n",
       "       '1 Scope of the market risk chapter',\n",
       "       '2 Scope of the internal model approach',\n",
       "       '3 Regulatory back-testing of VaR models',\n",
       "       '4 Aspects of internal validation of market risk models',\n",
       "       '5 Methodology for VaR and stressed VaR',\n",
       "       '6 Methodology for IRC models focusing on default risk',\n",
       "       '7 Risks-not-in-the-model engines',\n",
       "       '1 Scope of the counterparty credit risk chapter',\n",
       "       '2 Trade coverage', '3 Margin period of risk and cash flows',\n",
       "       '4 Collateral modelling', '5 Modelling of initial margin',\n",
       "       '6 Maturity', '7 Granularity , number of time steps and scenarios',\n",
       "       '8 Calibration frequency and stress calibration', '9 Use test',\n",
       "       '10 Validation', '11 Effective expected positive exposure',\n",
       "       '12 Alpha parameter',\n",
       "       '13 Risks not in effective expected positive exposure'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Level_1_Label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8a65703d-20b1-45e2-a6f1-582d1f8a8662",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([None, '1.1 Relevant regulatory references',\n",
       "       '1.2 Guidelines at consolidated and subsidiary levels',\n",
       "       '1.3 Documentation of internal models',\n",
       "       '1.4 Implementation of a model risk management framework',\n",
       "       '1.5 Identification of management body and senior management',\n",
       "       '1.6 General principles for internal validation',\n",
       "       '1.7 General principles for internal audit',\n",
       "       '1.8 General principles on climate-related and environmental risks',\n",
       "       '1.9 General principles for the implementation of a changed or',\n",
       "       '2.1 Relevant regulatory references',\n",
       "       '2.2 Application of the IRB approach',\n",
       "       '2.3 Governance of the roll-out plan for the IRB approach',\n",
       "       '2.4 Changes to the roll-out plan for the IRB approach',\n",
       "       '2.5 Monitoring of compliance with permanent partial use provisions',\n",
       "       '2.6 Reversion to a less sophisticated app roach',\n",
       "       '2.7 Internal models in the context of consolidations',\n",
       "       '3.1 Relevant regulatory references',\n",
       "       '3.2 Materiality of rating system s',\n",
       "       '3.3 Decision-making responsibilities', '3.4 Management reporting',\n",
       "       '3.5 Understanding of the rating systems',\n",
       "       '3.6 The credit risk control unit (CRCU)',\n",
       "       '3.7 Review of estimates', '4.1 Relevant regulatory references',\n",
       "       '4.2 Validation level and responsibilities',\n",
       "       '4.3 Content and frequency of tasks of the validation function',\n",
       "       '4.4 Reporting and follow-up',\n",
       "       '5.1 Relevant regulatory references',\n",
       "       '5.2 Scope and frequency of the review of the rating systems',\n",
       "       '6.1 Relevant regulatory references', '6.2 Use test requirement',\n",
       "       '6.3 Risk management, credit approval and decision-making process',\n",
       "       '6.4 Internal capital assessment and allocation',\n",
       "       '6.5 Corporate governance functions',\n",
       "       '6.6 Assignment of exposures to grades or pools',\n",
       "       '7.1 Relevant regulatory references',\n",
       "       '7.2 Content of the change policy', '7.3 Notification',\n",
       "       '7.4 Classification', '7.5 Impact assessment',\n",
       "       '7.6 Re-rating process', '8.1 Relevant regulatory references',\n",
       "       '8.2 Preliminary principles',\n",
       "       '8.3 Third-party involvement in interna l functions and tasks',\n",
       "       '2.2 IT systems: infrastructure and implementation testing',\n",
       "       '2.3 Policies, roles and responsibilities in data processing and data',\n",
       "       '2.4 Data quality management framework',\n",
       "       '3.2 Use of external data',\n",
       "       '3.3 Use of external bureau scores or external ratings as input variables',\n",
       "       '3.4 Use of pooled data24',\n",
       "       '3.5 Use of purchased rating sy stems or models (pool models26 27)',\n",
       "       '3.6 Use of human judgement',\n",
       "       '3.7 Use of data in the case of co nsolidations',\n",
       "       '4.2 Consistency of the application',\n",
       "       '4.3 Days past due criterion', '4.4 Unlikeness to pay criterion',\n",
       "       '4.5 Return to non-defaulted status',\n",
       "       '4.6 Consistency of external data',\n",
       "       '4.7 Adjustments to risk estimates in the case of changes to the',\n",
       "       '5.1 Structure of PD models', '5.2 PD risk quantification',\n",
       "       '6.1 Realised LGD', '6.2 LGD structure', '6.3 Risk quantification',\n",
       "       '6.4 Estimation of EL BE and LGD in-default',\n",
       "       '7.1 Commitments, unadvised limits and scope of applicat ion',\n",
       "       '7.2 Realised CCFs', '7.3 CCF structure',\n",
       "       '7.4 CCF risk quantification',\n",
       "       '9.1 Relevant regulatory references',\n",
       "       '10.1 Relevant regulatory references', '2.1 Regulatory references',\n",
       "       '2.2 Delimitation of the regulatory trading book',\n",
       "       '2.3 Treatment of banking book positions',\n",
       "       '2.4 Partial use models',\n",
       "       '2.5 Exclusion of positions in the regulatory trading book from the scope',\n",
       "       '2.6 Treatment of specific positions',\n",
       "       '2.7 Aggregation requirements', '3.1 Regula tory references',\n",
       "       '3.2 Scope of application of regulatory back-testing',\n",
       "       '3.3 Historical period used to perform bac k-testing, definition of',\n",
       "       '3.4 Calculation of actual P&L',\n",
       "       '3.5 Calculation of hypothetical P&L',\n",
       "       '3.6 Counting of overshootings', '3.7 Analysis of ov ershootings',\n",
       "       '4.1 Regulatory references',\n",
       "       '4.2 Frequency of interna l validation',\n",
       "       '4.3 Internal back-testing of VaR models',\n",
       "       '4.4 Validation on hypothetical portfolios',\n",
       "       '4.5 Validation based on hypothetical portfolios for VaR models by',\n",
       "       '5.1 Regulatory references', '5.2 General requirements',\n",
       "       '5.3 Data inputs, length of the time series used to calibrate VaR and',\n",
       "       '5.4 Data quality',\n",
       "       '5.5 Proxies, beta approximation and regressions',\n",
       "       '5.6 Risk factors in the model',\n",
       "       '5.7 Pricing functions and methods in the model',\n",
       "       '6.1 Regulatory references', '6.2 General requirements',\n",
       "       '6.3 Data inputs', '6.4 Distribution and correlation assumptions',\n",
       "       '6.5 Ratings, probabilities of default and recovery rate assumptions',\n",
       "       '6.6 Treatment of groups of connected issuers',\n",
       "       '7.1 Regulatory references',\n",
       "       '7.2 The framework for risks-not-in-the-model engines',\n",
       "       '7.3 Identification of RNIME', '7.4 Quantificat ion of RNIME',\n",
       "       '7.5 Management of RNIME and implementation in an institu tion’s risk',\n",
       "       '2.1 Relevan t regulatory references',\n",
       "       '2.2 Principles for ECB Banking Supervision',\n",
       "       '3.2 Principles for ECB Banking Supervision',\n",
       "       '4.2 Principles for ECB Banking Supervision',\n",
       "       '5.2 Principles for ECB Banking Supervision',\n",
       "       '6.2 Principles for ECB Banking Supervision',\n",
       "       '7.2 Principles for ECB Banking Supervision',\n",
       "       '8.2 Principles for ECB Banking Supervision',\n",
       "       '9.2 Principles for ECB Banking Supervision',\n",
       "       '10.1 Relev ant regulatory references',\n",
       "       '10.2 Principles for ECB Banking Supervision',\n",
       "       '11.1 Relevant regulatory references',\n",
       "       '11.2 Principles for ECB Banking Supervision',\n",
       "       '12.1 Relevant regulatory references', '12.2 Supervisory actions',\n",
       "       '13.1 Relevant regulatory r eferences',\n",
       "       '13.2 Principles for ECB Banking Supervision'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Level_2_Label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6ca3c140-e9d4-459d-9463-142b87ce269e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Counterparty credit risk 215', 'General topics', 'Credit risk',\n",
       "       'Market risk', 'Counterparty credit risk'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Level_0_Label'].unique()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}