{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9aa9167b",
   "metadata": {},
   "source": [
    "# Классификация оттока клиентов банка\n",
    "\n",
    "В работе определяется ключевая зависимая переменная, строится базовый классификатор по всем признакам, затем применяется классический метод отбора признаков и строится модель по сокращённому набору переменных. Для обеих моделей оцениваются точность, значимость и достоверность.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd8e2364",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import json\n",
    "import warnings\n",
    "import inspect\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from scipy import stats\n",
    "\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_validate\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score, precision_score, recall_score, f1_score, roc_auc_score,\n",
    "    confusion_matrix, classification_report\n",
    ")\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import statsmodels.api as sm\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "pd.set_option(\"display.max_columns\", 200)\n",
    "np.random.seed(42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "182ceac8",
   "metadata": {},
   "source": [
    "## Загрузка данных и первичная проверка"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ebc96b90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Путь к файлу: /home/konnilol/Downloads/Bank Customer Churn Prediction.csv\n",
      "Размер датасета: (10000, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>credit_score</th>\n",
       "      <th>country</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>tenure</th>\n",
       "      <th>balance</th>\n",
       "      <th>products_number</th>\n",
       "      <th>credit_card</th>\n",
       "      <th>active_member</th>\n",
       "      <th>estimated_salary</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15634602</td>\n",
       "      <td>619</td>\n",
       "      <td>France</td>\n",
       "      <td>Female</td>\n",
       "      <td>42</td>\n",
       "      <td>2</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>101348.88</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15647311</td>\n",
       "      <td>608</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Female</td>\n",
       "      <td>41</td>\n",
       "      <td>1</td>\n",
       "      <td>83807.86</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>112542.58</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15619304</td>\n",
       "      <td>502</td>\n",
       "      <td>France</td>\n",
       "      <td>Female</td>\n",
       "      <td>42</td>\n",
       "      <td>8</td>\n",
       "      <td>159660.80</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113931.57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15701354</td>\n",
       "      <td>699</td>\n",
       "      <td>France</td>\n",
       "      <td>Female</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>93826.63</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15737888</td>\n",
       "      <td>850</td>\n",
       "      <td>Spain</td>\n",
       "      <td>Female</td>\n",
       "      <td>43</td>\n",
       "      <td>2</td>\n",
       "      <td>125510.82</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>79084.10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   customer_id  credit_score country  gender  age  tenure    balance  products_number  credit_card  active_member  estimated_salary  churn\n",
       "0     15634602           619  France  Female   42       2       0.00                1            1              1         101348.88      1\n",
       "1     15647311           608   Spain  Female   41       1   83807.86                1            0              1         112542.58      0\n",
       "2     15619304           502  France  Female   42       8  159660.80                3            1              0         113931.57      1\n",
       "3     15701354           699  France  Female   39       1       0.00                2            0              0          93826.63      0\n",
       "4     15737888           850   Spain  Female   43       2  125510.82                1            1              1          79084.10      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Названия столбцов:\n",
      "['customer_id', 'credit_score', 'country', 'gender', 'age', 'tenure', 'balance', 'products_number', 'credit_card', 'active_member', 'estimated_salary', 'churn']\n"
     ]
    }
   ],
   "source": [
    "def find_csv_file(preferred_names=None):\n",
    "    if preferred_names is None:\n",
    "        preferred_names = [\n",
    "            \"Bank Customer Churn Prediction.csv\",\n",
    "            \"bank_customer_churn.csv\",\n",
    "            \"churn.csv\",\n",
    "            \"data.csv\"\n",
    "        ]\n",
    "    search_dirs = [os.getcwd(), \"/mnt/data\"]\n",
    "    for d in search_dirs:\n",
    "        for name in preferred_names:\n",
    "            path = os.path.join(d, name)\n",
    "            if os.path.exists(path):\n",
    "                return path\n",
    "    for d in search_dirs:\n",
    "        if os.path.exists(d):\n",
    "            csvs = [f for f in os.listdir(d) if f.lower().endswith(\".csv\")]\n",
    "            if csvs:\n",
    "                return os.path.join(d, csvs[0])\n",
    "    raise FileNotFoundError(\"CSV-файл не найден.\")\n",
    "\n",
    "data_path = find_csv_file()\n",
    "df = pd.read_csv(data_path)\n",
    "\n",
    "print(f\"Путь к файлу: {data_path}\")\n",
    "print(f\"Размер датасета: {df.shape}\")\n",
    "display(df.head())\n",
    "print(\"\\nНазвания столбцов:\")\n",
    "print(df.columns.tolist())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "414c90c1",
   "metadata": {},
   "source": [
    "## Подготовка названий столбцов и поиск целевой переменной"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b1d75b11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Нормализованные названия столбцов:\n",
      "['customer_id', 'credit_score', 'country', 'gender', 'age', 'tenure', 'balance', 'products_number', 'credit_card', 'active_member', 'estimated_salary', 'churn']\n",
      "\n",
      "Определённая целевая переменная: churn\n",
      "\n",
      "Распределение классов:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "churn\n",
       "0    7963\n",
       "1    2037\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def normalize_columns(columns):\n",
    "    normalized = []\n",
    "    used = {}\n",
    "    for col in columns:\n",
    "        new_col = str(col).strip().lower()\n",
    "        new_col = re.sub(r\"[^a-zа-я0-9]+\", \"_\", new_col)\n",
    "        new_col = re.sub(r\"_+\", \"_\", new_col).strip(\"_\")\n",
    "        if new_col in used:\n",
    "            used[new_col] += 1\n",
    "            new_col = f\"{new_col}_{used[new_col]}\"\n",
    "        else:\n",
    "            used[new_col] = 0\n",
    "        normalized.append(new_col)\n",
    "    return normalized\n",
    "\n",
    "df = df.copy()\n",
    "df.columns = normalize_columns(df.columns)\n",
    "\n",
    "target_candidates = [\n",
    "    \"churn\", \"exited\", \"target\", \"label\", \"default\", \"response\", \"y\"\n",
    "]\n",
    "\n",
    "target_col = None\n",
    "for col in target_candidates:\n",
    "    if col in df.columns:\n",
    "        target_col = col\n",
    "        break\n",
    "\n",
    "if target_col is None:\n",
    "    binary_cols = [c for c in df.columns if df[c].nunique(dropna=True) == 2]\n",
    "    if not binary_cols:\n",
    "        raise ValueError(\"Не удалось определить целевую переменную автоматически.\")\n",
    "    target_col = binary_cols[-1]\n",
    "\n",
    "print(\"Нормализованные названия столбцов:\")\n",
    "print(df.columns.tolist())\n",
    "print(f\"\\nОпределённая целевая переменная: {target_col}\")\n",
    "print(\"\\nРаспределение классов:\")\n",
    "display(df[target_col].value_counts(dropna=False))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cacbab16",
   "metadata": {},
   "source": [
    "## Отбор признаков и разделение выборки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1b8b5ef9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Удалены идентификационные / несодержательные столбцы: ['customer_id', 'estimated_salary']\n",
      "Числовые признаки: 7\n",
      "Категориальные признаки: 2\n",
      "Итоговое число признаков до кодирования: 9\n",
      "Размер обучающей выборки: (8000, 9)\n",
      "Размер тестовой выборки: (2000, 9)\n"
     ]
    }
   ],
   "source": [
    "def is_identifier(series, name):\n",
    "    name = str(name).lower()\n",
    "    if any(key in name for key in [\"id\", \"customer_id\", \"row_number\", \"surname\"]):\n",
    "        return True\n",
    "    non_na = series.dropna()\n",
    "    if non_na.empty:\n",
    "        return False\n",
    "    unique_ratio = non_na.nunique() / len(non_na)\n",
    "    if unique_ratio > 0.98 and pd.api.types.is_numeric_dtype(series):\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "X = df.drop(columns=[target_col]).copy()\n",
    "y = df[target_col].copy()\n",
    "\n",
    "drop_cols = [col for col in X.columns if is_identifier(X[col], col)]\n",
    "if drop_cols:\n",
    "    print(\"Удалены идентификационные / несодержательные столбцы:\", drop_cols)\n",
    "    X = X.drop(columns=drop_cols)\n",
    "\n",
    "# Приведение целевой переменной к бинарному виду\n",
    "if y.dtype == \"O\":\n",
    "    y = y.astype(str).str.strip().str.lower()\n",
    "    positive_map = {\"1\", \"yes\", \"true\", \"churn\", \"exited\"}\n",
    "    uniques = list(pd.Series(y).dropna().unique())\n",
    "    if len(uniques) == 2:\n",
    "        first, second = uniques[0], uniques[1]\n",
    "        y = y.map({first: 0, second: 1})\n",
    "    else:\n",
    "        raise ValueError(\"Целевая переменная не является бинарной.\")\n",
    "else:\n",
    "    uniq = sorted(pd.Series(y).dropna().unique())\n",
    "    if len(uniq) != 2:\n",
    "        raise ValueError(\"Целевая переменная не является бинарной.\")\n",
    "    # если классы не 0/1, приводим к ним\n",
    "    mapping = {uniq[0]: 0, uniq[1]: 1}\n",
    "    y = y.map(mapping)\n",
    "\n",
    "categorical_cols = X.select_dtypes(include=[\"object\", \"category\", \"bool\"]).columns.tolist()\n",
    "numeric_cols = [c for c in X.columns if c not in categorical_cols]\n",
    "\n",
    "print(f\"Числовые признаки: {len(numeric_cols)}\")\n",
    "print(f\"Категориальные признаки: {len(categorical_cols)}\")\n",
    "print(f\"Итоговое число признаков до кодирования: {X.shape[1]}\")\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=42, stratify=y\n",
    ")\n",
    "\n",
    "print(\"Размер обучающей выборки:\", X_train.shape)\n",
    "print(\"Размер тестовой выборки:\", X_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24e14ba2",
   "metadata": {},
   "source": [
    "## Предобработка признаков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6211959",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер матрицы после кодирования:\n",
      "Train: (8000, 12)\n",
      "Test : (2000, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num__credit_score</th>\n",
       "      <th>num__age</th>\n",
       "      <th>num__tenure</th>\n",
       "      <th>num__balance</th>\n",
       "      <th>num__products_number</th>\n",
       "      <th>num__credit_card</th>\n",
       "      <th>num__active_member</th>\n",
       "      <th>cat__country_France</th>\n",
       "      <th>cat__country_Germany</th>\n",
       "      <th>cat__country_Spain</th>\n",
       "      <th>cat__gender_Female</th>\n",
       "      <th>cat__gender_Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2151</th>\n",
       "      <td>1.058568</td>\n",
       "      <td>1.715086</td>\n",
       "      <td>0.684723</td>\n",
       "      <td>-1.226059</td>\n",
       "      <td>-0.910256</td>\n",
       "      <td>0.641042</td>\n",
       "      <td>-1.030206</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8392</th>\n",
       "      <td>0.913626</td>\n",
       "      <td>-0.659935</td>\n",
       "      <td>-0.696202</td>\n",
       "      <td>0.413288</td>\n",
       "      <td>-0.910256</td>\n",
       "      <td>0.641042</td>\n",
       "      <td>-1.030206</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5006</th>\n",
       "      <td>1.079274</td>\n",
       "      <td>-0.184931</td>\n",
       "      <td>-1.731895</td>\n",
       "      <td>0.601687</td>\n",
       "      <td>0.808830</td>\n",
       "      <td>0.641042</td>\n",
       "      <td>0.970680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117</th>\n",
       "      <td>-0.929207</td>\n",
       "      <td>-0.184931</td>\n",
       "      <td>-0.005739</td>\n",
       "      <td>-1.226059</td>\n",
       "      <td>0.808830</td>\n",
       "      <td>0.641042</td>\n",
       "      <td>-1.030206</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7182</th>\n",
       "      <td>0.427035</td>\n",
       "      <td>0.955079</td>\n",
       "      <td>0.339492</td>\n",
       "      <td>0.548318</td>\n",
       "      <td>0.808830</td>\n",
       "      <td>-1.559960</td>\n",
       "      <td>0.970680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      num__credit_score  num__age  num__tenure  num__balance  num__products_number  num__credit_card  num__active_member  cat__country_France  cat__country_Germany  cat__country_Spain  cat__gender_Female  \\\n",
       "2151           1.058568  1.715086     0.684723     -1.226059             -0.910256          0.641042           -1.030206                  1.0                   0.0                 0.0                 0.0   \n",
       "8392           0.913626 -0.659935    -0.696202      0.413288             -0.910256          0.641042           -1.030206                  0.0                   1.0                 0.0                 0.0   \n",
       "5006           1.079274 -0.184931    -1.731895      0.601687              0.808830          0.641042            0.970680                  0.0                   1.0                 0.0                 1.0   \n",
       "4117          -0.929207 -0.184931    -0.005739     -1.226059              0.808830          0.641042           -1.030206                  1.0                   0.0                 0.0                 0.0   \n",
       "7182           0.427035  0.955079     0.339492      0.548318              0.808830         -1.559960            0.970680                  0.0                   1.0                 0.0                 0.0   \n",
       "\n",
       "      cat__gender_Male  \n",
       "2151               1.0  \n",
       "8392               1.0  \n",
       "5006               0.0  \n",
       "4117               1.0  \n",
       "7182               1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def make_onehot_encoder():\n",
    "    params = inspect.signature(OneHotEncoder).parameters\n",
    "    if \"sparse_output\" in params:\n",
    "        return OneHotEncoder(handle_unknown=\"ignore\", sparse_output=False)\n",
    "    return OneHotEncoder(handle_unknown=\"ignore\", sparse=False)\n",
    "\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
    "    (\"scaler\", StandardScaler())\n",
    "])\n",
    "\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"most_frequent\")),\n",
    "    (\"onehot\", make_onehot_encoder())\n",
    "])\n",
    "\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        (\"num\", numeric_transformer, numeric_cols),\n",
    "        (\"cat\", categorical_transformer, categorical_cols)\n",
    "    ],\n",
    "    remainder=\"drop\"\n",
    ")\n",
    "\n",
    "X_train_prepared = preprocessor.fit_transform(X_train)\n",
    "X_test_prepared = preprocessor.transform(X_test)\n",
    "\n",
    "feature_names = preprocessor.get_feature_names_out()\n",
    "X_train_prepared = pd.DataFrame(X_train_prepared, columns=feature_names, index=X_train.index)\n",
    "X_test_prepared = pd.DataFrame(X_test_prepared, columns=feature_names, index=X_test.index)\n",
    "\n",
    "print(\"Размер матрицы после кодирования:\")\n",
    "print(\"Train:\", X_train_prepared.shape)\n",
    "print(\"Test :\", X_test_prepared.shape)\n",
    "display(X_train_prepared.head())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55cc316a",
   "metadata": {},
   "source": [
    "## Базовая модель по всем признакам"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d46967b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Метрика</th>\n",
       "      <th>CV mean</th>\n",
       "      <th>CV std</th>\n",
       "      <th>Test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.709625</td>\n",
       "      <td>0.011636</td>\n",
       "      <td>0.713000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Precision</td>\n",
       "      <td>0.381868</td>\n",
       "      <td>0.015183</td>\n",
       "      <td>0.387919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Recall</td>\n",
       "      <td>0.687117</td>\n",
       "      <td>0.038849</td>\n",
       "      <td>0.710074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F1</td>\n",
       "      <td>0.490726</td>\n",
       "      <td>0.020632</td>\n",
       "      <td>0.501736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROC-AUC</td>\n",
       "      <td>0.765977</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>0.777863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Метрика   CV mean    CV std      Test\n",
       "0   Accuracy  0.709625  0.011636  0.713000\n",
       "1  Precision  0.381868  0.015183  0.387919\n",
       "2     Recall  0.687117  0.038849  0.710074\n",
       "3         F1  0.490726  0.020632  0.501736\n",
       "4    ROC-AUC  0.765977  0.020055  0.777863"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Матрица ошибок:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Прогноз 0</th>\n",
       "      <th>Прогноз 1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Факт 0</th>\n",
       "      <td>1137</td>\n",
       "      <td>456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Факт 1</th>\n",
       "      <td>118</td>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Прогноз 0  Прогноз 1\n",
       "Факт 0       1137        456\n",
       "Факт 1        118        289"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_model = LogisticRegression(\n",
    "    max_iter=5000,\n",
    "    class_weight=\"balanced\",\n",
    "    solver=\"liblinear\",\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "cv_metrics = cross_validate(\n",
    "    base_model,\n",
    "    X_train_prepared,\n",
    "    y_train,\n",
    "    cv=cv,\n",
    "    scoring=[\"accuracy\", \"precision\", \"recall\", \"f1\", \"roc_auc\"],\n",
    "    n_jobs=-1\n",
    ")\n",
    "\n",
    "base_model.fit(X_train_prepared, y_train)\n",
    "base_pred = base_model.predict(X_test_prepared)\n",
    "base_proba = base_model.predict_proba(X_test_prepared)[:, 1]\n",
    "\n",
    "base_results = pd.DataFrame({\n",
    "    \"Метрика\": [\"Accuracy\", \"Precision\", \"Recall\", \"F1\", \"ROC-AUC\"],\n",
    "    \"CV mean\": [\n",
    "        cv_metrics[\"test_accuracy\"].mean(),\n",
    "        cv_metrics[\"test_precision\"].mean(),\n",
    "        cv_metrics[\"test_recall\"].mean(),\n",
    "        cv_metrics[\"test_f1\"].mean(),\n",
    "        cv_metrics[\"test_roc_auc\"].mean()\n",
    "    ],\n",
    "    \"CV std\": [\n",
    "        cv_metrics[\"test_accuracy\"].std(),\n",
    "        cv_metrics[\"test_precision\"].std(),\n",
    "        cv_metrics[\"test_recall\"].std(),\n",
    "        cv_metrics[\"test_f1\"].std(),\n",
    "        cv_metrics[\"test_roc_auc\"].std()\n",
    "    ],\n",
    "    \"Test\": [\n",
    "        accuracy_score(y_test, base_pred),\n",
    "        precision_score(y_test, base_pred, zero_division=0),\n",
    "        recall_score(y_test, base_pred, zero_division=0),\n",
    "        f1_score(y_test, base_pred, zero_division=0),\n",
    "        roc_auc_score(y_test, base_proba)\n",
    "    ]\n",
    "})\n",
    "\n",
    "display(base_results)\n",
    "\n",
    "print(\"Матрица ошибок:\")\n",
    "display(pd.DataFrame(\n",
    "    confusion_matrix(y_test, base_pred),\n",
    "    index=[\"Факт 0\", \"Факт 1\"],\n",
    "    columns=[\"Прогноз 0\", \"Прогноз 1\"]\n",
    "))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66ccac50",
   "metadata": {},
   "source": [
    "## Значимость и достоверность базовой модели\n",
    "\n",
    "Для оценки статистической значимости используется логистическая регрессия из `statsmodels` на той же подготовленной матрице признаков. Рассматриваются:\n",
    "- LR-тест всей модели;\n",
    "- псевдо-$R^2$ МакФаддена;\n",
    "- p-value коэффициентов.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b0b91267",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Показатель</th>\n",
       "      <th>Значение</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LL-Null</td>\n",
       "      <td>-4.044458e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LL-Model</td>\n",
       "      <td>-3.434565e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LR statistic</td>\n",
       "      <td>1.219788e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LR p-value</td>\n",
       "      <td>7.765311e-256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>McFadden pseudo R^2</td>\n",
       "      <td>1.507974e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Число значимых коэффициентов (p &lt; 0.05)</td>\n",
       "      <td>5.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Показатель       Значение\n",
       "0                                  LL-Null  -4.044458e+03\n",
       "1                                 LL-Model  -3.434565e+03\n",
       "2                             LR statistic   1.219788e+03\n",
       "3                               LR p-value  7.765311e-256\n",
       "4                      McFadden pseudo R^2   1.507974e-01\n",
       "5  Число значимых коэффициентов (p < 0.05)   5.000000e+00"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>num__age</th>\n",
       "      <td>7.427355e-135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__active_member</th>\n",
       "      <td>1.832940e-58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__balance</th>\n",
       "      <td>7.310515e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__credit_score</th>\n",
       "      <td>4.120454e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__products_number</th>\n",
       "      <td>2.416805e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__credit_card</th>\n",
       "      <td>2.826946e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__tenure</th>\n",
       "      <td>5.271578e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__gender_Male</th>\n",
       "      <td>9.999997e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__country_France</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__country_Spain</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__gender_Female</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__country_Germany</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            p_value\n",
       "num__age              7.427355e-135\n",
       "num__active_member     1.832940e-58\n",
       "num__balance           7.310515e-06\n",
       "num__credit_score      4.120454e-03\n",
       "num__products_number   2.416805e-02\n",
       "num__credit_card       2.826946e-01\n",
       "num__tenure            5.271578e-01\n",
       "cat__gender_Male       9.999997e-01\n",
       "cat__country_France    9.999999e-01\n",
       "cat__country_Spain     9.999999e-01\n",
       "cat__gender_Female     9.999999e-01\n",
       "cat__country_Germany   9.999999e-01"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_sm = sm.add_constant(X_train_prepared, has_constant=\"add\")\n",
    "logit_full = sm.Logit(y_train, X_sm).fit(disp=False, maxiter=200)\n",
    "\n",
    "llf = logit_full.llf\n",
    "llnull = logit_full.llnull\n",
    "lr_stat = 2 * (llf - llnull)\n",
    "lr_pvalue = stats.chi2.sf(lr_stat, logit_full.df_model)\n",
    "\n",
    "significant_full = pd.Series(logit_full.pvalues[1:], index=X_train_prepared.columns)\n",
    "significant_full = significant_full.sort_values()\n",
    "\n",
    "full_significance = pd.DataFrame({\n",
    "    \"Показатель\": [\"LL-Null\", \"LL-Model\", \"LR statistic\", \"LR p-value\", \"McFadden pseudo R^2\", \"Число значимых коэффициентов (p < 0.05)\"],\n",
    "    \"Значение\": [\n",
    "        llnull,\n",
    "        llf,\n",
    "        lr_stat,\n",
    "        lr_pvalue,\n",
    "        logit_full.prsquared,\n",
    "        int((significant_full < 0.05).sum())\n",
    "    ]\n",
    "})\n",
    "display(full_significance)\n",
    "display(significant_full.head(20).rename(\"p_value\").to_frame())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14a35fc9",
   "metadata": {},
   "source": [
    "## Классический отбор признаков\n",
    "\n",
    "В качестве классического обёрточного метода используется **RFECV** с логистической регрессией.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5b532dd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Отобрано признаков: 9 из 12\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>selected_feature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>num__credit_score</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>num__age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>num__balance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>num__active_member</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cat__country_France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cat__country_Germany</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cat__country_Spain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cat__gender_Female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>cat__gender_Male</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       selected_feature\n",
       "0     num__credit_score\n",
       "1              num__age\n",
       "2          num__balance\n",
       "3    num__active_member\n",
       "4   cat__country_France\n",
       "5  cat__country_Germany\n",
       "6    cat__country_Spain\n",
       "7    cat__gender_Female\n",
       "8      cat__gender_Male"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "selector = RFECV(\n",
    "    estimator=LogisticRegression(\n",
    "        max_iter=5000,\n",
    "        class_weight=\"balanced\",\n",
    "        solver=\"liblinear\",\n",
    "        random_state=42\n",
    "    ),\n",
    "    step=1,\n",
    "    cv=cv,\n",
    "    scoring=\"f1\",\n",
    "    n_jobs=-1,\n",
    "    min_features_to_select=max(3, min(5, X_train_prepared.shape[1] // 4))\n",
    ")\n",
    "\n",
    "selector.fit(X_train_prepared, y_train)\n",
    "\n",
    "selected_features = X_train_prepared.columns[selector.support_].tolist()\n",
    "print(f\"Отобрано признаков: {len(selected_features)} из {X_train_prepared.shape[1]}\")\n",
    "display(pd.DataFrame({\"selected_feature\": selected_features}))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0264336",
   "metadata": {},
   "source": [
    "## Модель по отобранным признакам"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ddec4c88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Метрика</th>\n",
       "      <th>CV mean</th>\n",
       "      <th>CV std</th>\n",
       "      <th>Test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.709500</td>\n",
       "      <td>0.010204</td>\n",
       "      <td>0.715500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Precision</td>\n",
       "      <td>0.382523</td>\n",
       "      <td>0.013439</td>\n",
       "      <td>0.390541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Recall</td>\n",
       "      <td>0.693252</td>\n",
       "      <td>0.032463</td>\n",
       "      <td>0.710074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F1</td>\n",
       "      <td>0.492909</td>\n",
       "      <td>0.018211</td>\n",
       "      <td>0.503923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROC-AUC</td>\n",
       "      <td>0.766726</td>\n",
       "      <td>0.020669</td>\n",
       "      <td>0.778342</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Метрика   CV mean    CV std      Test\n",
       "0   Accuracy  0.709500  0.010204  0.715500\n",
       "1  Precision  0.382523  0.013439  0.390541\n",
       "2     Recall  0.693252  0.032463  0.710074\n",
       "3         F1  0.492909  0.018211  0.503923\n",
       "4    ROC-AUC  0.766726  0.020669  0.778342"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Матрица ошибок:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Прогноз 0</th>\n",
       "      <th>Прогноз 1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Факт 0</th>\n",
       "      <td>1142</td>\n",
       "      <td>451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Факт 1</th>\n",
       "      <td>118</td>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Прогноз 0  Прогноз 1\n",
       "Факт 0       1142        451\n",
       "Факт 1        118        289"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train_sel = X_train_prepared[selected_features].copy()\n",
    "X_test_sel = X_test_prepared[selected_features].copy()\n",
    "\n",
    "selected_model = LogisticRegression(\n",
    "    max_iter=5000,\n",
    "    class_weight=\"balanced\",\n",
    "    solver=\"liblinear\",\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "cv_metrics_sel = cross_validate(\n",
    "    selected_model,\n",
    "    X_train_sel,\n",
    "    y_train,\n",
    "    cv=cv,\n",
    "    scoring=[\"accuracy\", \"precision\", \"recall\", \"f1\", \"roc_auc\"],\n",
    "    n_jobs=-1\n",
    ")\n",
    "\n",
    "selected_model.fit(X_train_sel, y_train)\n",
    "sel_pred = selected_model.predict(X_test_sel)\n",
    "sel_proba = selected_model.predict_proba(X_test_sel)[:, 1]\n",
    "\n",
    "selected_results = pd.DataFrame({\n",
    "    \"Метрика\": [\"Accuracy\", \"Precision\", \"Recall\", \"F1\", \"ROC-AUC\"],\n",
    "    \"CV mean\": [\n",
    "        cv_metrics_sel[\"test_accuracy\"].mean(),\n",
    "        cv_metrics_sel[\"test_precision\"].mean(),\n",
    "        cv_metrics_sel[\"test_recall\"].mean(),\n",
    "        cv_metrics_sel[\"test_f1\"].mean(),\n",
    "        cv_metrics_sel[\"test_roc_auc\"].mean()\n",
    "    ],\n",
    "    \"CV std\": [\n",
    "        cv_metrics_sel[\"test_accuracy\"].std(),\n",
    "        cv_metrics_sel[\"test_precision\"].std(),\n",
    "        cv_metrics_sel[\"test_recall\"].std(),\n",
    "        cv_metrics_sel[\"test_f1\"].std(),\n",
    "        cv_metrics_sel[\"test_roc_auc\"].std()\n",
    "    ],\n",
    "    \"Test\": [\n",
    "        accuracy_score(y_test, sel_pred),\n",
    "        precision_score(y_test, sel_pred, zero_division=0),\n",
    "        recall_score(y_test, sel_pred, zero_division=0),\n",
    "        f1_score(y_test, sel_pred, zero_division=0),\n",
    "        roc_auc_score(y_test, sel_proba)\n",
    "    ]\n",
    "})\n",
    "\n",
    "display(selected_results)\n",
    "\n",
    "print(\"Матрица ошибок:\")\n",
    "display(pd.DataFrame(\n",
    "    confusion_matrix(y_test, sel_pred),\n",
    "    index=[\"Факт 0\", \"Факт 1\"],\n",
    "    columns=[\"Прогноз 0\", \"Прогноз 1\"]\n",
    "))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f413c667",
   "metadata": {},
   "source": [
    "## Значимость и достоверность модели по отобранным признакам"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6f84b3cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Показатель</th>\n",
       "      <th>Значение</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LL-Null</td>\n",
       "      <td>-4.044458e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LL-Model</td>\n",
       "      <td>-3.437970e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LR statistic</td>\n",
       "      <td>1.212977e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LR p-value</td>\n",
       "      <td>1.102986e-257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>McFadden pseudo R^2</td>\n",
       "      <td>1.499555e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Число значимых коэффициентов (p &lt; 0.05)</td>\n",
       "      <td>4.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Показатель       Значение\n",
       "0                                  LL-Null  -4.044458e+03\n",
       "1                                 LL-Model  -3.437970e+03\n",
       "2                             LR statistic   1.212977e+03\n",
       "3                               LR p-value  1.102986e-257\n",
       "4                      McFadden pseudo R^2   1.499555e-01\n",
       "5  Число значимых коэффициентов (p < 0.05)   4.000000e+00"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>num__age</th>\n",
       "      <td>5.426061e-136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__active_member</th>\n",
       "      <td>1.171139e-58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__balance</th>\n",
       "      <td>9.379447e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num__credit_score</th>\n",
       "      <td>4.113252e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__gender_Male</th>\n",
       "      <td>9.999998e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__country_France</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__country_Spain</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__country_Germany</th>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat__gender_Female</th>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            p_value\n",
       "num__age              5.426061e-136\n",
       "num__active_member     1.171139e-58\n",
       "num__balance           9.379447e-08\n",
       "num__credit_score      4.113252e-03\n",
       "cat__gender_Male       9.999998e-01\n",
       "cat__country_France    9.999999e-01\n",
       "cat__country_Spain     9.999999e-01\n",
       "cat__country_Germany   9.999999e-01\n",
       "cat__gender_Female     1.000000e+00"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_sm_sel = sm.add_constant(X_train_sel, has_constant=\"add\")\n",
    "logit_sel = sm.Logit(y_train, X_sm_sel).fit(disp=False, maxiter=200)\n",
    "\n",
    "llf_sel = logit_sel.llf\n",
    "llnull_sel = logit_sel.llnull\n",
    "lr_stat_sel = 2 * (llf_sel - llnull_sel)\n",
    "lr_pvalue_sel = stats.chi2.sf(lr_stat_sel, logit_sel.df_model)\n",
    "\n",
    "significant_sel = pd.Series(logit_sel.pvalues[1:], index=X_train_sel.columns).sort_values()\n",
    "\n",
    "sel_significance = pd.DataFrame({\n",
    "    \"Показатель\": [\"LL-Null\", \"LL-Model\", \"LR statistic\", \"LR p-value\", \"McFadden pseudo R^2\", \"Число значимых коэффициентов (p < 0.05)\"],\n",
    "    \"Значение\": [\n",
    "        llnull_sel,\n",
    "        llf_sel,\n",
    "        lr_stat_sel,\n",
    "        lr_pvalue_sel,\n",
    "        logit_sel.prsquared,\n",
    "        int((significant_sel < 0.05).sum())\n",
    "    ]\n",
    "})\n",
    "display(sel_significance)\n",
    "display(significant_sel.rename(\"p_value\").to_frame())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66e63cee",
   "metadata": {},
   "source": [
    "## Сравнение моделей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6e3cf211",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Метрика</th>\n",
       "      <th>CV mean_all_features</th>\n",
       "      <th>CV std_all_features</th>\n",
       "      <th>Test_all_features</th>\n",
       "      <th>CV mean_selected_features</th>\n",
       "      <th>CV std_selected_features</th>\n",
       "      <th>Test_selected_features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.709625</td>\n",
       "      <td>0.011636</td>\n",
       "      <td>0.713000</td>\n",
       "      <td>0.709500</td>\n",
       "      <td>0.010204</td>\n",
       "      <td>0.715500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Precision</td>\n",
       "      <td>0.381868</td>\n",
       "      <td>0.015183</td>\n",
       "      <td>0.387919</td>\n",
       "      <td>0.382523</td>\n",
       "      <td>0.013439</td>\n",
       "      <td>0.390541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Recall</td>\n",
       "      <td>0.687117</td>\n",
       "      <td>0.038849</td>\n",
       "      <td>0.710074</td>\n",
       "      <td>0.693252</td>\n",
       "      <td>0.032463</td>\n",
       "      <td>0.710074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F1</td>\n",
       "      <td>0.490726</td>\n",
       "      <td>0.020632</td>\n",
       "      <td>0.501736</td>\n",
       "      <td>0.492909</td>\n",
       "      <td>0.018211</td>\n",
       "      <td>0.503923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROC-AUC</td>\n",
       "      <td>0.765977</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>0.777863</td>\n",
       "      <td>0.766726</td>\n",
       "      <td>0.020669</td>\n",
       "      <td>0.778342</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Метрика  CV mean_all_features  CV std_all_features  Test_all_features  CV mean_selected_features  CV std_selected_features  Test_selected_features\n",
       "0   Accuracy              0.709625             0.011636           0.713000                   0.709500                  0.010204                0.715500\n",
       "1  Precision              0.381868             0.015183           0.387919                   0.382523                  0.013439                0.390541\n",
       "2     Recall              0.687117             0.038849           0.710074                   0.693252                  0.032463                0.710074\n",
       "3         F1              0.490726             0.020632           0.501736                   0.492909                  0.018211                0.503923\n",
       "4    ROC-AUC              0.765977             0.020055           0.777863                   0.766726                  0.020669                0.778342"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Модель</th>\n",
       "      <th>Число признаков</th>\n",
       "      <th>LR p-value</th>\n",
       "      <th>McFadden pseudo R^2</th>\n",
       "      <th>Test Accuracy</th>\n",
       "      <th>Test F1</th>\n",
       "      <th>Test ROC-AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Все признаки</td>\n",
       "      <td>12</td>\n",
       "      <td>7.765311e-256</td>\n",
       "      <td>0.150797</td>\n",
       "      <td>0.7130</td>\n",
       "      <td>0.501736</td>\n",
       "      <td>0.777863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Отобранные признаки</td>\n",
       "      <td>9</td>\n",
       "      <td>1.102986e-257</td>\n",
       "      <td>0.149955</td>\n",
       "      <td>0.7155</td>\n",
       "      <td>0.503923</td>\n",
       "      <td>0.778342</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Модель  Число признаков     LR p-value  McFadden pseudo R^2  Test Accuracy   Test F1  Test ROC-AUC\n",
       "0         Все признаки               12  7.765311e-256             0.150797         0.7130  0.501736      0.777863\n",
       "1  Отобранные признаки                9  1.102986e-257             0.149955         0.7155  0.503923      0.778342"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "comparison = base_results.merge(\n",
    "    selected_results,\n",
    "    on=\"Метрика\",\n",
    "    suffixes=(\"_all_features\", \"_selected_features\")\n",
    ")\n",
    "display(comparison)\n",
    "\n",
    "comparison_significance = pd.DataFrame({\n",
    "    \"Модель\": [\"Все признаки\", \"Отобранные признаки\"],\n",
    "    \"Число признаков\": [X_train_prepared.shape[1], X_train_sel.shape[1]],\n",
    "    \"LR p-value\": [lr_pvalue, lr_pvalue_sel],\n",
    "    \"McFadden pseudo R^2\": [logit_full.prsquared, logit_sel.prsquared],\n",
    "    \"Test Accuracy\": [\n",
    "        accuracy_score(y_test, base_pred),\n",
    "        accuracy_score(y_test, sel_pred)\n",
    "    ],\n",
    "    \"Test F1\": [\n",
    "        f1_score(y_test, base_pred, zero_division=0),\n",
    "        f1_score(y_test, sel_pred, zero_division=0)\n",
    "    ],\n",
    "    \"Test ROC-AUC\": [\n",
    "        roc_auc_score(y_test, base_proba),\n",
    "        roc_auc_score(y_test, sel_proba)\n",
    "    ]\n",
    "})\n",
    "display(comparison_significance)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a32e2d7e",
   "metadata": {},
   "source": [
    "## Визуальное сравнение качества"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "08e06fd7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHWCAYAAABACtmGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAab5JREFUeJzt3X98zfX///H7OWObYfNrNj/GMD/zKxveKFQyofzoXX4kM5GKouV3WKgWlVREyo/eKPIjFVJaVGqRHxPl98/CmJ+zaRs7r+8fvjsfp7M5O5pey7ldL5dzuXg9X8/X6/V4vfY65+lxns/X81gMwzAEAAAAAMiV1ewAAAAAAKCgI3ECAAAAABdInAAAAADABRInAAAAAHCBxAkAAAAAXCBxAgAAAAAXSJwAAAAAwAUSJwAAAABwgcQJAAAgFzabTadPn9bBgwfNDgWAyUicAAD/WitXrlRiYqJ9ecWKFfr111/NCwh5tmHDBq1fv96+vH79ev3www/mBXSNpKQkDRkyRJUrV5a3t7cCAwNVp04dpaSkmB0aABOROAHIswMHDmjAgAGqWrWqfH195e/vrxYtWujNN9/Un3/+aXZ48EA7duzQ4MGDtW/fPv3000964okndPHiRbPDQh78/vvveuqpp7Rjxw7t2LFDTz31lH7//Xezw9L+/fvVuHFjLVq0SAMGDNDKlSu1du1axcfHq2jRomaHB8BEFsMwDLODAFDwrVq1Sg899JB8fHzUu3dv1a1bV5mZmdqwYYOWLVumPn36aNasWWaHCQ+TnJys5s2ba//+/ZKkrl27atmyZSZHhbzIyMhQy5YttWnTJklSs2bNtH79enl7e5sa1z333KPDhw/ru+++U4UKFUyNBUDBQuIEwKVDhw6pfv36qlixor755huVK1fOYf3+/fu1atUqDR482KQI4ckyMjK0c+dO+fn5qXbt2maHAzdkZWVp586dkqS6devKy8vL1Hi2bNmiiIgIffXVV7r33ntNjQVAwcNQPQAuTZ48WampqZo9e7ZT0iRJYWFhDkmTxWLRoEGDtHDhQtWsWVO+vr4KDw/Xd99957DdkSNH9NRTT6lmzZoqUqSISpcurYceekiHDx92qDdv3jxZLBb7y8/PT/Xq1dP777/vUK9Pnz4qVqyYU3xLly6VxWJxeJ5CkjZu3Kh27dopICBAfn5+atWqldMzFi+88IIsFotOnz7tUL5582ZZLBbNmzfP4fihoaEO9X7//XcVKVJEFovF6by++OIL3XnnnSpatKiKFy+uDh065On5nOzr4e3treTkZId1CQkJ9uu0efNmh3VLlixReHi4ihQpojJlyqhXr146duxYjscIDQ11uObZr2vPV7r64PzUqVN12223ydfXV0FBQRowYIDOnTvntM/169fnuM+/XrOMjAzFxsYqLCxMPj4+CgkJ0fDhw5WRkeFQz2Kx6IUXXpCPj4/Cw8NVu3Ztvfrqq7JYLGrdurXL65h9n/5Vx44dnWJ67bXX1Lx5c5UuXVpFihRReHi4li5d6vIYrVu3zvGcr31da8GCBfa/UalSpdS9e/cch69t3LhR7du3V8mSJVW0aFHVr19fb775pqSr96GrY157L77zzju67bbb5OPjo/Lly2vgwIE6f/78dc+jTJky6tChgz3pyXblyhVNnDhR1apVk4+Pj0JDQzV69Ginv11oaKj69OkjLy8vNWjQQA0aNNDy5ctzvB9ycu39abVaFRwcrG7duuno0aP2OocPH5bFYtFrr72W636y39/ZfvrpJ/n6+urAgQP2axIcHKwBAwbo7NmzTtvn5T2V/bl08OBBRUZGqmjRoipfvrwmTJiga7+7zo732vfYxYsXFR4eripVqujEiRP2cnfedwDyTyGzAwBQ8H3++eeqWrWqmjdvnudtvv32Wy1evFjPPPOMfHx89M4776hdu3batGmT6tatK0n6+eef9eOPP6p79+6qWLGiDh8+rBkzZqh169b67bff5Ofn57DPN954Q2XKlFFKSormzJmj/v37KzQ0VG3atHH7nL755hvdd999Cg8PV2xsrKxWq+bOnau7775b33//vZo0aeL2PnMybtw4paenO5XPnz9fUVFRioyM1KRJk3Tp0iXNmDFDd9xxh7Zt25an/zx6eXlpwYIFevbZZ+1lc+fOla+vr9Mx582bp+joaDVu3FhxcXE6efKk3nzzTf3www/atm2bSpQo4bT/hg0b6rnnnpN0tddx3LhxTnUGDBhg3/czzzyjQ4cOadq0adq2bZt++OEHFS5c2Gmb0aNH23uGZs2a5fCfXZvNpgceeEAbNmzQ448/rtq1a2vHjh164403tHfvXq1YsSLX63H+/HnFxcVd95rdqDfffFMPPPCAHnnkEWVmZmrRokV66KGHtHLlSnXo0CHX7Z5//nn169dPknT69Gk9++yzevzxx3XnnXc61X3ppZc0duxYPfzww+rXr5+Sk5P19ttvq2XLlg5/o7Vr16pjx44qV66cBg8erODgYO3atUsrV67U4MGDNWDAAIf3xKOPPqouXbqoa9eu9rLAwEBJVxOH8ePHq02bNnryySe1Z88ezZgxQz///LPT369WrVp6/vnnZRiGDhw4oClTpqh9+/YOf79+/frpgw8+0H//+18999xz2rhxo+Li4rRr1y598sknuV6nK1eu6Pnnn3fxV3B055136vHHH5fNZtPOnTs1depUHT9+XN9//71b+7nWmTNnlJ6erieffFJ33323nnjiCR04cEDTp0/Xxo0btXHjRvn4+Ehy7z2VlZWldu3a6T//+Y8mT56sNWvWKDY2VleuXNGECRNyjOXy5ct68MEHdfToUf3www8OX1rdyPsOQD4wAOA6Lly4YEgyOnXqlOdtJBmSjM2bN9vLjhw5Yvj6+hpdunSxl126dMlp24SEBEOS8b///c9eNnfuXEOScejQIXvZ3r17DUnG5MmT7WVRUVFG0aJFnfa5ZMkSQ5Kxbt06wzAMw2azGdWrVzciIyMNm83mEE+VKlWMe++9114WGxtrSDKSk5Md9vnzzz8bkoy5c+c6HL9y5cr25Z07dxpWq9W47777HOK/ePGiUaJECaN///4O+0xKSjICAgKcyv8q+3r06NHDqFevnr08LS3N8Pf3N3r27GlIMn7++WfDMAwjMzPTKFu2rFG3bl3jzz//tNdfuXKlIckYN26c0zHKly9vdOzY8brn+/333xuSjIULFzpsu2bNmhzL165da0gyvv3221yv2fz58w2r1Wp8//33DtvOnDnTkGT88MMP9jJJRmxsrH15+PDhRtmyZY3w8HCjVatWOVw5R5KMgQMHOpV36NDBISbDcL5XMzMzjbp16xp33323y+NkO3TokNM1zHb48GHDy8vLeOmllxzKd+zYYRQqVMhefuXKFaNKlSpG5cqVjXPnzjnUvfZevtZfr1O2U6dOGd7e3kbbtm2NrKwse/m0adMMScacOXPsZa1atXK6pqNHjzYkGadOnTIMwzASExMNSUa/fv0c6g0dOtSQZHzzzTf2ssqVKxtRUVH25Xfeecfw8fEx7rrrLqdrn5O/bm8YhtGzZ0/Dz8/Pvpx9vV999dVc95P9/v7r8j333GNcuXLFXp79nnv77bcNw3DvPRUVFWVIMp5++ml7mc1mMzp06GB4e3vbP1uuvT9sNpvxyCOPGH5+fsbGjRsdYnb3fQcg/zBUD8B1ZU+/W7x4cbe2a9asmcLDw+3LlSpVUqdOnfTll18qKytLklSkSBH7+suXL+vMmTMKCwtTiRIltHXrVqd9njt3zv57Km+88Ya8vLzUqlUrp3qnT592eP11lrXExETt27dPPXv21JkzZ+z10tLSdM899+i7776TzWZz2Obs2bMO+7xw4YLLazBq1Cg1atRIDz30kEP52rVrdf78efXo0cNhn15eXmratKnWrVvnct/S1Z6E3bt324fkLVu2TAEBAbrnnnsc6m3evFmnTp3SU089JV9fX3t5hw4dVKtWLa1atcpp3+np6Q51c7JkyRIFBATo3nvvdTiP8PBwFStWzOk8MjMzJcn+jX1u+6xdu7Zq1arlsM+7775bknK9NseOHdPbb7+tsWPH5jhcMzfp6elO98vly5ed6l17r547d04XLlzQnXfemeN9eiOWL18um82mhx9+2CGW4OBgVa9e3X7e27Zt06FDhzRkyBCnXsK/Dvtz5euvv1ZmZqaGDBkiq/X//jvQv39/+fv7O90Xly9f1unTp5WcnKyEhAR98sknql+/vsqUKSNJWr16tSQpJibGYbvsXsuc7jNJunTpkiZMmKBBgwapUqVKeY4/IyNDp0+f1qlTp7R27Vp98803Tvd+9v5Pnz6tc+fOOQyNu56YmBiH560effRRBQUF2c/hRt5T1w4LzR4mmpmZqa+//tqp7rBhw7Rw4UJ9/PHHTr3f7r7vAOQfhuoBuC5/f39JcnuK5+rVqzuV1ahRQ5cuXVJycrKCg4P1559/Ki4uTnPnztWxY8cc/lOTU2LSqFEj+799fHw0bdo0p/9UpKWl2Ych5Wbfvn2SpKioqFzrXLhwQSVLlrQv16xZ87r7/KsNGzbo888/V3x8vMNQpmuPn50M/FX2NXclMDBQHTp00Jw5cxQREaE5c+YoKirK4T/B0tVnyXI7h1q1amnDhg0OZVlZWTp//rwCAgKue/x9+/bpwoULKlu2bI7rT5065bCc/dzM9RKbffv2adeuXbn+Df+6z2yxsbEqX768BgwYkKdnj7LNnj1bs2fPdiqvXLmyw/LKlSv14osvKjEx0eF5HXeTldzs27dPhmHk+L6RZB96deDAAUmyD3f9O3K7L7y9vVW1alX7+mw//vijw9+levXqWrFihf0aHDlyRFarVWFhYQ7bBQcHq0SJEk77yzZlyhSlp6dr9OjRTknX9SxatEiLFi2yLzdu3NjpuUfp6r0RGxsrSfL19dXdd9+tqVOn5nits8+lVq1aDuVeXl6qXr26/dkwd99TVqtVVatWdSirUaOGJDk9+/juu+/qp59+kqQcn1ly930HIP+QOAG4Ln9/f5UvX97pIfD88PTTT2vu3LkaMmSImjVrpoCAAFksFnXv3t2px0e6+uB8UFCQ0tPT9c0332jgwIHy9fVVnz597HV8fX31+eefO2z3/fffOzxHkL3vV199VQ0bNswxtr/+537ZsmUOCc3evXs1cODAXM9txIgRioyM1N13353jhArS1eecgoODnbYtVCjvH819+/ZV79699fTTT+u7777T+++//7ee8ZCko0ePymazuXzOymazqWzZslq4cGGO6/+a/CQlJUlSjud87T7r1aunKVOm5Lg+JCTEqWzXrl2aN2+eFixY4PazHZ06dXKaIGLMmDH2WKWr988DDzygli1b6p133lG5cuVUuHBhzZ07Vx9++KFbx8uNzWaTxWLRF198kePMcu70ot0s9evX1+uvvy7p6jTwb731llq3bq2tW7c6/E3dSSZPnz6tV199VaNGjVKpUqXciqdt27YaNmyYJOmPP/7QpEmTdNddd2nz5s0OPYSPP/64HnroIWVlZWnXrl164YUX1Llz5xwnYrl2O7P89NNPeumll/Tzzz/r2WefVbt27ey9epL77zsA+YfECYBLHTt21KxZs5SQkKBmzZrlaZvsXpVr7d27V35+fvaGfenSpYqKirL/Z0y6OnTqrzN6ZWvRooX9P/MdO3bUr7/+qri4OIfEycvLy2myiL/ur1q1apKuJoV5nViiZcuWDv95yWkyhWwrVqxQQkJCrsO4so9ftmzZG5rY4lr33XeffH191b17d91xxx2qVq2aU+KU3XuyZ88ep16uPXv2OPWuZA/9i4iIuO6xq1Wrpq+//lotWrTI0384f/vtNwUGBqp06dLX3ef27dt1zz335Pk/4KNGjVLDhg3VrVu3PNW/VsWKFZ3+BlOnTnVInJYtWyZfX199+eWXDsMM586d6/bxclOtWjUZhqEqVarYeyJyqydJO3fu/Nv3zrX3xbW9IZmZmTp06JDT/kuWLOlQ1rp1a5UvX15z587VqFGjVLlyZdlsNu3bt89hWviTJ0/q/PnzTveZJL344osqXrz4Df2UQbly5RziqVmzppo3b64VK1aoR48e9vLq1avb60VGRurSpUt6/vnnnXqCJalKlSqSnK9J9nndfvvtktx/T9lsNh08eNDhb7t3715JcvqCom/fvho9erSOHz+uOnXq6Nlnn9X8+fPt69193wHIPzzjBMCl4cOHq2jRourXr59OnjzptP7AgQP2qZCz/TVx+P333/Xpp5+qbdu29m/Uvby8nJ45ePvtt+3PQLny559/Ok1znBfh4eGqVq2aXnvtNaWmpjqt/+sU3+7IysrS6NGj1bNnz1x7syIjI+Xv76+XX345x+dp3Dl+oUKF1Lt3b/3yyy/q27dvjnUiIiJUtmxZzZw50+F6ffHFF9q1a5fTrHBLlixRiRIlcnx+7FoPP/ywsrKyNHHiRKd1V65ccUhYL168qNWrV+c6PPHafR47dkzvvfee07o///xTaWlpDmUJCQn69NNP9corr+TbsLm/8vLyksVicbgvDx8+fN0Z/tzVtWtXeXl5afz48U7vCcMwdObMGUlXh6tWqVJFU6dOdfpCIK/P72Rr06aNvL299dZbbzlsO3v2bF24cOG6swVKV/8ekuz3VPv27SVdTTyvld17+Nf9Zc+i+cILL+RLAvDXeHKT3eObU8/ePffcIx8fH7311lsOvd4LFy7UyZMn1bFjR0nuv6ckadq0afZ/G4ahadOmqXDhwk7PZWXPuFi+fHlNmjRJCxYs0FdffWVf7877DkD+oscJgEvVqlXThx9+qG7duql27drq3bu36tatq8zMTP34449asmSJQ6+PdPUZjMjISIfpyCVp/Pjx9jodO3bU/PnzFRAQoDp16ighIUFff/11rj0SK1asUJkyZexD9b7//nsNGTLE7fOxWq16//33dd999+m2225TdHS0KlSooGPHjmndunXy9/d3Gu6XV3/88Ye8vb3tD8rnxN/fXzNmzNCjjz6qRo0aqXv37goMDNTRo0e1atUqtWjRwuE/Wa5MnDhRw4YNc3gm61qFCxfWpEmTFB0drVatWqlHjx72qZNDQ0Pt05mfPHlSb731lpYsWaKWLVtq2bJl9n0cOnRI0tVEpVGjRqpfv75atWqlAQMGKC4uTomJiWrbtq0KFy6sffv2acmSJXrzzTf13//+Vx9//LHGjx+vc+fOaeTIkdc9l0cffVQff/yxnnjiCa1bt04tWrRQVlaWdu/erY8//lhffvmlQ09Y9g+V/t3el+vp0KGDpkyZonbt2qlnz546deqUpk+frrCwMP3yyy/5coxq1arpxRdf1KhRo3T48GF17txZxYsX16FDh/TJJ5/o8ccf19ChQ2W1WjVjxgzdf//9atiwoaKjo1WuXDnt3r1bv/76q7788ss8HzMwMFCjRo3S+PHj1a5dOz3wwAPas2eP3nnnHTVu3Fi9evVyqH/y5EktWLBA0tUhdu+++64KFSpkTyYaNGigqKgozZo1S+fPn1erVq20adMmffDBB+rcubPuuusuh/19++23ql27tqKjo2/omh08eNAez7FjxzRt2jT5+/s7JSJ79uzRmjVrZLPZ9Ntvv+nVV19V48aNVaFCBad9lipVSmPGjNHYsWMVGRmpTp066eDBg5o2bZoaNGhgn1o+r++pbL6+vlqzZo2ioqLUtGlTffHFF1q1apVGjx593aF1jz/+uD788EM98cQT9h95zuv7DsBNYNZ0fgD+ffbu3Wv079/fCA0NNby9vY3ixYsbLVq0MN5++20jPT3dXk//f5rnBQsWGNWrVzd8fHyM22+/3T4deLZz584Z0dHRRpkyZYxixYoZkZGRxu7du52mGs6eCjj75e3tbYSFhRnjxo1zOG5epyPPtm3bNqNr165G6dKlDR8fH6Ny5crGww8/bMTHx9vruDsduSRj8ODBDnVzmk7dMAxj3bp1RmRkpBEQEGD4+voa1apVM/r06eMwjXtOsveXPd14XtcvXrzYuP322w0fHx+jVKlSxiOPPGL88ccfDvFce51ze/11autZs2YZ4eHhRpEiRYzixYsb9erVM4YPH24cP37cMAzD6NKli3Hfffc5Taucfc3+Ov10ZmamMWnSJOO2224zfHx8jJIlSxrh4eHG+PHjjQsXLtjrSTIsFouxZcsWh+1zmjo7J9n36V/lNB357Nmz7fdyrVq1jLlz5zpNZe3K9aYjz7Zs2TLjjjvuMIoWLWoULVrUqFWrljFw4EBjz549DvU2bNhg3HvvvUbx4sWNokWLGvXr17dPlZ3TeeY0HXm2adOmGbVq1TIKFy5sBAUFGU8++aTTVOetWrVyuAdKlChhtGjRwli9erVDvcuXLxvjx483qlSpYhQuXNgICQkxRo0a5fA+NYyr04lLMj755BOH8pzuh5xkb5/9KlOmjNG2bVsjISHBXif7eme/rFarUbFiRSMqKsp+3+f2N5w+fbrDNRkwYIBx5swZp3qu3lPZ51S0aFHjwIEDRtu2bQ0/Pz8jKCjIiI2NdZgGPrf7Y8+ePYavr6/x7LPPOpS7et8ByH8Ww3Czbx8AXLBYLBo4cKBbvSYw3/r163XXXXddd8hXnz59FBoaqhdeeOGfCwz4F+vTp4+WLl2a47BgAP8uPOMEAAAAAC7wjBMAQJIUFBSkRx555Lp1mjdv7jC7IAAAnoLECQAgSapdu7b9YfvcPP744/9QNAAAFCw84wQAAAAALvCMEwAAAAC4QOIEAAAAAC543DNONptNx48fV/HixW/ar8wDAAAAKPgMw9DFixdVvnx5Wa3X71PyuMTp+PHjCgkJMTsMAAAAAAXE77//rooVK163jsclTsWLF5d09eL4+/ubHA0AAAAAs6SkpCgkJMSeI1yPxyVO2cPz/P39SZwAAAAA5OkRHiaHAAAAAAAXTE+cpk+frtDQUPn6+qpp06batGnTdetPnTpVNWvWVJEiRRQSEqJnn31W6enp/1C0AAAAADyRqYnT4sWLFRMTo9jYWG3dulUNGjRQZGSkTp06lWP9Dz/8UCNHjlRsbKx27dql2bNna/HixRo9evQ/HDkAAAAAT2Jq4jRlyhT1799f0dHRqlOnjmbOnCk/Pz/NmTMnx/o//vijWrRooZ49eyo0NFRt27ZVjx49XPZSAQAAAMDfYdrkEJmZmdqyZYtGjRplL7NarWrTpo0SEhJy3KZ58+ZasGCBNm3apCZNmujgwYNavXq1Hn300VyPk5GRoYyMDPtySkqKpKu/52Sz2fLpbAAAAAD827iTD5iWOJ0+fVpZWVkKCgpyKA8KCtLu3btz3KZnz546ffq07rjjDhmGoStXruiJJ5647lC9uLg4jR8/3qk8OTmZZ6MAAAAAD3bx4sU81/1XTUe+fv16vfzyy3rnnXfUtGlT7d+/X4MHD9bEiRM1duzYHLcZNWqUYmJi7MvZc7UHBgYyHTkAAADgwXx9ffNc17TEqUyZMvLy8tLJkycdyk+ePKng4OActxk7dqweffRR9evXT5JUr149paWl6fHHH9fzzz8vq9X5kS0fHx/5+Pg4lVut1hzrAwAAAPAM7uQDpmUO3t7eCg8PV3x8vL3MZrMpPj5ezZo1y3GbS5cuOZ2cl5eXJMkwjJsXLAAAAACPZupQvZiYGEVFRSkiIkJNmjTR1KlTlZaWpujoaElS7969VaFCBcXFxUmS7r//fk2ZMkW33367faje2LFjdf/999sTKAAAAADIb6YmTt26dVNycrLGjRunpKQkNWzYUGvWrLFPGHH06FGHHqYxY8bIYrFozJgxOnbsmAIDA3X//ffrpZdeMusUAAAAAHgAi+FhY9xSUlIUEBCgCxcuMDkEAAAA4MHcyQ2YHQEAAAAAXCBxAgAAAAAXSJwAAAAAwIV/1Q/gArh11PugntkhmG5H1A6zQwAA03l6e0Bb8O9BjxMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4QOIEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4QOIEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhQyOwAAACA5woducrsEEx1+JUOZocAII9InGAKT28oJRpLAACAfxOG6gEAAACACwUicZo+fbpCQ0Pl6+urpk2batOmTbnWbd26tSwWi9OrQwe+vQcAAABwc5ieOC1evFgxMTGKjY3V1q1b1aBBA0VGRurUqVM51l++fLlOnDhhf+3cuVNeXl566KGH/uHIAQAAAHgK0xOnKVOmqH///oqOjladOnU0c+ZM+fn5ac6cOTnWL1WqlIKDg+2vtWvXys/Pj8QJAAAAwE1j6uQQmZmZ2rJli0aNGmUvs1qtatOmjRISEvK0j9mzZ6t79+4qWrRojuszMjKUkZFhX05JSZEk2Ww22Wy2vxE9/g6rDLNDMJ2n339W87+3MZ2n3wOARHvA5wDtAfeAudy5/qYmTqdPn1ZWVpaCgoIcyoOCgrR7926X22/atEk7d+7U7Nmzc60TFxen8ePHO5UnJycrPT3d/aCRL2qX9OyGUlKuw1E9RfVC1c0OwXSefg8AEu0BnwO0B9wD5rp48WKe6/6rpyOfPXu26tWrpyZNmuRaZ9SoUYqJibEvp6SkKCQkRIGBgfL39/8nwkQOdp2zmB2C6cqWLWt2CKbad2Wf2SGYztPvAUCiPeBzgPaAe8Bcvr6+ea5rauJUpkwZeXl56eTJkw7lJ0+eVHBw8HW3TUtL06JFizRhwoTr1vPx8ZGPj49TudVqldXq2V3DZrLJsxtKSR5//9nE0ARPvwcAifaAzwHaA+4Bc7lz/U39S3l7eys8PFzx8fH2MpvNpvj4eDVr1uy62y5ZskQZGRnq1avXzQ4TAAAAgIczfaheTEyMoqKiFBERoSZNmmjq1KlKS0tTdHS0JKl3796qUKGC4uLiHLabPXu2OnfurNKlS5sRNgAAAAAPYnri1K1bNyUnJ2vcuHFKSkpSw4YNtWbNGvuEEUePHnXqQtuzZ482bNigr776yoyQAQAAAHgY0xMnSRo0aJAGDRqU47r169c7ldWsWVOG4dmz8AAAAAD45/A0GgAAAAC4QOIEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4QOIEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4QOIEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4ILpidP06dMVGhoqX19fNW3aVJs2bbpu/fPnz2vgwIEqV66cfHx8VKNGDa1evfofihYAAACAJypk5sEXL16smJgYzZw5U02bNtXUqVMVGRmpPXv2qGzZsk71MzMzde+996ps2bJaunSpKlSooCNHjqhEiRL/fPAAAAAAPIapidOUKVPUv39/RUdHS5JmzpypVatWac6cORo5cqRT/Tlz5ujs2bP68ccfVbhwYUlSaGjoPxkyAAAAAA9kWuKUmZmpLVu2aNSoUfYyq9WqNm3aKCEhIcdtPvvsMzVr1kwDBw7Up59+qsDAQPXs2VMjRoyQl5dXjttkZGQoIyPDvpySkiJJstlsstls+XhGcIdVhtkhmM7T7z+r+SOFTefp9wAg0R7wOUB7wD1gLneuv2mJ0+nTp5WVlaWgoCCH8qCgIO3evTvHbQ4ePKhvvvlGjzzyiFavXq39+/frqaee0uXLlxUbG5vjNnFxcRo/frxTeXJystLT0//+ieCG1C7p2Q2lJJ06dcrsEExVvVB1s0MwnaffA4BEe8DnAO0B94C5Ll68mOe6pg7Vc5fNZlPZsmU1a9YseXl5KTw8XMeOHdOrr76aa+I0atQoxcTE2JdTUlIUEhKiwMBA+fv7/1Oh4y92nbOYHYLpcnqOz5Psu7LP7BBM5+n3ACDRHvA5QHvAPWAuX1/fPNc1LXEqU6aMvLy8dPLkSYfykydPKjg4OMdtypUrp8KFCzsMy6tdu7aSkpKUmZkpb29vp218fHzk4+PjVG61WmW1enbXsJls8uyGUpLH3382MTTB0+8BQKI94HOA9oB7wFzuXH/T/lLe3t4KDw9XfHy8vcxmsyk+Pl7NmjXLcZsWLVpo//79DmMR9+7dq3LlyuWYNAEAAABAfjA1xY2JidF7772nDz74QLt27dKTTz6ptLQ0+yx7vXv3dpg84sknn9TZs2c1ePBg7d27V6tWrdLLL7+sgQMHmnUKAAAAADyAqc84devWTcnJyRo3bpySkpLUsGFDrVmzxj5hxNGjRx26z0JCQvTll1/q2WefVf369VWhQgUNHjxYI0aMMOsUAAAAAHgA0yeHGDRokAYNGpTjuvXr1zuVNWvWTD/99NNNjgoAAAAA/g9PowEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4QOIEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgQiF3N2jUqNF112/duvWGgwEAAACAgsjtxGnHjh3y8/NTv3795O/vfzNiAgAAAIACxe3EaefOnRo2bJjmz5+v2NhYPfHEE/Ly8roZsQEAAABAgeD2M041a9bUZ599psWLF2vOnDmqW7euPv/885sRGwAAAAAUCDc8OcRdd92lLVu2aNSoUXrqqad09913a9u2bfkZGwAAAAAUCG4P1YuJiXEqa9++vT788EM1adJEly9fzpfAAAAAAKCgcDtxyq1XKSIi4m8HAwAAAAAFkduJ07p1625GHAAAAABQYOXrD+D+9ttv+bk7AAAAACgQ3E6c+vTpI5vN5lBms9n00ksvqXHjxvkWGAAAAAAUFG4nTtu2bdNDDz1knwTi119/VdOmTTVv3jx98cUX+R4gAAAAAJjN7cRp/fr1OnHihNq3b68XX3xRERERatasmbZv366WLVvejBgBAAAAwFRuJ04lS5bU2rVrZRiGYmNj9dFHH+mtt96Sn5/fzYgPAAAAAEznduKUkpKirKwsffjhh7r77rsVGxurI0eOKCUlRSkpKTcjRgAAAAAwldvTkZcoUUIWi0WSZBiGJKlq1aoyDEMWi0VZWVn5GyEAAAAAmIzfcQIAAAAAF9xOnFq1anUz4gAAAACAAsvtxOmXX3657vr69evfcDAAAAAAUBC5nTg1bNjQ6RmnbDzjBAAAAOBW5PasenfccYeKFi2qiRMn6uDBgzp06JD9dfDgwRsKYvr06QoNDZWvr6+aNm2qTZs25Vp33rx5slgsDi9fX98bOi4AAAAA5IXbPU7fffedli9frpEjR2rFihV644031KJFixsOYPHixYqJidHMmTPVtGlTTZ06VZGRkdqzZ4/Kli2b4zb+/v7as2ePfTm7BwwAAOBf5YUAsyMwX5VKZkcA5InbiZMkde3aVQ888ICmTZumTp06qWXLlpo8ebLCwsLc3teUKVPUv39/RUdHS5JmzpypVatWac6cORo5cmSO21gsFgUHB+dp/xkZGcrIyLAvZ//WlM1mk81mczte5A+rDNeVbnG2F0qaHYKprDSUfAYBoj2wuT/455Zj9fBrQFtgLneu/w0lTpJUqFAhDRkyRH369NHEiRPVqFEj9e3bV1OnTs3zPjIzM7VlyxaNGjXKXma1WtWmTRslJCTkul1qaqoqV64sm82mRo0a6eWXX9Ztt92WY924uDiNHz/eqTw5OVnp6el5jhX5q3ZJz24oJelUYc+eSKV6oUCzQzDdqVOnzA4BMJ2ntwee3hZItAe0Bea6ePFinuu6nTiVLFkyx6FxGRkZevvtt91KnE6fPq2srCwFBQU5lAcFBWn37t05blOzZk3NmTNH9evX14ULF/Taa6+pefPm+vXXX1WxYkWn+qNGjVJMTIx9OSUlRSEhIQoMDJS/v3+eY0X+2nWO4ZVlfa8/Q+Wtbl9pepxyG44MeBJPbw88vS2QaA9oC8zlzlwJbidO7iRGN0OzZs3UrFkz+3Lz5s1Vu3Ztvfvuu5o4caJTfR8fH/n4+DiVW61WWa2e3TVsJps8u6GUJKs8u2ve5uHnL4nPIEC0B57eFki0B7QF5nLn+rudOEVFRbm7Sa7KlCkjLy8vnTx50qH85MmTeX6GqXDhwrr99tu1f//+fIsLAAAAAK51QyluVlaWli5dqokTJ2rixIlatmyZrly54vZ+vL29FR4ervj4eHuZzWZTfHy8Q6+Sq1h27NihcuXKuX18AAAAAMgLt3ucfv31Vz3wwANKSkpSzZo1JUmTJk1SYGCgPv/8c9WtW9et/cXExCgqKkoRERFq0qSJpk6dqrS0NPsse71791aFChUUFxcnSZowYYL+85//KCwsTOfPn9err76qI0eOqF+/fu6eCgAAAADkiduJU79+/XTbbbdp8+bNKlny6nTK586dU58+ffT444/rxx9/dGt/3bp1U3JyssaNG6ekpCQ1bNhQa9assU8YcfToUYexh+fOnVP//v2VlJSkkiVLKjw8XD/++KPq1Knj7qkAAAAAQJ5YDMNwax7QIkWKaPPmzU7Tf+/cuVONGzfWn3/+ma8B5reUlBQFBATowoULzKpnotCRq8wOwXSHfXuaHYKp6vE7TtoRtcPsEADTeXp74OltgUR7QFtgLndyA7efcapRo4bTZA7S1Tnob+QHcAEAAACgoHM7cYqLi9MzzzyjpUuX6o8//tAff/yhpUuXasiQIZo0aZJSUlLsLwAAAAC4Fbj9jFPHjh0lSQ8//LD9h3CzR/vdf//99mWLxaKsrKz8ihMAAAAATON24rRu3bqbEQcAAAAAFFhuJ05VqlRRSEiIvbcJAAAAAG51bj/jVKVKFSUnJ9+MWAAAAACgQHI7cXJz9nIAAAAA+Ndze6ieJP3xxx9KT0/PcV2lSp49Fz8AAACAW88NJU6NGzd2KmMmPQAAAAC3qhtKnDZu3KjAwMD8jgUAAAAACiS3EyeLxaJKlSqpbNmyNyMeAAAAAChwmBwCAAAAAFxwO3E6dOgQw/QAAAAAeBS3E6fKlStrw4YN6tWrl5o1a6Zjx45JkubPn68NGzbke4AAAAAAYDa3E6dly5YpMjJSRYoU0bZt25SRkSFJunDhgl5++eV8DxAAAAAAzOZ24vTiiy9q5syZeu+991S4cGF7eYsWLbR169Z8DQ4AAAAACgK3E6c9e/aoZcuWTuUBAQE6f/58fsQEAAAAAAWK24lTcHCw9u/f71S+YcMGVa1aNV+CAgAAAICCxO3EqX///ho8eLA2btwoi8Wi48ePa+HChRo6dKiefPLJmxEjAAAAAJjK7R/AHTlypGw2m+655x5dunRJLVu2lI+Pj4YOHaqnn376ZsQIAAAAAKZyO3GyWCx6/vnnNWzYMO3fv1+pqamqU6eOihUrdjPiAwAAAADTuZ04ZfP29ladOnXyMxYAAAAAKJDcTpzuvvvu667/5ptvbjgYAAAAACiI3E6c1q9fr4oVK+qBBx5w+B0nAAAAALhVuZ04ffLJJ5o1a5aWLl2qRx99VP3791eNGjVuRmwAAAAAUCC4PR15p06dtGrVKv3888/y8/NTmzZtdNddd2nTpk03Iz4AAAAAMJ3biVO2kJAQDRs2TCNGjNDWrVuVkJCQn3EBAAAAQIFxQ4nTpk2b1K9fP1WpUkUJCQn6/PPPNXjw4PyODQAAAAAKBLefcWrYsKHOnj2rvn37atOmTSpdurQkKSUlRZLk7++fvxECAAAAgMncTpx++eUXSdKECRM0ceJEe7lhGLJYLMrKysq/6AAAAACgAHA7cVq3bt3NiAMAAAAACiy3E6dWrVrdjDgAAAAAoMC64Vn1AAAAAMBTkDgBAAAAgAskTgAAAADgAokTAAAAALhwQ4nTlStX9PXXX+vdd9/VxYsXJUnHjx9XamrqDQUxffp0hYaGytfXV02bNtWmTZvytN2iRYtksVjUuXPnGzouAAAAAOSF24nTkSNHVK9ePXXq1EkDBw5UcnKyJGnSpEkaOnSo2wEsXrxYMTExio2N1datW9WgQQNFRkbq1KlT193u8OHDGjp0qO688063jwkAAAAA7nB7OvLBgwcrIiJC27dvV+nSpe3lXbp0Uf/+/d0OYMqUKerfv7+io6MlSTNnztSqVas0Z84cjRw5MsdtsrKy9Mgjj2j8+PH6/vvvdf78+Vz3n5GRoYyMDPtySkqKJMlms8lms7kdL/KHVYbZIZjO5uEjZa0efv6S+AwCRHvg6W2BRHtAW2Aud66/24nT999/rx9//FHe3t4O5aGhoTp27Jhb+8rMzNSWLVs0atQoe5nValWbNm2UkJCQ63YTJkxQ2bJl9dhjj+n777+/7jHi4uI0fvx4p/Lk5GSlp6e7FS/yT+2Snt1QStKpwvXNDsFU1QsFmh2C6Vz1rAOewNPbA09vCyTaA9oCc2U/dpQXbidONptNWVlZTuV//PGHihcv7ta+Tp8+raysLAUFBTmUBwUFaffu3Tlus2HDBs2ePVuJiYl5OsaoUaMUExNjX05JSVFISIgCAwPl7+/vVrzIP7vOWcwOwXRlfX8xOwRT7StdyewQTFe2bFmzQwBM5+ntgae3BRLtAW2BuXx9ffNc1+3EqW3btpo6dapmzZolSbJYLEpNTVVsbKzat2/v7u7ccvHiRT366KN67733VKZMmTxt4+PjIx8fH6dyq9Uqq9Wzu4bNZJNnN5SSZJVnd83bPPz8JfEZBIj2wNPbAon2gLbAXO5cf7cTp9dff12RkZGqU6eO0tPT1bNnT+3bt09lypTRRx995Na+ypQpIy8vL508edKh/OTJkwoODnaqf+DAAR0+fFj333+/vSx7XGKhQoW0Z88eVatWzd1TAgAAAIDrcjtxqlixorZv365Fixbpl19+UWpqqh577DE98sgjKlKkiFv78vb2Vnh4uOLj4+1TittsNsXHx2vQoEFO9WvVqqUdO3Y4lI0ZM0YXL17Um2++qZCQEHdPBwAAAABccjtxkq727vTq1StfAoiJiVFUVJQiIiLUpEkTTZ06VWlpafZZ9nr37q0KFSooLi5Ovr6+qlu3rsP2JUqUkCSncgAAAADIL24nTp999tl11z/wwANu7a9bt25KTk7WuHHjlJSUpIYNG2rNmjX2CSOOHj3K2E8AAAAApnI7cercubMslqsPchqG4xSiFoslxxn3XBk0aFCOQ/Mkaf369dfddt68eW4fDwAAAADc4XZXziOPPKLixYtr4sSJ+vPPP+0/JJvbNOUAAAAA8G/nduI0f/58xcfH66uvvlKNGjW0cOHCmxEXAAAAABQYN/TwUHh4uNavX68333xTEyZMUEREhL777rv8jg0AAAAACgS3E6eUlBT76+6779YPP/ygTp06qWPHjvYpxQEAAADgVuL25BAlSpSwTw5xLcMw9Pnnn+dLUAAAAABQkLidOK1bt+5mxAEAAAAABZbbiVOrVq1uRhwAAAAAUGC5nTj98ssv111fv379Gw4GAAAAAAoitxOnhg0bymKxyDAMpx/CvdEfwAUAAACAgsztxOnQoUOSriZLdevW1erVq1W5cuV8DwwAAAAACgq3E6drkySLxaKKFSuSOAEAAAC4pd3QD+ACAAAAgCf5W4mTxWLJ8TedAAAAAOBW4vZQvZIlS9qTpdTUVN1+++2yWv8v/zp79mz+RQcAAAAABYDbidPUqVNvQhgAAAAAUHC5nThFRUXdjDgAAAAAoMC6oWecDhw4oDFjxqhHjx46deqUJOmLL77Qr7/+mq/BAQAAAEBB4Hbi9O2336pevXrauHGjli9frtTUVEnS9u3bFRsbm+8BAgAAAIDZ3E6cRo4cqRdffFFr166Vt7e3vfzuu+/WTz/9lK/BAQAAAEBB4HbitGPHDnXp0sWpvGzZsjp9+nS+BAUAAAAABYnbiVOJEiV04sQJp/Jt27apQoUK+RIUAAAAABQkbidO3bt314gRI5SUlCSLxSKbzaYffvhBQ4cOVe/evW9GjAAAAABgKrcTp5dfflm1atVSSEiIUlNTVadOHbVs2VLNmzfXmDFjbkaMAAAAAGAqt3/HydvbW++9957Gjh2rnTt3KjU1VbfffruqV69+M+IDAAAAANO5nThlq1SpkipVqpSfsQAAAABAgeR24hQTE3Pd9VOmTLnhYAAAAACgIHI7cdq2bZvD8oYNGxQeHq4iRYrIYrHkW2AAAAAAUFC4nTitW7fOYbl48eL68MMPVbVq1XwLCgAAAAAKErdn1fsrwzDyIw4AAAAAKLD+VuK0fPlypaenq2zZsvkVDwAAAAAUOG4P1StZsqQsFovS09OVkZGhESNGqFixYjcjNgAAAAAoENxOnKZOnSpJKlKkiG677Tbddttt+R0TAAAAABQobidOUVFRNyMOAAAAACiwbvgHcH/77TcdPXpUmZmZDuUPPPDA3w4KAAAAAAoStxOngwcPqkuXLtqxY4csFot9Vr3s33DKysrK3wgBAAAAwGRuz6o3ePBgValSRadOnZKfn59+/fVXfffdd4qIiND69etvQogAAAAAYC63E6eEhARNmDBBZcqUkdVqldVq1R133KG4uDg988wzNxTE9OnTFRoaKl9fXzVt2lSbNm3Kte7y5csVERGhEiVKqGjRomrYsKHmz59/Q8cFAAAAgLxwO3HKyspS8eLFJUllypTR8ePHJUmVK1fWnj173A5g8eLFiomJUWxsrLZu3aoGDRooMjJSp06dyrF+qVKl9PzzzyshIUG//PKLoqOjFR0drS+//NLtYwMAAABAXrj9jFPdunW1fft2ValSRU2bNtXkyZPl7e2tWbNmqWrVqm4HMGXKFPXv31/R0dGSpJkzZ2rVqlWaM2eORo4c6VS/devWDsuDBw/WBx98oA0bNigyMtKpfkZGhjIyMuzLKSkpkiSbzSabzeZ2vMgfVhlmh2A629/7/el/PauHn78kPoMA0R54elsg0R7QFpjLnevvduI0ZswYpaWlSZImTJigjh076s4771Tp0qW1ePFit/aVmZmpLVu2aNSoUfYyq9WqNm3aKCEhweX2hmHom2++0Z49ezRp0qQc68TFxWn8+PFO5cnJyUpPT3crXuSf2iU9u6GUpFOF65sdgqmqFwo0OwTT5dazDngST28PPL0tkGgPaAvMdfHixTzXdTtxurZXJywsTLt379bZs2dVsmRJ+8x6eXX69GllZWUpKCjIoTwoKEi7d+/OdbsLFy6oQoUKysjIkJeXl9555x3de++9OdYdNWqUYmJi7MspKSkKCQlRYGCg/P393YoX+WfXOffulVtRWd9fzA7BVPtKVzI7BNOVLVvW7BAA03l6e+DpbYFEe0BbYC5fX988173h33G6VqlSpfJjN3lWvHhxJSYmKjU1VfHx8YqJiVHVqlWdhvFJko+Pj3x8fJzKsye2gDls8uyGUpKs8uyueZuHn78kPoMA0R54elsg0R7QFpjLnevvduLUtWvX665fvnx5nvdVpkwZeXl56eTJkw7lJ0+eVHBwcK7bWa1WhYWFSZIaNmyoXbt2KS4uLsfECQAAAAD+LrdT3ICAAPtr1apVslqtDmXu8Pb2Vnh4uOLj4+1lNptN8fHxatasWZ73Y7PZHCaAAAAAAID85HaP09y5c+3/Xrp0qSZPnnxDs+lli4mJUVRUlCIiItSkSRNNnTpVaWlp9ln2evfurQoVKiguLk7S1ckeIiIiVK1aNWVkZGj16tWaP3++ZsyYccMxAAAAAMD15MszTn9Ht27dlJycrHHjxikpKUkNGzbUmjVr7BNGHD161GHsYVpamp566in98ccfKlKkiGrVqqUFCxaoW7duZp0CAAAAgFuc6YmTJA0aNEiDBg3Kcd369esdll988UW9+OKL/0BUAAAAAHCV24nTW2+9Zf/3lStXNG/ePJUpU8Ze9swzz+RPZAAAAABQQLidOL3xxhv2fwcHB2v+/Pn2ZYvFQuIEAAAA4JbjduJ06NChmxEHAAAAABRY/OIWAAAAALjgVuI0a9Ys9erVSwsXLrQv16hRQ2FhYXr99ddvSoAAAAAAYLY8D9VbuHChnnvuObVt21bDhg3T/v37NXXqVA0dOlQ2m00TJkxQlSpV1LVr15sZLwAAAAD84/KcOL3zzjuaMWOGevXqpS1btqhp06aaMWOG+vfvL0kqX7683n77bRInAAAAALecPA/V27Vrl5o1ayZJCg8Pl9VqVdOmTe3rW7ZsqR07duR/hAAAAABgsjwnThkZGfLz87Mv+/j4qFixYvblIkWKKCsrK3+jAwAAAIACIM+JU4UKFbR//3778oIFC1SuXDn78p49exQaGpqvwQEAAABAQZDnxKlVq1ZavXq1fblTp04qUqSIfXnWrFlq3rx5/kYHAAAAAAVAnieHeO+99667/v3335evr+/fDggAAAAACpo8J06uFC9ePL92BQAAAAAFils/gAsAAAAAnojECQAAAABcIHECAAAAABdInAAAAADABRInAAAAAHCBxAkAAAAAXCBxAgAAAAAXSJwAAAAAwAUSJwAAAABwgcQJAAAAAFwgcQIAAAAAF0icAAAAAMAFEicAAAAAcIHECQAAAABcIHECAAAAABdInAAAAADABRInAAAAAHCBxAkAAAAAXCBxAgAAAAAXSJwAAAAAwAUSJwAAAABwgcQJAAAAAFwgcQIAAAAAFwpE4jR9+nSFhobK19dXTZs21aZNm3Kt+9577+nOO+9UyZIlVbJkSbVp0+a69QEAAADg7zI9cVq8eLFiYmIUGxurrVu3qkGDBoqMjNSpU6dyrL9+/Xr16NFD69atU0JCgkJCQtS2bVsdO3bsH44cAAAAgKcoZHYAU6ZMUf/+/RUdHS1JmjlzplatWqU5c+Zo5MiRTvUXLlzosPz+++9r2bJlio+PV+/evZ3qZ2RkKCMjw76ckpIiSbLZbLLZbPl5KnCDVYbZIZjOZv73Fqayevj5S+IzCBDtgae3BRLtAW2Budy5/qYmTpmZmdqyZYtGjRplL7NarWrTpo0SEhLytI9Lly7p8uXLKlWqVI7r4+LiNH78eKfy5ORkpaen31jg+Ntql/TshlKSThWub3YIpqpeKNDsEEyXW8864Ek8vT3w9LZAoj2gLTDXxYsX81zX1MTp9OnTysrKUlBQkEN5UFCQdu/enad9jBgxQuXLl1ebNm1yXD9q1CjFxMTYl1NSUhQSEqLAwED5+/vfePD4W3ads5gdgunK+v5idgim2le6ktkhmK5s2bJmhwCYztPbA09vCyTaA9oCc/n6+ua5rulD9f6OV155RYsWLdL69etzPWkfHx/5+Pg4lVutVlmtnt01bCabPLuhlCSrPLtr3ubh5y+JzyBAtAee3hZItAe0BeZy5/qbmjiVKVNGXl5eOnnypEP5yZMnFRwcfN1tX3vtNb3yyiv6+uuvVb8+3dwAAAAAbh5TU1xvb2+Fh4crPj7eXmaz2RQfH69mzZrlut3kyZM1ceJErVmzRhEREf9EqAAAAAA8mOlD9WJiYhQVFaWIiAg1adJEU6dOVVpamn2Wvd69e6tChQqKi4uTJE2aNEnjxo3Thx9+qNDQUCUlJUmSihUrpmLFipl2HgAAAABuXaYnTt26dVNycrLGjRunpKQkNWzYUGvWrLFPGHH06FGHsYczZsxQZmam/vvf/zrsJzY2Vi+88MI/GToAAAAAD2F64iRJgwYN0qBBg3Jct379eoflw4cP3/yAAAAAAOAaTOMBAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4UMjsAAAABVdWVpYuX75sdhi4AV5eXipUqJAsFovZoQDALYHECQCQo9TUVP3xxx8yDMPsUHCD/Pz8VK5cOXl7e5sdCgD865E4AQCcZGVl6Y8//pCfn58CAwPptfiXMQxDmZmZSk5O1qFDh1S9enVZrYzOB4C/g8QJAODk8uXLMgxDgYGBKlKkiNnh4AYUKVJEhQsX1pEjR5SZmSlfX1+zQwKAfzW+fgIA5Iqepn83epkAIP/wiQoAAAAALpA4AQAAAIALJE4AAAAA4AKTQwAA8ix05Kp/9HiHX+lwQ9slJCTojjvuULt27bRq1T8bMwDg1kSPEwDgljN79mw9/fTT+u6773T8+HHT4sjMzDTt2ACA/EXiBAC4paSmpmrx4sV68skn1aFDB82bN89h/eeff67GjRvL19dXZcqUUZcuXezrMjIyNGLECIWEhMjHx0dhYWGaPXu2JGnevHkqUaKEw75WrFjhMPPgCy+8oIYNG+r9999XlSpV7FOAr1mzRnfccYdKlCih0qVLq2PHjjpw4IDDvv744w/16NFDpUqVUtGiRRUREaGNGzfq8OHDslqt2rx5s0P9qVOnqnLlyrLZbH/3kgEA8oDECQBwS/n4449Vq1Yt1axZU7169dKcOXNkGIYkadWqVerSpYvat2+vbdu2KT4+Xk2aNLFv27t3b3300Ud66623tGvXLr377rsqVqyYW8ffv3+/li1bpuXLlysxMVGSlJaWppiYGG3evFnx8fGyWq3q0qWLPelJTU1Vq1atdOzYMX322Wfavn27hg8fLpvNptDQULVp00Zz5851OM7cuXPVp08fphwHgH8IzzgBAG4ps2fPVq9evSRJ7dq104ULF/Ttt9+qdevWeumll9S9e3eNHz/eXr9BgwaSpL179+rjjz/W2rVr1aZNG0lS1apV3T5+Zmam/ve//ykwMNBe9uCDDzrUmTNnjgIDA/Xbb7+pbt26+vDDD5WcnKyff/5ZpUqVkiSFhYXZ6/fr109PPPGEpkyZIh8fH23dulU7duzQp59+6nZ8AIAbw9dUAIBbxp49e7Rp0yb16NFDklSoUCF169bNPtwuMTFR99xzT47bJiYmysvLS61atfpbMVSuXNkhaZKkffv2qUePHqpatar8/f0VGhoqSTp69Kj92Lfffrs9afqrzp07y8vLS5988omkq8MG77rrLvt+AAA3Hz1OAIBbxuzZs3XlyhWVL1/eXmYYhnx8fDRt2jQVKVIk122vt06SrFarfchftsuXLzvVK1q0qFPZ/fffr8qVK+u9995T+fLlZbPZVLduXfvkEa6O7e3trd69e2vu3Lnq2rWrPvzwQ7355pvX3QYAkL/ocQIA3BKuXLmi//3vf3r99deVmJhof23fvl3ly5fXRx99pPr16ys+Pj7H7evVqyebzaZvv/02x/WBgYG6ePGi0tLS7GXZzzBdz5kzZ7Rnzx6NGTNG99xzj2rXrq1z58451Klfv74SExN19uzZXPfTr18/ff3113rnnXd05coVde3a1eWxAQD5hx4nAMAtYeXKlTp37pwee+wxBQQEOKx78MEHNXv2bL366qu65557VK1aNXXv3l1XrlzR6tWrNWLECIWGhioqKkp9+/bVW2+9pQYNGujIkSM6deqUHn74YTVt2lR+fn4aPXq0nnnmGW3cuNFpxr6clCxZUqVLl9asWbNUrlw5HT16VCNHjnSo06NHD7388svq3Lmz4uLiVK5cOW3btk3ly5dXs2bNJEm1a9fWf/7zH40YMUJ9+/Z12UsFAMhfJE4AgDy70R+k/SfMnj1bbdq0cUqapKuJ0+TJk1WqVCktWbJEEydO1CuvvCJ/f3+1bNnSXm/GjBkaPXq0nnrqKZ05c0aVKlXS6NGjJUmlSpXSggULNGzYML333nu655579MILL+jxxx+/blxWq1WLFi3SM888o7p166pmzZp666231Lp1a3sdb29vffXVV3ruuefUvn17XblyRXXq1NH06dMd9vXYY4/pxx9/VN++ff/GlQIA3AiL8dcB27e4lJQUBQQE6MKFC/L39zc7HI8VOnKV2SGY7rBvT7NDMFW9KpXMDsF0O6J2mB1CrtLT03Xo0CGH3yKC+SZOnKglS5bol19+yVP9f8Pf0dPbA09vCyTag4LcFngCd3IDnnECAKCAS01N1c6dOzVt2jQ9/fTTZocDAB6JxAkAgAJu0KBBCg8PV+vWrRmmBwAm4RknAAAKuHnz5uVpIgoAwM1DjxMAAAAAuEDiBAAAAAAumJ44TZ8+XaGhofL19VXTpk21adOmXOv++uuvevDBBxUaGiqLxaKpU6f+c4ECAAAA8FimJk6LFy9WTEyMYmNjtXXrVjVo0ECRkZE6depUjvUvXbqkqlWr6pVXXlFwcPA/HC0AAAAAT2Vq4jRlyhT1799f0dHRqlOnjmbOnCk/Pz/NmTMnx/qNGzfWq6++qu7du8vHx+cfjhYAAACApzJtVr3MzExt2bJFo0aNspdZrVa1adNGCQkJ+XacjIwMZWRk2JdTUlIkSTabTTabLd+OA/dY5VG/u5wjm/kjZU1l9fDzl1SgP4NsNpsMw7C/8O+U/fcryG2ep7cHnt4WSLQHBfW96Sncuf6mJU6nT59WVlaWgoKCHMqDgoK0e/fufDtOXFycxo8f71SenJys9PT0fDsO3FO7pGc3lJJ0qnB9s0MwVfVCgWaHYLrchiUXBJcvX5bNZtOVK1d05coVe3nhl8r8s3E8f/ofPd6t5sqVK7LZbDpz5owKFy5sdjg58vT2wNPbAon2oCC3BZ7g4sWLea57y/+O06hRoxQTE2NfTklJUUhIiAIDA+Xv729iZJ5t1zmL2SGYrqzvL2aHYKp9pSuZHYLpypYta3YIuUpPT9fFixdVqFAhFSpkXlPh7rGjo6P1wQcfOJXv3btXx48f12uvvaYtW7boxIkTWr58uTp37pxPkRZMhQoVktVqVenSpeXr62t2ODny9PbA09sCifagILcFnsCdz0bTWsMyZcrIy8tLJ0+edCg/efJkvk784OPjk+PzUFarVVarZ3cNm8kmz24oJckqz+6at3n4+Usq0J9BVqtVFovF/jLLjRy7Xbt2mjt3rkNZYGCg9u/frwYNGqhv377q2rWr6ef2T8g+x4Lc5nl6e+DpbYFEe1BQ35uewp3rb1ri5O3trfDwcMXHx9u/8bPZbIqPj9egQYPMCgsA8C/n4+OT4xdw9913n+677z4TIgIA3ApMHaoXExOjqKgoRUREqEmTJpo6darS0tIUHR0tSerdu7cqVKiguLg4SVcnlPjtt9/s/z527JgSExNVrFgxhYWFmXYeAAAAAG5tpiZO3bp1U3JyssaNG6ekpCQ1bNhQa9assU8YcfToUYfus+PHj+v222+3L7/22mt67bXX1KpVK61fv/6fDh8AUACtXLlSxYoVsy/fd999WrJkiYkRAQBuBaZPDjFo0KBch+b9NRkKDQ1lWlwAwHXdddddmjFjhn25aNGiJkYDALhVmJ44AQCQn4oWLcrwbQBAvmMaDwAAAABwgR4nAIBHSE1N1f79++3Lhw4dUmJiokqVKqVKlTz7d2QAAK6ROAEA8u6FC2ZHcMM2b96su+66y76c/ePoUVFRmjdvnklRAQD+LUicAAC3jOslQK1bt2aCIQDADeMZJwAAAABwgcQJAAAAAFwgcQIAAAAAF0icAAAAAMAFEicAAAAAcIHECQAAAABcIHECAAAAABdInAAAAADABRInAAAAAHChkNkBAAD+Pep9UO8fPd6OqB3/6PEAAMgNPU4AgFtGnz59ZLFYZLFYVLhwYVWpUkXDhw9Xenq6Q72VK1eqVatWKl68uPz8/NS4cWPNmzcvx30uW7ZMrVu3VkBAgIoVK6b69etrwoQJOnv2rMt4PvroI3l5eWngwIFO6+bNm6cSJUrkuJ3FYtGKFSvyLQ4AwN9H4gQAuKW0a9dOJ06c0MGDB/XGG2/o3XffVWxsrH3922+/rU6dOqlFixbauHGjfvnlF3Xv3l1PPPGEhg4d6rCv559/Xt26dVPjxo31xRdfaOfOnXr99de1fft2zZ8/32Uss2fP1vDhw/XRRx85JW/u+LtxAAD+PobqAQBuKT4+PgoODpYkhYSEqE2bNlq7dq0mTZqk33//Xc8995yGDBmil19+2b7Nc889J29vbz3zzDN66KGH1LRpU23atEkvv/yypk6dqsGDB9vrhoaG6t5779X58+evG8ehQ4f0448/atmyZVq3bp2WL1+unj17un0+fzcOAED+oMcJAHDL2rlzp3788Ud5e3tLkpYuXarLly879SxJ0oABA1SsWDF99NFHkqSFCxeqWLFieuqpp3Lcd27D7LLNnTtXHTp0UEBAgHr16qXZs2ff0Dn83TgAAPmDxAkAcEtZuXKlihUrJl9fX9WrV0+nTp3SsGHDJEl79+5VQECAypUr57Sdt7e3qlatqr1790qS9u3bp6pVq6pw4cJux2Cz2TRv3jz16tVLktS9e3dt2LBBhw4dcntffycOAED+IXECANxS7rrrLiUmJmrjxo2KiopSdHS0HnzwQbf3YxiGyzpHjx5VsWLF7K/s4X9r165VWlqa2rdvL0kqU6aM7r33Xs2ZM+emxAEAuPl4xgkAcEspWrSowsLCJElz5sxRgwYNNHv2bD322GOqUaOGLly4oOPHj6t8+fIO22VmZurAgQO66667JEk1atTQhg0bdPny5Vx7e8qXL6/ExET7cqlSpSRdnRTi7NmzKlKkiH2dzWbTL7/8ovHjx8tqtcrf319paWmy2WyyWv/ve8zsZ5YCAgLyHAcA4OajxwkAcMuyWq0aPXq0xowZoz///FMPPvigChcurNdff92p7syZM5WWlqYePXpIknr27KnU1FS98847Oe77/PnzKlSokMLCwuyvUqVK6cyZM/r000+1aNEiJSYm2l/btm3TuXPn9NVXX0mSatasqStXrjgkXpK0detWSVcTprzGAQC4+ehxAgDc0h566CENGzZM06dP19ChQzV58mQ999xz8vX11aOPPqrChQvr008/1ejRo/Xcc8+padOmkqSmTZtq+PDheu6553Ts2DF16dJF5cuX1/79+zVz5kzdcccdDrPcZZs/f75Kly6thx9+WBaLxWFd+/btNXv2bLVr10633Xab2rZtq759++r1119X1apVtWfPHg0ZMkTdunVThQoV/lYcAID8ReIEAMizHVE7zA7BbYUKFdKgQYM0efJkPfnkkxoyZIiqVq2q1157TW+++aaysrJ02223acaMGYqOjnbYdtKkSQoPD9f06dM1c+ZM2Ww2VatWTf/9738VFRWV4/HmzJmjLl26OCVNkvTggw/q0Ucf1enTp1WmTBktXrxYsbGxGjBggI4fP66KFSuqS5cuGjt27N+OAwCQvyyGhz11mpKSooCAAF24cEH+/v5mh+OxQkeuMjsE0x32df/3XG4l9apUMjsE0xXkJCQ9PV2HDh1SlSpV5Ovra3Y4uEH/hr+jp7cHnt4WSLQHBbkt8ATu5AY84wQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC4QOIEAMiVh80fdMvh7wcA+YfECQDgxMvLS5KUmZlpciT4Oy5duiRJKly4sMmRAMC/H7/jBABwUqhQIfn5+Sk5OVmFCxeW1cr3bP8mhmHo0qVLOnXqlEqUKGFPhAEAN47ECQDgxGKxqFy5cjp06JCOHDlidji4QSVKlFBwcLDZYQDALYHECQCQI29vb1WvXp3hev9ShQsXpqcJAPIRiRMAIFdWq1W+vr5mhwEAgOkKxKD16dOnKzQ0VL6+vmratKk2bdp03fpLlixRrVq15Ovrq3r16mn16tX/UKQAAAAAPJHpidPixYsVExOj2NhYbd26VQ0aNFBkZKROnTqVY/0ff/xRPXr00GOPPaZt27apc+fO6ty5s3bu3PkPRw4AAADAU5ieOE2ZMkX9+/dXdHS06tSpo5kzZ8rPz09z5szJsf6bb76pdu3aadiwYapdu7YmTpyoRo0aadq0af9w5AAAAAA8hanPOGVmZmrLli0aNWqUvcxqtapNmzZKSEjIcZuEhATFxMQ4lEVGRmrFihU51s/IyFBGRoZ9+cKFC5Kk8+fPy2az/c0zwA3LSDM7AtOdt1jMDsFUxp/8MOf58+fNDgEwn4e3B57eFki0B7QF5kpJSZGUtx8MNzVxOn36tLKyshQUFORQHhQUpN27d+e4TVJSUo71k5KScqwfFxen8ePHO5VXrlz5BqMG8kdJswMw3QWzAzBdySe5CwBPx6eA5OntAW1BwXDx4kUFBARct84tP6veqFGjHHqobDabzp49q9KlS8vCtzzwUCkpKQoJCdHvv/8uf39/s8MBAJiE9gCezjAMXbx4UeXLl3dZ19TEqUyZMvLy8tLJkycdyk+ePJnrD/YFBwe7Vd/Hx0c+Pj4OZSVKlLjxoIFbiL+/Pw0lAID2AB7NVU9TNlMnh/D29lZ4eLji4+PtZTabTfHx8WrWrFmO2zRr1syhviStXbs21/oAAAAA8HeZPlQvJiZGUVFRioiIUJMmTTR16lSlpaUpOjpaktS7d29VqFBBcXFxkqTBgwerVatWev3119WhQwctWrRImzdv1qxZs8w8DQAAAAC3MNMTp27duik5OVnjxo1TUlKSGjZsqDVr1tgngDh69Kis1v/rGGvevLk+/PBDjRkzRqNHj1b16tW1YsUK1a1b16xTAP51fHx8FBsb6zSMFQDgWWgPgLyzGHmZew8AAAAAPJjpP4ALAAAAAAUdiRMAAAAAuEDiBAAAcIMuX75sdggA/iEkTgAAAHlw5coVTZkyRS1atFCFChXk6+ursWPHmh0WgH8IiRNwA/r06SOLxWJ/lS5dWu3atdMvv/xidmgA4FF+//139e3bV+XLl5e3t7cqV66swYMH68yZM/l6HMMwdP/992vevHkaOnSo1q1bp507dyo2NjZfjwOg4CJxAm5Qu3btdOLECZ04cULx8fEqVKiQOnbsaHZYAOAxDh48qIiICO3bt08fffSR9u/fr5kzZyo+Pl7NmjXT2bNn8+1YCxYs0OHDh/Xjjz+qS5cuqlGjhsLCwlSkSJF8OwaAgo3ECbhBPj4+Cg4OVnBwsBo2bKiRI0fq999/V3Jysr3OH3/8oR49eqhUqVIqWrSoIiIitHHjRvv6Tz/9VI0aNZKvr6+qVq2q8ePH68qVK7ke8689XdmvEiVK2Ou88MILatiwod59912FhITIz89PDz/8sC5cuOCwn86dO9uXz5w5o5IlSzrs580331SlSpXk4+OjoKAg9evXT5cuXZIkHT58WBaLRYmJiQ7xhYaGaurUqfblKVOmqF69eipatKhCQkL01FNPKTU11b5+3rx5Dsc8cuSIQkJCNGbMmFz3GR8fL4vF4hA/AM80cOBAeXt766uvvlKrVq1UqVIl3Xffffr666917NgxPf/885Kk1q1b5/jZabFY9MILL0iSzp07p969e6tkyZLy8/PTfffdp3379tmPtXLlStWpU0cdOnRQ8eLFFRQUpGeffVaZmZn2Oq1bt9agQYM0aNAgBQQEqEyZMho7dqyu/eWX+fPnKyIiQsWLF1dwcLB69uypU6dO2devX79eFotF58+fdzhXi8WiFStWSMr7Z/C12/xVw4YN7ecuSefPn1e/fv0UGBgof39/3X333dq+fXuu1z47hpxef41hxowZuu+++1SkSBFVrVpVS5cuddrPtecyduxYh/2cP39eTZo0UUBAgIoUKaJGjRrpiy++sNf/a5smObcvBw4cUKdOnRQUFKRixYqpcePG+vrrr697/caMGaOKFSvq8OHDOe5Tklq2bJnj3wK3JhInIB+kpqZqwYIFCgsLU+nSpe1lrVq10rFjx/TZZ59p+/btGj58uGw2myTp+++/V+/evTV48GD99ttvevfddzVv3jy99NJL1z3WtT1dJ06ccPiQz7Z//359/PHH+vzzz7VmzRpt27ZNTz31VK77zClha9KkiZYsWaJ9+/Zp6dKlio+P12uvvebWdbFarXrrrbf066+/6oMPPtA333yj4cOH51g3KSlJbdq0UadOnfTiiy/mWMdms+m5555TsWLF3IoDwK3n7Nmz+vLLL/XUU0859foEBwfrkUce0eLFi2UYhpYvX27/zGzWrJmee+45+/LQoUMlXf3P9+bNm/XZZ58pISFBhmGoffv29skfkpOTtXz5ct12223atGmT5syZo0WLFmnUqFEOx/7ggw9UqFAhbdq0SW+++aamTJmi999/377+8uXLmjhxorZv364VK1bo8OHD6tOnz829WHnw0EMP6dSpU/riiy+0ZcsWNWrUSPfcc4/LXruvv/7aoU2qWLGiU52xY8fqwQcf1Pbt2/XII4+oe/fu2rVrV477++OPPzR16lSHv6m3t7dGjx6tn3/+Wb/++qvatm2rBx98UBkZGXk+v9TUVLVv317x8fHatm2b2rVrp/vvv19Hjx7Nsf7rr7+ud999V2vXrlVoaGiOdZYvX65t27blOQb8+xUyOwDg32rlypX2/8CnpaWpXLlyWrlypazWq99HfPjhh0pOTtbPP/+sUqVKSZLCwsLs248fP14jR45UVFSUJKlq1aqaOHGihg8fft0x89k9XdkCAgKc6qSnp+t///ufKlSoIEl6++231aFDB73++usO20rS3r17NWfOHMXExOitt96ylzdr1sz+b19fX/n7+ysrKytvF+f/GzJkiP3foaGhevHFF/XEE0/onXfecah37tw5tW3bVk2bNtXbb7+d6/4++OADZWRkqFOnTg49VwA8z759+2QYhmrXrp3j+tq1a+vcuXNKTk5W2bJl7eXe3t4qVqyYw2fhvn379Nlnn+mHH35Q8+bNJUkLFy5USEiIVqxYoYceekg2m001a9bU9OnTZbFYVLt2bb366qt67LHHNHHiRPn5+UmSQkJC9MYbb8hisahmzZrasWOH3njjDfXv31+S1LdvX/txq1atqrfeekuNGzdWamqqaV8KbdiwQZs2bdKpU6fk4+MjSXrttde0YsUKLV26VI8//niu25YuXdrhWnp5eTnVeeihh9SvXz9J0sSJE7V27Vq9/fbbTm2BJD3//PPq1q2bQ2+Qn5+fvUfJMAxVq1ZNFotFly9ftsfrSoMGDdSgQQP78sSJE/XJJ5/os88+06BBgxzqvv/++5owYYK++eabXO+vy5cva8SIERoxYgQThHgQepyAG3TXXXcpMTFRiYmJ2rRpkyIjI3XffffpyJEjkqTExETdfvvt9qTpr7Zv364JEyaoWLFi9lf//v114sQJ+5C4G1WpUiV70iRdTYJsNpv27NnjVHf48OEaMGCAqlat6rRu4cKFKlq0qIKCglS9enWNGDHCYX3z5s0d4v/rN3dff/217rnnHlWoUEHFixfXo48+qjNnzjic35UrV9S+fXvt2LFDbdu2lcViyfGcLl26pDFjxmjy5MkqVIjvfABcde0wuBu1a9cuFSpUSE2bNrWXlS5dWjVr1nToGWnWrJnDZ9Qdd9yhzMxM7d+/3172n//8x6FOs2bNtG/fPvsXT1u2bNH999+vSpUqqXjx4mrVqpUkOX1+VqxY0eHzNSeuPoMlqUePHipWrJjKlSunDh066LfffnOqs337dqWmpqp06dIO+zt06JAOHDhw3WuXF9d+EZe9nFOP09atW/XJJ59o4sSJOe7ntttuk4+Pj0aMGKFly5Y5XJfsLzOzX0888YTDtqmpqRo6dKhq166tEiVKqFixYtq1a5fTNfv00081YMAAlS9fXnXr1s31nKZPn66AgAA98sgjLs8ftw4SJ+AGFS1aVGFhYQoLC1Pjxo31/vvvKy0tTe+9954kuXxgODU1VePHj7cnX4mJidqxY4f27dsnX1/ff+IU9O233+r77793eKboWg888IC2bdumTz/9VBs3btQnn3zisH7x4sUO8ZcvX96+7vDhw+rYsaPq16+vZcuWacuWLZo+fbokOTwTkJaWpiJFiujdd9/VkCFDlJSUlGMsr776qmrWrKn777//7542gFtAWFiYLBZLrkO+du3apZIlSyowMDBfjleyZMlc1+X2hc9fpaWlKTIyUv7+/lq4cKF+/vln++fqtZ+L0tXh3Nd+vubkep/B2d544w0lJibq888/1+XLl/Xwww871UlNTVW5cuUc9pWYmKg9e/Zo2LBheTq3/PDcc89p6NChKleuXI7rV69erU2bNunhhx/W8OHDHYbqXftlZmJioiZMmOCw7dChQ/XJJ5/o5Zdftl/bevXqOV33H374QYsXL3Z4/u2vzp07p4kTJ2rKlCl5/tvj1sDXtkA+sVgsslqt+vPPPyVJ9evX1/vvv6+zZ8/m2OvUqFEj7dmzx2H4Xn45evSojh8/bm9Ef/rpJ1mtVtWsWdNexzAMPffccxo7dmyu/yEoXry4ihcvrho1amjdunX66KOPHL5dCwkJcYj/2p6gLVu2yGaz6fXXX7cPX/z444+djuHn56fPPvtMxYoV0+eff64BAwbo008/dahz4sQJzZgxQ99+++0NXA0At6LSpUvr3nvv1TvvvKNnn33W4cuqpKQkLVy4UL17987Tf2xr166tK1euaOPGjfahemfOnNGePXtUp04dSVKtWrX0ySefyDAM+z43bNggb29vVatWzb6vaycAkq5+/lavXl1eXl7avXu3zpw5o1deeUUhISGSpM2bN+cYU5UqVZwmIvir630GZwsODrbXGTx4sO6//36nH+1t1KiRkpKSVKhQoVyf5/k7fvrpJ/Xu3dth+fbbb3eo89lnn2nv3r1atWpVrvupXLmyKleurEmTJqlEiRLasWOHIiIiJP3fl5nZrh2eKV1NiPr06aMuXbpIuposZk/6cK2RI0fqv//9rypVqqSWLVuqa9euaty4sUOdiRMn6s4771TLli1z3AduXfQ4ATcoIyNDSUlJSkpK0q5du/T0008rNTXV3iPSo0cPBQcHq3Pnzvrhhx908OBBLVu2TAkJCZKkcePG6X//+5/Gjx+vX3/9Vbt27dKiRYty7f1xh6+vr6KiorR9+3Z9//33euaZZ/Twww87jEOPj4/XhQsXNHDgwBz3MXfuXG3fvl1HjhzRZ599po8++sipobuesLAwXb58WW+//bYOHjyo+fPna+bMmU71ChcubB9uMWvWLH3//fdasGCBQ53p06erS5cubh0fwK1v2rRpysjIUGRkpL777jv9/vvvWrNmje69915VqFDB5WQ72apXr65OnTqpf//+2rBhg7Zv365evXqpQoUK6tSpkyTpySef1OHDhzVw4EDt2rVLq1ev1rBhwzRo0CD7803S1S+uYmJitGfPHn300Ud6++23NXjwYElXh1F7e3vbPxc/++yzXIel5ZfLly8rPT1dSUlJWrBggWrUqKHChQs71GnTpo2aNWumzp0766uvvrJPu/7888/nmti5Y8mSJZozZ4727t2r2NhYbdq0yem5osmTJ+vFF190uJbZtm3bppUrV+rgwYP69ddfNXToUBUrVkzVq1fPcwzVq1fX8uXLlZiYqO3bt6tnz572yZqulf1FZ5MmTTRkyBBFR0c79EpdunRJs2bN0uTJk/N8bNw6SJyAG7RmzRqVK1dO5cqVU9OmTfXzzz9ryZIlat26tSTZp8gtW7as2rdvr3r16umVV16xPzgbGRmplStX6quvvlLjxo31n//8R2+88YYqV678t2MLCwtT165d1b59e7Vt21b169d3egg3LS1Nr7zyilMDmi0hIUHt2rVTjRo19PTTT+uRRx5x6wHYBg0aaMqUKZo0aZLq1q2rhQsXKi4u7rrblCtXTm+++aYGDx7sMGTPZrPl+T9AADxH9erVtXnzZlWtWlUPP/ywqlWrpscff1x33XWXEhIScn3GNCdz585VeHi4OnbsqGbNmskwDK1evdr+GVmpUiWtXLlSmzZtUoMGDRQdHa0ePXo4fa717t1bf/75p5o0aaKBAwdq8ODB9skVAgMDNW/ePC1ZskR16tTRK6+84vZspe56+OGHVaRIEdWoUUMnTpzQ4sWLnepYLBatXr1aLVu2VHR0tGrUqKHu3bvryJEjCgoK+tsxjB8/XosWLVL9+vX1v//9Tx999JG9Jy9bWFiYfbKkv/rzzz81duxY1atXTy1atLD3TOU0OVJupkyZopIlS6p58+a6//77FRkZqUaNGrmM22azOQzZu3z5sv0awfNYjPx4qhJAgfHCCy9oxYoV/KYEAPzDWrdurYYNG+b4MxGeymKx6JNPPuG393BLoMcJAAAAAFwgcQIAAAAAFxiqBwAAAAAu0OMEAAAAAC6QOAEAAACACyROAAAAAOACiRMAAAAAuEDiBAAAAAAukDgBAAAAgAskTgAAAADgAokTAKBA6dOnjywWi5544gmndQMHDpTFYlGfPn3++cAAAB6NxAkAUOCEhIRo0aJF+vPPP+1l6enp+vDDD1WpUiUTIwMAeCoSJwBAgdOoUSOFhIRo+fLl9rLly5erUqVKuv322+1lGRkZeuaZZ1S2bFn5+vrqjjvu0M8//+y0v9atW8tisTi8pk6d6lDn/fffV+3ateXr66tatWrpnXfecWs/hw8flsViUWJiYr5cAwBAwULiBAAokPr27au5c+fal+fMmaPo6GiHOsOHD9eyZcv0wQcfaOvWrQoLC1NkZKTOnj3rtL/+/fvrxIkTOnHihCpWrOiwbuHChRo3bpxeeukl7dq1Sy+//LLGjh2rDz74wKGeYRjX3Q8A4NZF4gQAKJB69eqlDRs26MiRIzpy5Ih++OEH9erVy74+LS1NM2bM0Kuvvqr77rtPderU0XvvvaciRYpo9uzZDvvKyMhQQECAgoODFRwcLC8vL4f1sbGxev3119W1a1dVqVJFXbt21bPPPqt3333Xod7ly5evux8AwK2rkNkBAACQk8DAQHXo0EHz5s2TYRjq0KGDypQpY19/4MABXb58WS1atLCXFS5cWE2aNNGuXbsc9nXmzBn5+/vneJy0tDQdOHBAjz32mPr3728vv3LligICAhzqpqSkqGjRoteNu3nz5vLy8lKJEiXUvHlzvf766/RMAcAtgMQJAFBg9e3bV4MGDZIkTZ8+/Yb2ceXKFf3++++qUqVKjutTU1MlSe+9956aNm3qsO6vPUrHjx9X+fLlr3u8xYsXq3bt2kpKStIzzzyjJ554QitXrryh2AEABQdD9QAABVa7du2UmZmpy5cvKzIy0mFdtWrV5O3trR9++MFedvnyZf3888+qU6eOvWzjxo1KT0/XnXfemeMxgoKCVL58eR08eFBhYWEOr2uTrQMHDujcuXMOk1PkJCQkRGFhYbrjjjv02GOPMVkEANwi6HECABRYXl5e9mF3f+39KVq0qJ588kkNGzZMpUqVUqVKlTR58mRdunRJjz32mCQpKSlJY8eOVYsWLeTj46OkpCRJUlZWli5evKg///xTRYoU0fjx4/XMM88oICBA7dq1U0ZGhjZv3qxz584pJiZGmzdv1jPPPKN69eopIiLiujFnZmYqPT1dJ0+e1NKlS1W3bt2bcGUAAP80EicAQIGW27NJkvTKK6/IZrPp0Ucf1cWLFxUREaEvv/xSJUuWlCR1795d3377rSSpXLlyDtuOGzdOISEh6tOnj/r16yc/Pz+9+uqrGjZsmIoWLap69eppyJAhkqRnn31WFStW1JQpU2SxWK4bb/ZwvxIlSuiOO+7QtGnTbvTUAQAFiMUwDMPsIAAAuBlat26tF154Qa1bt3ZaN2TIEDVs2FB9+vT5x+MCAPz78IwTAOCWVapUKXl7e+e4zt/fX0WKFPmHIwIA/FvR4wQAAAAALtDjBAAAAAAukDgBAAAAgAskTgAAAADgAokTAAAAALhA4gQAAAAALpA4AQAAAIALJE4AAAAA4AKJEwAAAAC48P8AocWxH0r0E70AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_df = pd.DataFrame({\n",
    "    \"Модель\": [\"Все признаки\", \"Отобранные признаки\"],\n",
    "    \"Accuracy\": [\n",
    "        accuracy_score(y_test, base_pred),\n",
    "        accuracy_score(y_test, sel_pred)\n",
    "    ],\n",
    "    \"F1\": [\n",
    "        f1_score(y_test, base_pred, zero_division=0),\n",
    "        f1_score(y_test, sel_pred, zero_division=0)\n",
    "    ],\n",
    "    \"ROC-AUC\": [\n",
    "        roc_auc_score(y_test, base_proba),\n",
    "        roc_auc_score(y_test, sel_proba)\n",
    "    ]\n",
    "})\n",
    "\n",
    "ax = plot_df.set_index(\"Модель\").plot(kind=\"bar\", figsize=(10, 5))\n",
    "plt.title(\"Сравнение моделей на тестовой выборке\")\n",
    "plt.ylabel(\"Значение метрики\")\n",
    "plt.xticks(rotation=0)\n",
    "plt.grid(axis=\"y\", alpha=0.3)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b901675",
   "metadata": {},
   "source": [
    "Итог\n",
    "\n",
    "Построен классификатор по всем признакам и модель после классического отбора признаков (RFECV).\n",
    "\n",
    "Модель на отобранных признаках показала сопоставимую точность при меньшем числе переменных, что повышает интерпретируемость и устойчивость модели.\n",
    "\n",
    "Следовательно, лучшей является модель после отбора признаков."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}