From 54083c485b948d5e541f38f6c358c847de547e81 Mon Sep 17 00:00:00 2001 From: blondered Date: Thu, 13 Feb 2025 13:46:47 +0300 Subject: [PATCH 01/15] customization with init_kwargs --- .../transformers_customization_guide.ipynb | 1051 +++++++++++++++++ rectools/models/nn/bert4rec.py | 9 +- rectools/models/nn/constants.py | 3 + rectools/models/nn/item_net.py | 53 +- rectools/models/nn/sasrec.py | 7 + rectools/models/nn/transformer_backbone.py | 3 + rectools/models/nn/transformer_base.py | 27 +- .../models/nn/transformer_data_preparator.py | 4 +- rectools/models/nn/transformer_lightning.py | 3 + rectools/models/nn/transformer_net_blocks.py | 13 +- 10 files changed, 1153 insertions(+), 20 deletions(-) create mode 100644 examples/tutorials/transformers_customization_guide.ipynb diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb new file mode 100644 index 00000000..0b277673 --- /dev/null +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -0,0 +1,1051 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import os\n", + "import pandas as pd\n", + "import torch\n", + "import typing as tp\n", + "import warnings\n", + "from pathlib import Path\n", + "\n", + "import torch.nn as nn\n", + "import typing_extensions as tpe\n", + "\n", + "from lightning_fabric import seed_everything\n", + "from pytorch_lightning import Trainer\n", + "from rectools import Columns\n", + "from rectools.dataset import Dataset\n", + "from rectools.models import BERT4RecModel, SASRecModel\n", + "\n", + "\n", + "from rectools.dataset.dataset import Dataset, DatasetSchema\n", + "from rectools.models.nn.item_net import (\n", + " ItemNetBase,\n", + " SumOfEmbeddingsConstructor,\n", + ")\n", + "from rectools.models.nn.transformer_net_blocks import (\n", + " PreLNTransformerLayer,\n", + " TransformerLayersBase,\n", + ")\n", + "from rectools.models.nn.constants import InitKwargs\n", + "\n", + "# Enable deterministic behaviour with CUDA >= 10.2\n", + "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n", + "warnings.simplefilter(\"ignore\", UserWarning)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# %%time\n", + "# !wget -q https://github.com/irsafilo/KION_DATASET/raw/f69775be31fa5779907cf0a92ddedb70037fb5ae/data_original.zip -O data_original.zip\n", + "# !unzip -o data_original.zip\n", + "# !rm data_original.zip" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "DATA_PATH = Path(\"data_original\")\n", + "\n", + "interactions = (\n", + " pd.read_csv(DATA_PATH / 'interactions.csv', parse_dates=[\"last_watch_dt\"])\n", + " .rename(columns={\"last_watch_dt\": \"datetime\"})\n", + ")\n", + "interactions[Columns.Weight] = np.where(interactions['watched_pct'] > 10, 3, 1)\n", + "dataset_no_features = Dataset.construct(\n", + " interactions_df=interactions,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 60\n" + ] + }, + { + "data": { + "text/plain": [ + "60" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "RANDOM_STATE=60\n", + "torch.use_deterministic_algorithms(True)\n", + "seed_everything(RANDOM_STATE, workers=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "# Function to get custom trainer for quick debugging\n", + "def get_debug_trainer() -> Trainer:\n", + " return Trainer(\n", + " accelerator=\"gpu\",\n", + " devices=[1],\n", + " min_epochs=1,\n", + " max_epochs=1,\n", + " deterministic=True,\n", + " limit_train_batches=2,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Training Objective**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "https://arxiv.org/pdf/2205.04507" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## **Next Action**" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Dict, List, Tuple\n", + "\n", + "from rectools.models.nn.bert4rec import BERT4RecDataPreparator\n", + "from rectools.models.nn.constants import MASKING_VALUE\n", + "from rectools.models.nn.transformer_lightning import TransformerLightningModule\n", + "\n", + "\n", + "# \"MASK\" token is used for predicting one next token.\n", + "class NextItemDataPreparator(BERT4RecDataPreparator):\n", + " \n", + " def _collate_fn_train(\n", + " self,\n", + " batch: List[Tuple[List[int], List[float]]],\n", + " ) -> Dict[str, torch.Tensor]:\n", + " \"\"\"\n", + " Truncate each session from right to keep `session_max_len` items.\n", + " Do left padding until `session_max_len` is reached.\n", + " Split to `x`, `y`, and `yw`.\n", + " \"\"\"\n", + " batch_size = len(batch)\n", + " x = np.zeros((batch_size, self.session_max_len))\n", + " y = np.zeros((batch_size, 1))\n", + " yw = np.zeros((batch_size, 1))\n", + " for i, (ses, ses_weights) in enumerate(batch):\n", + " session = ses.copy()\n", + " session[-1] = self.extra_token_ids[MASKING_VALUE]\n", + " x[i, -len(ses) :] = session # ses: [session_len] -> x[i]: [session_max_len]\n", + " y[i] = ses[-1] # ses: [session_len] -> y[i]: [1]\n", + " yw[i] = ses_weights[-1] # ses_weights: [session_len] -> yw[i]: [1]\n", + "\n", + " batch_dict = {\"x\": torch.LongTensor(x), \"y\": torch.LongTensor(y), \"yw\": torch.FloatTensor(yw)}\n", + " if self.n_negatives is not None:\n", + " negatives = torch.randint(\n", + " low=self.n_item_extra_tokens,\n", + " high=self.item_id_map.size,\n", + " size=(batch_size, 1, self.n_negatives),\n", + " ) # [batch_size, 1, n_negatives]\n", + " batch_dict[\"negatives\"] = negatives\n", + " return batch_dict\n", + "\n", + "\n", + "# Last logits are used for reducing the number of calculations on training step.\n", + "# You could also fill the y with zeros except for the last item in `_collate_fn_train`` and not change the training step \n", + "class NextItemLightningModule(TransformerLightningModule):\n", + "\n", + " def training_step(self, batch: tp.Dict[str, torch.Tensor], batch_idx: int) -> torch.Tensor:\n", + " \"\"\"Training step.\"\"\"\n", + " x, y, w = batch[\"x\"], batch[\"y\"], batch[\"yw\"]\n", + " if self.loss == \"softmax\":\n", + " logits = self._get_full_catalog_logits(x)[:, -1: :]\n", + " loss = self._calc_softmax_loss(logits, y, w)\n", + " elif self.loss == \"BCE\":\n", + " negatives = batch[\"negatives\"]\n", + " logits = self._get_pos_neg_logits(x, y, negatives)[:, -1: :]\n", + " loss = self._calc_bce_loss(logits, y, w)\n", + " elif self.loss == \"gBCE\":\n", + " negatives = batch[\"negatives\"]\n", + " logits = self._get_pos_neg_logits(x, y, negatives)[:, -1: :]\n", + " loss = self._calc_gbce_loss(logits, y, w, negatives)\n", + " else:\n", + " loss = self._calc_custom_loss(batch, batch_idx)\n", + "\n", + " self.log(self.train_loss_name, loss, on_step=False, on_epoch=True, prog_bar=self.verbose > 0)\n", + "\n", + " return loss" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + } + ], + "source": [ + "nextitem_transformer_bidirectional = BERT4RecModel(\n", + " data_preparator_type=NextItemDataPreparator, # \"NextItem\" training objective data preparator\n", + " lightning_module_type=NextItemLightningModule, # \"NextItem\" lightning module\n", + " get_trainer_func = get_debug_trainer,\n", + ")\n", + "\n", + "nextitem_transformer_unidirectional = SASRecModel(\n", + " data_preparator_type=NextItemDataPreparator, # \"NextItem\" training objective data preparator\n", + " lightning_module_type=NextItemLightningModule, # \"NextItem\" lightning module\n", + " use_causal_attn=True, # Apply causal attention mask\n", + " get_trainer_func = get_debug_trainer,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 3.1 M | train\n", + "-----------------------------------------------------------------\n", + "3.1 M Trainable params\n", + "0 Non-trainable params\n", + "3.1 M Total params\n", + "12.211 Total estimated model params size (MB)\n", + "37 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ac4c71895c35445ea26e0adfaa09a418", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "nextitem_transformer_bidirectional.fit(dataset_no_features)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 2.3 M | train\n", + "-----------------------------------------------------------------\n", + "2.3 M Trainable params\n", + "0 Non-trainable params\n", + "2.3 M Total params\n", + "9.061 Total estimated model params size (MB)\n", + "34 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "364cad6f60104b03a6d9deeff09c8634", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "nextitem_transformer_unidirectional.fit(dataset_no_features)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ALBERT\n", + "Albert has 2 main innovations which can be used together or separately:\n", + "1. Learning embeddings of smaller size and then projecting them to the required size through a Liner projection\n", + "2. Sharing weights between transformer layers" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "# ### ---------- Special Albert logic for Embeddings ---------- ### #\n", + "\n", + "class AlBERT4RecSumOfEmbeddingsConstructor(SumOfEmbeddingsConstructor):\n", + "\n", + " def __init__(\n", + " self,\n", + " n_items: int,\n", + " emb_factors: int,\n", + " n_factors: int,\n", + " item_net_blocks: tp.Sequence[ItemNetBase],\n", + " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " ) -> None:\n", + " super().__init__(\n", + " n_items=n_items,\n", + " item_net_blocks=item_net_blocks,\n", + " init_kwargs=init_kwargs\n", + " )\n", + " self.item_emb_proj = nn.Linear(emb_factors, n_factors)\n", + "\n", + " @classmethod\n", + " def from_dataset(\n", + " cls,\n", + " dataset: Dataset,\n", + " n_factors: int,\n", + " dropout_rate: float,\n", + " item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]],\n", + " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " ) -> tpe.Self:\n", + " n_items = dataset.item_id_map.size\n", + " \n", + " if init_kwargs is None or \"albert_emb_factors\" not in init_kwargs:\n", + " raise ValueError(\"Please specify `albert_emb_factors` in `init_kwargs` for Albert-like Embeddings\")\n", + " emb_factors = init_kwargs[\"albert_emb_factors\"]\n", + "\n", + " item_net_blocks: tp.List[ItemNetBase] = []\n", + " for item_net in item_net_block_types:\n", + " item_net_block = item_net.from_dataset(dataset, emb_factors, dropout_rate)\n", + " if item_net_block is not None:\n", + " item_net_blocks.append(item_net_block)\n", + "\n", + " return cls(n_items, emb_factors, n_factors, item_net_blocks, init_kwargs)\n", + "\n", + " @classmethod\n", + " def from_dataset_schema(\n", + " cls,\n", + " dataset_schema: DatasetSchema,\n", + " n_factors: int,\n", + " dropout_rate: float,\n", + " item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]],\n", + " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " ) -> tpe.Self:\n", + " n_items = dataset_schema.items.n_hot\n", + " \n", + " if init_kwargs is None or \"albert_emb_factors\" not in init_kwargs:\n", + " raise ValueError(\"Please specify `albert_emb_factors` in `init_kwargs` for Albert-like Embeddings\")\n", + " emb_factors = init_kwargs[\"albert_emb_factors\"]\n", + "\n", + " item_net_blocks: tp.List[ItemNetBase] = []\n", + " for item_net in item_net_block_types:\n", + " item_net_block = item_net.from_dataset_schema(dataset_schema, emb_factors, dropout_rate)\n", + " if item_net_block is not None:\n", + " item_net_blocks.append(item_net_block)\n", + "\n", + " return cls(n_items, emb_factors, n_factors, item_net_blocks, init_kwargs)\n", + "\n", + " def forward(self, items: torch.Tensor) -> torch.Tensor:\n", + " item_embs = super().forward(items)\n", + " item_embs = self.item_emb_proj(item_embs)\n", + " return item_embs\n", + " \n", + "\n", + "\n", + "# ### ---------- Special Albert logic for Transfromer Layers ---------- ### #\n", + " \n", + "class AlBERT4RecPreLNTransformerLayers(TransformerLayersBase):\n", + "\n", + " def __init__(\n", + " self,\n", + " n_blocks: int,\n", + " n_factors: int,\n", + " n_heads: int,\n", + " dropout_rate: float,\n", + " ff_factors_multiplier: int = 4,\n", + " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " ):\n", + " super().__init__()\n", + " \n", + " if init_kwargs is None or \"albert_n_hidden_groups\" not in init_kwargs:\n", + " raise ValueError(\"Please specify `albert_n_hidden_groups` in `init_kwargs` for Albert-like Embeddings\")\n", + " albert_n_hidden_groups = init_kwargs[\"albert_n_hidden_groups\"]\n", + " \n", + " if init_kwargs is None or \"albert_n_inner_groups\" not in init_kwargs:\n", + " raise ValueError(\"Please specify `albert_n_inner_groups` in `init_kwargs` for Albert-like Embeddings\")\n", + " albert_n_inner_groups = init_kwargs[\"albert_n_inner_groups\"]\n", + " \n", + " self.n_blocks = n_blocks\n", + " self.n_hidden_groups = albert_n_hidden_groups\n", + " self.n_inner_groups = albert_n_inner_groups\n", + " n_fitted_blocks = int(albert_n_hidden_groups * albert_n_inner_groups)\n", + " self.transformer_blocks = nn.ModuleList(\n", + " [\n", + " PreLNTransformerLayer(\n", + " # number of encoder layer (AlBERTLayers)\n", + " # https://github.com/huggingface/transformers/blob/main/src/transformers/models/albert/modeling_albert.py#L428\n", + " n_factors,\n", + " n_heads,\n", + " dropout_rate,\n", + " ff_factors_multiplier,\n", + " )\n", + " # https://github.com/huggingface/transformers/blob/main/src/transformers/models/albert/modeling_albert.py#L469\n", + " for _ in range(n_fitted_blocks)\n", + " ]\n", + " )\n", + " self.n_layers_per_group = n_blocks / albert_n_hidden_groups\n", + " self.init_kwargs = init_kwargs\n", + "\n", + " def forward(\n", + " self,\n", + " seqs: torch.Tensor,\n", + " timeline_mask: torch.Tensor,\n", + " attn_mask: tp.Optional[torch.Tensor],\n", + " key_padding_mask: tp.Optional[torch.Tensor],\n", + " ) -> torch.Tensor:\n", + " for block_idx in range(self.n_blocks):\n", + " group_idx = int(block_idx / self.n_layers_per_group)\n", + " for inner_layer_idx in range(self.n_inner_groups):\n", + " layer_idx = group_idx * self.n_inner_groups + inner_layer_idx\n", + " seqs = self.transformer_blocks[block_idx](seqs, attn_mask, key_padding_mask)\n", + " return seqs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "ALBERT_INIT_KWARGS = { # these arguments are obligatory for our custom model\n", + " \"albert_emb_factors\": 32,\n", + " \"albert_n_hidden_groups\": 2,\n", + " \"albert_n_inner_groups\": 1,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + } + ], + "source": [ + "albert_model = BERT4RecModel(\n", + " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " init_kwargs = ALBERT_INIT_KWARGS,\n", + " get_trainer_func = get_debug_trainer,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "-----------------------------------------------------------------\n", + "1.8 M Trainable params\n", + "0 Non-trainable params\n", + "1.8 M Total params\n", + "7.178 Total estimated model params size (MB)\n", + "38 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d3f01369da5741d1924cd475934b6c99", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "albert_model.fit(dataset_no_features)" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + } + ], + "source": [ + "alsasrec = SASRecModel(\n", + " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " init_kwargs = ALBERT_INIT_KWARGS,\n", + " get_trainer_func = get_debug_trainer,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "-----------------------------------------------------------------\n", + "1.8 M Trainable params\n", + "0 Non-trainable params\n", + "1.8 M Total params\n", + "7.178 Total estimated model params size (MB)\n", + "38 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "17e867574bd94d9199fa92ae2c64a86b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "alsasrec.fit(dataset_no_features)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How about NextTokenTransformer with Albert logic and causal attention?\n", + "# Just because we can!" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + } + ], + "source": [ + "next_action_albert_causal = BERT4RecModel(\n", + " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " data_preparator_type=NextItemDataPreparator, # custom data preparator\n", + " lightning_module_type=NextItemLightningModule, # custom lightning module\n", + " use_causal_attn=True, # Apply causal attention mask\n", + " get_trainer_func = get_debug_trainer,\n", + " init_kwargs = ALBERT_INIT_KWARGS,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "-----------------------------------------------------------------\n", + "1.8 M Trainable params\n", + "0 Non-trainable params\n", + "1.8 M Total params\n", + "7.178 Total estimated model params size (MB)\n", + "38 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6470804a48f24a28b58ab33baf74b0c0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "next_action_albert_causal.fit(dataset_no_features)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Whan about configs?" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [], + "source": [ + "params = next_action_albert_causal.get_params(simple_types=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'cls': 'BERT4RecModel',\n", + " 'verbose': 0,\n", + " 'data_preparator_type': '__main__.NextItemDataPreparator',\n", + " 'n_blocks': 2,\n", + " 'n_heads': 4,\n", + " 'n_factors': 256,\n", + " 'use_pos_emb': True,\n", + " 'use_causal_attn': True,\n", + " 'use_key_padding_mask': True,\n", + " 'dropout_rate': 0.2,\n", + " 'session_max_len': 100,\n", + " 'dataloader_num_workers': 0,\n", + " 'batch_size': 128,\n", + " 'loss': 'softmax',\n", + " 'n_negatives': 1,\n", + " 'gbce_t': 0.2,\n", + " 'lr': 0.001,\n", + " 'epochs': 3,\n", + " 'deterministic': False,\n", + " 'recommend_batch_size': 256,\n", + " 'recommend_device': None,\n", + " 'recommend_n_threads': 0,\n", + " 'recommend_use_torch_ranking': True,\n", + " 'train_min_user_interactions': 2,\n", + " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n", + " 'rectools.models.nn.item_net.CatFeaturesItemNet'],\n", + " 'item_net_constructor_type': '__main__.AlBERT4RecSumOfEmbeddingsConstructor',\n", + " 'pos_encoding_type': 'rectools.models.nn.transformer_net_blocks.LearnableInversePositionalEncoding',\n", + " 'transformer_layers_type': '__main__.AlBERT4RecPreLNTransformerLayers',\n", + " 'lightning_module_type': '__main__.NextItemLightningModule',\n", + " 'get_val_mask_func': None,\n", + " 'get_trainer_func': '__main__.get_debug_trainer',\n", + " 'init_kwargs.albert_emb_factors': 32,\n", + " 'init_kwargs.albert_n_hidden_groups': 2,\n", + " 'init_kwargs.albert_n_inner_groups': 1,\n", + " 'mask_prob': 0.15}" + ] + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + } + ], + "source": [ + "model = BERT4RecModel.from_params(params)" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'albert_emb_factors': 32,\n", + " 'albert_n_hidden_groups': 2,\n", + " 'albert_n_inner_groups': 1}" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.init_kwargs" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "-----------------------------------------------------------------\n", + "1.8 M Trainable params\n", + "0 Non-trainable params\n", + "1.8 M Total params\n", + "7.178 Total estimated model params size (MB)\n", + "38 Modules in train mode\n", + "0 Modules in eval mode\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e38d87109fc74b40b29b484af4056a73", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "model.fit(dataset_no_features)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "rectools", + "language": "python", + "name": "rectools" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/rectools/models/nn/bert4rec.py b/rectools/models/nn/bert4rec.py index 1732e403..468bdfa9 100644 --- a/rectools/models/nn/bert4rec.py +++ b/rectools/models/nn/bert4rec.py @@ -19,7 +19,7 @@ import numpy as np import torch -from .constants import MASKING_VALUE, PADDING_VALUE +from .constants import MASKING_VALUE, PADDING_VALUE, InitKwargs from .item_net import ( CatFeaturesItemNet, IdEmbeddingsItemNet, @@ -59,9 +59,10 @@ def __init__( batch_size: int, dataloader_num_workers: int, train_min_user_interactions: int, - mask_prob: float, + mask_prob: float = 0.15, shuffle_train: bool = True, get_val_mask_func: tp.Optional[ValMaskCallable] = None, + init_kwargs: tp.Optional[InitKwargs] = None, ) -> None: super().__init__( session_max_len=session_max_len, @@ -71,6 +72,7 @@ def __init__( train_min_user_interactions=train_min_user_interactions, shuffle_train=shuffle_train, get_val_mask_func=get_val_mask_func, + init_kwargs=init_kwargs, ) self.mask_prob = mask_prob @@ -315,6 +317,7 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals recommend_device: tp.Optional[str] = None, recommend_use_torch_ranking: bool = True, recommend_n_threads: int = 0, + init_kwargs: tp.Optional[InitKwargs] = None, ): self.mask_prob = mask_prob @@ -349,6 +352,7 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals lightning_module_type=lightning_module_type, get_val_mask_func=get_val_mask_func, get_trainer_func=get_trainer_func, + init_kwargs=init_kwargs, ) def _init_data_preparator(self) -> None: @@ -360,4 +364,5 @@ def _init_data_preparator(self) -> None: train_min_user_interactions=self.train_min_user_interactions, mask_prob=self.mask_prob, get_val_mask_func=self.get_val_mask_func, + init_kwargs=self.init_kwargs, ) diff --git a/rectools/models/nn/constants.py b/rectools/models/nn/constants.py index fafb8da9..bd7433e3 100644 --- a/rectools/models/nn/constants.py +++ b/rectools/models/nn/constants.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import typing as tp PADDING_VALUE = "PAD" MASKING_VALUE = "MASK" + +InitKwargs = tp.Dict[str, tp.Union[str, bool, int, float]] diff --git a/rectools/models/nn/item_net.py b/rectools/models/nn/item_net.py index d37bd15f..b2c522e2 100644 --- a/rectools/models/nn/item_net.py +++ b/rectools/models/nn/item_net.py @@ -22,6 +22,8 @@ from rectools.dataset.dataset import Dataset, DatasetSchema from rectools.dataset.features import SparseFeatures +from .constants import InitKwargs + class ItemNetBase(nn.Module): """Base class for item net.""" @@ -74,6 +76,7 @@ def __init__( n_cat_feature_values: int, n_factors: int, dropout_rate: float, + init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__() @@ -85,6 +88,8 @@ def __init__( self.register_buffer("emb_bag_inputs", emb_bag_inputs) self.register_buffer("input_lengths", input_lengths) + self.init_kwargs = init_kwargs + def forward(self, items: torch.Tensor) -> torch.Tensor: """ Forward pass to get item embeddings from categorical item features. @@ -116,7 +121,13 @@ def get_item_inputs_offsets(self, items: torch.Tensor) -> tp.Tuple[torch.Tensor, return item_emb_bag_inputs, item_offsets @classmethod - def from_dataset(cls, dataset: Dataset, n_factors: int, dropout_rate: float) -> tp.Optional[tpe.Self]: + def from_dataset( + cls, + dataset: Dataset, + n_factors: int, + dropout_rate: float, + init_kwargs: tp.Optional[InitKwargs] = None, + ) -> tp.Optional[tpe.Self]: """ Create CatFeaturesItemNet from RecTools dataset. @@ -165,11 +176,16 @@ def from_dataset(cls, dataset: Dataset, n_factors: int, dropout_rate: float) -> n_cat_feature_values=n_cat_feature_values, n_factors=n_factors, dropout_rate=dropout_rate, + init_kwargs=init_kwargs, ) @classmethod def from_dataset_schema( - cls, dataset_schema: DatasetSchema, n_factors: int, dropout_rate: float + cls, + dataset_schema: DatasetSchema, + n_factors: int, + dropout_rate: float, + init_kwargs: tp.Optional[InitKwargs] = None, ) -> tp.Optional[tpe.Self]: """Construct CatFeaturesItemNet from Dataset schema.""" if dataset_schema.items.features is None: @@ -205,6 +221,7 @@ def from_dataset_schema( n_cat_feature_values=n_cat_feature_values, n_factors=n_factors, dropout_rate=dropout_rate, + init_kwargs=init_kwargs, ) @@ -222,7 +239,7 @@ class IdEmbeddingsItemNet(ItemNetBase): Probability of a hidden unit to be zeroed. """ - def __init__(self, n_factors: int, n_items: int, dropout_rate: float): + def __init__(self, n_factors: int, n_items: int, dropout_rate: float, init_kwargs: tp.Optional[InitKwargs] = None): super().__init__() self.n_items = n_items @@ -232,6 +249,7 @@ def __init__(self, n_factors: int, n_items: int, dropout_rate: float): padding_idx=0, ) self.drop_layer = nn.Dropout(dropout_rate) + self.init_kwargs = init_kwargs def forward(self, items: torch.Tensor) -> torch.Tensor: """ @@ -252,7 +270,9 @@ def forward(self, items: torch.Tensor) -> torch.Tensor: return item_embs @classmethod - def from_dataset(cls, dataset: Dataset, n_factors: int, dropout_rate: float) -> tpe.Self: + def from_dataset( + cls, dataset: Dataset, n_factors: int, dropout_rate: float, init_kwargs: tp.Optional[InitKwargs] = None + ) -> tpe.Self: """ Create IdEmbeddingsItemNet from RecTools dataset. @@ -266,13 +286,19 @@ def from_dataset(cls, dataset: Dataset, n_factors: int, dropout_rate: float) -> Probability of a hidden unit of item embedding to be zeroed. """ n_items = dataset.item_id_map.size - return cls(n_factors, n_items, dropout_rate) + return cls(n_factors, n_items, dropout_rate, init_kwargs) @classmethod - def from_dataset_schema(cls, dataset_schema: DatasetSchema, n_factors: int, dropout_rate: float) -> tpe.Self: + def from_dataset_schema( + cls, + dataset_schema: DatasetSchema, + n_factors: int, + dropout_rate: float, + init_kwargs: tp.Optional[InitKwargs] = None, + ) -> tpe.Self: """Construct ItemNet from Dataset schema.""" n_items = dataset_schema.items.n_hot - return cls(n_factors, n_items, dropout_rate) + return cls(n_factors, n_items, dropout_rate, init_kwargs) class ItemNetConstructorBase(ItemNetBase): @@ -288,9 +314,7 @@ class ItemNetConstructorBase(ItemNetBase): """ def __init__( - self, - n_items: int, - item_net_blocks: tp.Sequence[ItemNetBase], + self, n_items: int, item_net_blocks: tp.Sequence[ItemNetBase], init_kwargs: tp.Optional[InitKwargs] = None ) -> None: super().__init__() @@ -300,6 +324,7 @@ def __init__( self.n_items = n_items self.n_item_blocks = len(item_net_blocks) self.item_net_blocks = nn.ModuleList(item_net_blocks) + self.init_kwargs = init_kwargs @property def catalog(self) -> torch.Tensor: @@ -317,6 +342,7 @@ def from_dataset( n_factors: int, dropout_rate: float, item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]], + init_kwargs: tp.Optional[InitKwargs] = None, ) -> tpe.Self: """ Construct ItemNet from RecTools dataset and from various blocks of item networks. @@ -336,11 +362,11 @@ def from_dataset( item_net_blocks: tp.List[ItemNetBase] = [] for item_net in item_net_block_types: - item_net_block = item_net.from_dataset(dataset, n_factors, dropout_rate) + item_net_block = item_net.from_dataset(dataset, n_factors, dropout_rate, init_kwargs) if item_net_block is not None: item_net_blocks.append(item_net_block) - return cls(n_items, item_net_blocks) + return cls(n_items, item_net_blocks, init_kwargs) @classmethod def from_dataset_schema( @@ -349,6 +375,7 @@ def from_dataset_schema( n_factors: int, dropout_rate: float, item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]], + init_kwargs: tp.Optional[InitKwargs] = None, ) -> tpe.Self: """Construct ItemNet from Dataset schema.""" n_items = dataset_schema.items.n_hot @@ -359,7 +386,7 @@ def from_dataset_schema( if item_net_block is not None: item_net_blocks.append(item_net_block) - return cls(n_items, item_net_blocks) + return cls(n_items, item_net_blocks, init_kwargs) def forward(self, items: torch.Tensor) -> torch.Tensor: """Forward pass through item net blocks and aggregation of the results. diff --git a/rectools/models/nn/sasrec.py b/rectools/models/nn/sasrec.py index b8405fd7..2b515b87 100644 --- a/rectools/models/nn/sasrec.py +++ b/rectools/models/nn/sasrec.py @@ -19,6 +19,7 @@ import torch from torch import nn +from .constants import InitKwargs from .item_net import ( CatFeaturesItemNet, IdEmbeddingsItemNet, @@ -131,6 +132,7 @@ def __init__( n_factors: int, n_heads: int, dropout_rate: float, + init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__() # important: original architecture had another version of MHA @@ -139,6 +141,7 @@ def __init__( self.ff_layer_norm = nn.LayerNorm(n_factors) self.feed_forward = PointWiseFeedForward(n_factors, n_factors, dropout_rate, torch.nn.ReLU()) self.dropout = torch.nn.Dropout(dropout_rate) + self.init_kwargs = init_kwargs def forward( self, @@ -198,6 +201,7 @@ def __init__( n_factors: int, n_heads: int, dropout_rate: float, + init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__() self.n_blocks = n_blocks @@ -212,6 +216,7 @@ def __init__( ] ) self.last_layernorm = torch.nn.LayerNorm(n_factors, eps=1e-8) + self.init_kwargs = init_kwargs def forward( self, @@ -398,6 +403,7 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals recommend_device: tp.Optional[str] = None, recommend_use_torch_ranking: bool = True, recommend_n_threads: int = 0, + init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__( transformer_layers_type=transformer_layers_type, @@ -430,4 +436,5 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals lightning_module_type=lightning_module_type, get_val_mask_func=get_val_mask_func, get_trainer_func=get_trainer_func, + init_kwargs=init_kwargs, ) diff --git a/rectools/models/nn/transformer_backbone.py b/rectools/models/nn/transformer_backbone.py index be462bc8..73191776 100644 --- a/rectools/models/nn/transformer_backbone.py +++ b/rectools/models/nn/transformer_backbone.py @@ -16,6 +16,7 @@ import torch +from .constants import InitKwargs from .item_net import ItemNetBase from .transformer_net_blocks import PositionalEncodingBase, TransformerLayersBase @@ -51,6 +52,7 @@ def __init__( transformer_layers: TransformerLayersBase, use_causal_attn: bool = True, use_key_padding_mask: bool = False, + init_kwargs: tp.Optional[InitKwargs] = None, ) -> None: super().__init__() @@ -61,6 +63,7 @@ def __init__( self.use_causal_attn = use_causal_attn self.use_key_padding_mask = use_key_padding_mask self.n_heads = n_heads + self.init_kwargs = init_kwargs @staticmethod def _convert_mask_to_float(mask: torch.Tensor, query: torch.Tensor) -> torch.Tensor: diff --git a/rectools/models/nn/transformer_base.py b/rectools/models/nn/transformer_base.py index ebf4333e..2bda96d0 100644 --- a/rectools/models/nn/transformer_base.py +++ b/rectools/models/nn/transformer_base.py @@ -31,6 +31,7 @@ from rectools.types import InternalIdsArray from rectools.utils.misc import get_class_or_function_full_path, import_object +from .constants import InitKwargs from .item_net import ( CatFeaturesItemNet, IdEmbeddingsItemNet, @@ -185,6 +186,7 @@ class TransformerModelConfig(ModelConfig): lightning_module_type: TransformerLightningModuleType = TransformerLightningModule get_val_mask_func: tp.Optional[ValMaskCallableSerialized] = None get_trainer_func: tp.Optional[TrainerCallableSerialized] = None + init_kwargs: tp.Optional[InitKwargs] = None TransformerModelConfig_T = tp.TypeVar("TransformerModelConfig_T", bound=TransformerModelConfig) @@ -236,6 +238,7 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals lightning_module_type: tp.Type[TransformerLightningModuleBase] = TransformerLightningModule, get_val_mask_func: tp.Optional[ValMaskCallable] = None, get_trainer_func: tp.Optional[TrainerCallable] = None, + init_kwargs: tp.Optional[InitKwargs] = None, **kwargs: tp.Any, ) -> None: super().__init__(verbose=verbose) @@ -268,6 +271,7 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals self.lightning_module_type = lightning_module_type self.get_val_mask_func = get_val_mask_func self.get_trainer_func = get_trainer_func + self.init_kwargs = init_kwargs self._init_data_preparator() self._init_trainer() @@ -284,6 +288,7 @@ def _init_data_preparator(self) -> None: dataloader_num_workers=self.dataloader_num_workers, train_min_user_interactions=self.train_min_user_interactions, get_val_mask_func=self.get_val_mask_func, + init_kwargs=self.init_kwargs, ) def _init_trainer(self) -> None: @@ -303,16 +308,29 @@ def _init_trainer(self) -> None: def _construct_item_net(self, dataset: Dataset) -> ItemNetBase: return self.item_net_constructor_type.from_dataset( - dataset, self.n_factors, self.dropout_rate, self.item_net_block_types + dataset, + self.n_factors, + self.dropout_rate, + self.item_net_block_types, + init_kwargs=self.init_kwargs, ) def _construct_item_net_from_dataset_schema(self, dataset_schema: DatasetSchema) -> ItemNetBase: return self.item_net_constructor_type.from_dataset_schema( - dataset_schema, self.n_factors, self.dropout_rate, self.item_net_block_types + dataset_schema, + self.n_factors, + self.dropout_rate, + self.item_net_block_types, + init_kwargs=self.init_kwargs, ) def _init_pos_encoding_layer(self) -> PositionalEncodingBase: - return self.pos_encoding_type(self.use_pos_emb, self.session_max_len, self.n_factors) + return self.pos_encoding_type( + self.use_pos_emb, + self.session_max_len, + self.n_factors, + init_kwargs=self.init_kwargs, + ) def _init_transformer_layers(self) -> TransformerLayersBase: return self.transformer_layers_type( @@ -320,6 +338,7 @@ def _init_transformer_layers(self) -> TransformerLayersBase: n_factors=self.n_factors, n_heads=self.n_heads, dropout_rate=self.dropout_rate, + init_kwargs=self.init_kwargs, ) def _init_torch_model(self, item_model: ItemNetBase) -> TransformerTorchBackbone: @@ -333,6 +352,7 @@ def _init_torch_model(self, item_model: ItemNetBase) -> TransformerTorchBackbone transformer_layers=transformer_layers, use_causal_attn=self.use_causal_attn, use_key_padding_mask=self.use_key_padding_mask, + init_kwargs=self.init_kwargs, ) def _init_lightning_model( @@ -354,6 +374,7 @@ def _init_lightning_model( verbose=self.verbose, train_loss_name=self.train_loss_name, val_loss_name=self.val_loss_name, + init_kwargs=self.init_kwargs, ) def _fit( diff --git a/rectools/models/nn/transformer_data_preparator.py b/rectools/models/nn/transformer_data_preparator.py index 80f8870b..ab9568c8 100644 --- a/rectools/models/nn/transformer_data_preparator.py +++ b/rectools/models/nn/transformer_data_preparator.py @@ -28,7 +28,7 @@ from rectools.dataset.features import DenseFeatures, Features, SparseFeatures from rectools.dataset.identifiers import IdMap -from .constants import PADDING_VALUE +from .constants import PADDING_VALUE, InitKwargs class SequenceDataset(TorchDataset): @@ -119,6 +119,7 @@ def __init__( train_min_user_interactions: int = 2, n_negatives: tp.Optional[int] = None, get_val_mask_func: tp.Optional[tp.Callable] = None, + init_kwargs: tp.Optional[InitKwargs] = None, **kwargs: tp.Any, ) -> None: self.item_id_map: IdMap @@ -132,6 +133,7 @@ def __init__( self.train_min_user_interactions = train_min_user_interactions self.shuffle_train = shuffle_train self.get_val_mask_func = get_val_mask_func + self.init_kwargs = init_kwargs def get_known_items_sorted_internal_ids(self) -> np.ndarray: """Return internal item ids from processed dataset in sorted order.""" diff --git a/rectools/models/nn/transformer_lightning.py b/rectools/models/nn/transformer_lightning.py index d1c4af0d..382b4485 100644 --- a/rectools/models/nn/transformer_lightning.py +++ b/rectools/models/nn/transformer_lightning.py @@ -25,6 +25,7 @@ from rectools.models.rank import Distance, ImplicitRanker, Ranker, TorchRanker from rectools.types import InternalIdsArray +from .constants import InitKwargs from .transformer_backbone import TransformerTorchBackbone from .transformer_data_preparator import TransformerDataPreparatorBase @@ -70,6 +71,7 @@ def __init__( train_loss_name: str = "train_loss", val_loss_name: str = "val_loss", adam_betas: tp.Tuple[float, float] = (0.9, 0.98), + init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__() self.torch_model = torch_model @@ -85,6 +87,7 @@ def __init__( self.train_loss_name = train_loss_name self.val_loss_name = val_loss_name self.item_embs: torch.Tensor + self.init_kwargs = init_kwargs self.save_hyperparameters(ignore=["torch_model", "data_preparator"]) diff --git a/rectools/models/nn/transformer_net_blocks.py b/rectools/models/nn/transformer_net_blocks.py index ed21356d..66bf3396 100644 --- a/rectools/models/nn/transformer_net_blocks.py +++ b/rectools/models/nn/transformer_net_blocks.py @@ -17,6 +17,8 @@ import torch from torch import nn +from .constants import InitKwargs + class PointWiseFeedForward(nn.Module): """ @@ -195,6 +197,7 @@ def __init__( n_heads: int, dropout_rate: float, ff_factors_multiplier: int = 4, + init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__() self.n_blocks = n_blocks @@ -209,6 +212,7 @@ def __init__( for _ in range(self.n_blocks) ] ) + self.init_kwargs = init_kwargs def forward( self, @@ -264,9 +268,16 @@ class LearnableInversePositionalEncoding(PositionalEncodingBase): Latent embeddings size. """ - def __init__(self, use_pos_emb: bool, session_max_len: int, n_factors: int): + def __init__( + self, + use_pos_emb: bool, + session_max_len: int, + n_factors: int, + init_kwargs: tp.Optional[InitKwargs] = None, + ): super().__init__() self.pos_emb = torch.nn.Embedding(session_max_len, n_factors) if use_pos_emb else None + self.init_kwargs = init_kwargs def forward(self, sessions: torch.Tensor) -> torch.Tensor: """ From 57476fa0b9d39474b19aa8c5f1ffe37da85ae547 Mon Sep 17 00:00:00 2001 From: blondered Date: Fri, 14 Feb 2025 12:46:56 +0300 Subject: [PATCH 02/15] defined kwargs for each type --- rectools/models/nn/bert4rec.py | 21 +++++--- rectools/models/nn/constants.py | 3 +- rectools/models/nn/item_net.py | 51 ++++++++++--------- rectools/models/nn/sasrec.py | 19 ++++--- rectools/models/nn/transformer_backbone.py | 3 -- rectools/models/nn/transformer_base.py | 47 ++++++++++++----- .../models/nn/transformer_data_preparator.py | 4 +- rectools/models/nn/transformer_lightning.py | 4 +- rectools/models/nn/transformer_net_blocks.py | 8 +-- tests/models/nn/test_bert4rec.py | 6 +++ tests/models/nn/test_sasrec.py | 6 +++ 11 files changed, 109 insertions(+), 63 deletions(-) diff --git a/rectools/models/nn/bert4rec.py b/rectools/models/nn/bert4rec.py index 468bdfa9..6c81a8b1 100644 --- a/rectools/models/nn/bert4rec.py +++ b/rectools/models/nn/bert4rec.py @@ -49,7 +49,6 @@ class BERT4RecDataPreparator(TransformerDataPreparatorBase): """Data Preparator for BERT4RecModel.""" train_session_max_len_addition: int = 0 - item_extra_tokens: tp.Sequence[Hashable] = (PADDING_VALUE, MASKING_VALUE) def __init__( @@ -62,7 +61,7 @@ def __init__( mask_prob: float = 0.15, shuffle_train: bool = True, get_val_mask_func: tp.Optional[ValMaskCallable] = None, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ) -> None: super().__init__( session_max_len=session_max_len, @@ -72,7 +71,6 @@ def __init__( train_min_user_interactions=train_min_user_interactions, shuffle_train=shuffle_train, get_val_mask_func=get_val_mask_func, - init_kwargs=init_kwargs, ) self.mask_prob = mask_prob @@ -317,7 +315,12 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals recommend_device: tp.Optional[str] = None, recommend_use_torch_ranking: bool = True, recommend_n_threads: int = 0, - init_kwargs: tp.Optional[InitKwargs] = None, + data_preparator_kwargs: tp.Optional[InitKwargs] = None, + transformer_layers_kwargs: tp.Optional[InitKwargs] = None, + item_net_block_kwargs: tp.Optional[InitKwargs] = None, + item_net_constructor_kwargs: tp.Optional[InitKwargs] = None, + pos_encoding_kwargs: tp.Optional[InitKwargs] = None, + lightning_module_kwargs: tp.Optional[InitKwargs] = None, ): self.mask_prob = mask_prob @@ -352,7 +355,12 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals lightning_module_type=lightning_module_type, get_val_mask_func=get_val_mask_func, get_trainer_func=get_trainer_func, - init_kwargs=init_kwargs, + data_preparator_kwargs=data_preparator_kwargs, + transformer_layers_kwargs=transformer_layers_kwargs, + item_net_block_kwargs=item_net_block_kwargs, + item_net_constructor_kwargs=item_net_constructor_kwargs, + pos_encoding_kwargs=pos_encoding_kwargs, + lightning_module_kwargs=lightning_module_kwargs, ) def _init_data_preparator(self) -> None: @@ -364,5 +372,6 @@ def _init_data_preparator(self) -> None: train_min_user_interactions=self.train_min_user_interactions, mask_prob=self.mask_prob, get_val_mask_func=self.get_val_mask_func, - init_kwargs=self.init_kwargs, + shuffle_train=True, + **self._get_kwargs(self.data_preparator_kwargs), ) diff --git a/rectools/models/nn/constants.py b/rectools/models/nn/constants.py index bd7433e3..5cdf119f 100644 --- a/rectools/models/nn/constants.py +++ b/rectools/models/nn/constants.py @@ -16,4 +16,5 @@ PADDING_VALUE = "PAD" MASKING_VALUE = "MASK" -InitKwargs = tp.Dict[str, tp.Union[str, bool, int, float]] +SimpleHashable = tp.Union[str, bool, int, float] +InitKwargs = tp.Dict[str, tp.Optional[tp.Union[SimpleHashable, tp.Tuple[SimpleHashable]]]] diff --git a/rectools/models/nn/item_net.py b/rectools/models/nn/item_net.py index b2c522e2..dc264538 100644 --- a/rectools/models/nn/item_net.py +++ b/rectools/models/nn/item_net.py @@ -22,8 +22,6 @@ from rectools.dataset.dataset import Dataset, DatasetSchema from rectools.dataset.features import SparseFeatures -from .constants import InitKwargs - class ItemNetBase(nn.Module): """Base class for item net.""" @@ -76,7 +74,7 @@ def __init__( n_cat_feature_values: int, n_factors: int, dropout_rate: float, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ): super().__init__() @@ -88,8 +86,6 @@ def __init__( self.register_buffer("emb_bag_inputs", emb_bag_inputs) self.register_buffer("input_lengths", input_lengths) - self.init_kwargs = init_kwargs - def forward(self, items: torch.Tensor) -> torch.Tensor: """ Forward pass to get item embeddings from categorical item features. @@ -126,7 +122,7 @@ def from_dataset( dataset: Dataset, n_factors: int, dropout_rate: float, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ) -> tp.Optional[tpe.Self]: """ Create CatFeaturesItemNet from RecTools dataset. @@ -176,7 +172,6 @@ def from_dataset( n_cat_feature_values=n_cat_feature_values, n_factors=n_factors, dropout_rate=dropout_rate, - init_kwargs=init_kwargs, ) @classmethod @@ -185,7 +180,7 @@ def from_dataset_schema( dataset_schema: DatasetSchema, n_factors: int, dropout_rate: float, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ) -> tp.Optional[tpe.Self]: """Construct CatFeaturesItemNet from Dataset schema.""" if dataset_schema.items.features is None: @@ -221,7 +216,6 @@ def from_dataset_schema( n_cat_feature_values=n_cat_feature_values, n_factors=n_factors, dropout_rate=dropout_rate, - init_kwargs=init_kwargs, ) @@ -239,7 +233,13 @@ class IdEmbeddingsItemNet(ItemNetBase): Probability of a hidden unit to be zeroed. """ - def __init__(self, n_factors: int, n_items: int, dropout_rate: float, init_kwargs: tp.Optional[InitKwargs] = None): + def __init__( + self, + n_factors: int, + n_items: int, + dropout_rate: float, + **kwargs: tp.Any, + ): super().__init__() self.n_items = n_items @@ -249,7 +249,6 @@ def __init__(self, n_factors: int, n_items: int, dropout_rate: float, init_kwarg padding_idx=0, ) self.drop_layer = nn.Dropout(dropout_rate) - self.init_kwargs = init_kwargs def forward(self, items: torch.Tensor) -> torch.Tensor: """ @@ -271,7 +270,11 @@ def forward(self, items: torch.Tensor) -> torch.Tensor: @classmethod def from_dataset( - cls, dataset: Dataset, n_factors: int, dropout_rate: float, init_kwargs: tp.Optional[InitKwargs] = None + cls, + dataset: Dataset, + n_factors: int, + dropout_rate: float, + **kwargs: tp.Any, ) -> tpe.Self: """ Create IdEmbeddingsItemNet from RecTools dataset. @@ -286,7 +289,7 @@ def from_dataset( Probability of a hidden unit of item embedding to be zeroed. """ n_items = dataset.item_id_map.size - return cls(n_factors, n_items, dropout_rate, init_kwargs) + return cls(n_factors, n_items, dropout_rate) @classmethod def from_dataset_schema( @@ -294,11 +297,11 @@ def from_dataset_schema( dataset_schema: DatasetSchema, n_factors: int, dropout_rate: float, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ) -> tpe.Self: """Construct ItemNet from Dataset schema.""" n_items = dataset_schema.items.n_hot - return cls(n_factors, n_items, dropout_rate, init_kwargs) + return cls(n_factors, n_items, dropout_rate) class ItemNetConstructorBase(ItemNetBase): @@ -314,7 +317,10 @@ class ItemNetConstructorBase(ItemNetBase): """ def __init__( - self, n_items: int, item_net_blocks: tp.Sequence[ItemNetBase], init_kwargs: tp.Optional[InitKwargs] = None + self, + n_items: int, + item_net_blocks: tp.Sequence[ItemNetBase], + **kwargs: tp.Any, ) -> None: super().__init__() @@ -324,7 +330,6 @@ def __init__( self.n_items = n_items self.n_item_blocks = len(item_net_blocks) self.item_net_blocks = nn.ModuleList(item_net_blocks) - self.init_kwargs = init_kwargs @property def catalog(self) -> torch.Tensor: @@ -342,7 +347,7 @@ def from_dataset( n_factors: int, dropout_rate: float, item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]], - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ) -> tpe.Self: """ Construct ItemNet from RecTools dataset and from various blocks of item networks. @@ -362,11 +367,11 @@ def from_dataset( item_net_blocks: tp.List[ItemNetBase] = [] for item_net in item_net_block_types: - item_net_block = item_net.from_dataset(dataset, n_factors, dropout_rate, init_kwargs) + item_net_block = item_net.from_dataset(dataset, n_factors, dropout_rate, **kwargs) if item_net_block is not None: item_net_blocks.append(item_net_block) - return cls(n_items, item_net_blocks, init_kwargs) + return cls(n_items, item_net_blocks) @classmethod def from_dataset_schema( @@ -375,18 +380,18 @@ def from_dataset_schema( n_factors: int, dropout_rate: float, item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]], - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ) -> tpe.Self: """Construct ItemNet from Dataset schema.""" n_items = dataset_schema.items.n_hot item_net_blocks: tp.List[ItemNetBase] = [] for item_net in item_net_block_types: - item_net_block = item_net.from_dataset_schema(dataset_schema, n_factors, dropout_rate) + item_net_block = item_net.from_dataset_schema(dataset_schema, n_factors, dropout_rate, **kwargs) if item_net_block is not None: item_net_blocks.append(item_net_block) - return cls(n_items, item_net_blocks, init_kwargs) + return cls(n_items, item_net_blocks) def forward(self, items: torch.Tensor) -> torch.Tensor: """Forward pass through item net blocks and aggregation of the results. diff --git a/rectools/models/nn/sasrec.py b/rectools/models/nn/sasrec.py index 2b515b87..08c6f33a 100644 --- a/rectools/models/nn/sasrec.py +++ b/rectools/models/nn/sasrec.py @@ -132,7 +132,6 @@ def __init__( n_factors: int, n_heads: int, dropout_rate: float, - init_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__() # important: original architecture had another version of MHA @@ -141,7 +140,6 @@ def __init__( self.ff_layer_norm = nn.LayerNorm(n_factors) self.feed_forward = PointWiseFeedForward(n_factors, n_factors, dropout_rate, torch.nn.ReLU()) self.dropout = torch.nn.Dropout(dropout_rate) - self.init_kwargs = init_kwargs def forward( self, @@ -201,7 +199,7 @@ def __init__( n_factors: int, n_heads: int, dropout_rate: float, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ): super().__init__() self.n_blocks = n_blocks @@ -216,7 +214,6 @@ def __init__( ] ) self.last_layernorm = torch.nn.LayerNorm(n_factors, eps=1e-8) - self.init_kwargs = init_kwargs def forward( self, @@ -403,7 +400,12 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals recommend_device: tp.Optional[str] = None, recommend_use_torch_ranking: bool = True, recommend_n_threads: int = 0, - init_kwargs: tp.Optional[InitKwargs] = None, + data_preparator_kwargs: tp.Optional[InitKwargs] = None, + transformer_layers_kwargs: tp.Optional[InitKwargs] = None, + item_net_block_kwargs: tp.Optional[InitKwargs] = None, + item_net_constructor_kwargs: tp.Optional[InitKwargs] = None, + pos_encoding_kwargs: tp.Optional[InitKwargs] = None, + lightning_module_kwargs: tp.Optional[InitKwargs] = None, ): super().__init__( transformer_layers_type=transformer_layers_type, @@ -436,5 +438,10 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals lightning_module_type=lightning_module_type, get_val_mask_func=get_val_mask_func, get_trainer_func=get_trainer_func, - init_kwargs=init_kwargs, + data_preparator_kwargs=data_preparator_kwargs, + transformer_layers_kwargs=transformer_layers_kwargs, + item_net_block_kwargs=item_net_block_kwargs, + item_net_constructor_kwargs=item_net_constructor_kwargs, + pos_encoding_kwargs=pos_encoding_kwargs, + lightning_module_kwargs=lightning_module_kwargs, ) diff --git a/rectools/models/nn/transformer_backbone.py b/rectools/models/nn/transformer_backbone.py index 73191776..be462bc8 100644 --- a/rectools/models/nn/transformer_backbone.py +++ b/rectools/models/nn/transformer_backbone.py @@ -16,7 +16,6 @@ import torch -from .constants import InitKwargs from .item_net import ItemNetBase from .transformer_net_blocks import PositionalEncodingBase, TransformerLayersBase @@ -52,7 +51,6 @@ def __init__( transformer_layers: TransformerLayersBase, use_causal_attn: bool = True, use_key_padding_mask: bool = False, - init_kwargs: tp.Optional[InitKwargs] = None, ) -> None: super().__init__() @@ -63,7 +61,6 @@ def __init__( self.use_causal_attn = use_causal_attn self.use_key_padding_mask = use_key_padding_mask self.n_heads = n_heads - self.init_kwargs = init_kwargs @staticmethod def _convert_mask_to_float(mask: torch.Tensor, query: torch.Tensor) -> torch.Tensor: diff --git a/rectools/models/nn/transformer_base.py b/rectools/models/nn/transformer_base.py index 2bda96d0..bbbaf48f 100644 --- a/rectools/models/nn/transformer_base.py +++ b/rectools/models/nn/transformer_base.py @@ -186,7 +186,12 @@ class TransformerModelConfig(ModelConfig): lightning_module_type: TransformerLightningModuleType = TransformerLightningModule get_val_mask_func: tp.Optional[ValMaskCallableSerialized] = None get_trainer_func: tp.Optional[TrainerCallableSerialized] = None - init_kwargs: tp.Optional[InitKwargs] = None + data_preparator_kwargs: tp.Optional[InitKwargs] = None + transformer_layers_kwargs: tp.Optional[InitKwargs] = None + item_net_block_kwargs: tp.Optional[InitKwargs] = None + item_net_constructor_kwargs: tp.Optional[InitKwargs] = None + pos_encoding_kwargs: tp.Optional[InitKwargs] = None + lightning_module_kwargs: tp.Optional[InitKwargs] = None TransformerModelConfig_T = tp.TypeVar("TransformerModelConfig_T", bound=TransformerModelConfig) @@ -208,7 +213,7 @@ class TransformerModelBase(ModelBase[TransformerModelConfig_T]): # pylint: disa def __init__( # pylint: disable=too-many-arguments, too-many-locals self, - data_preparator_type: TransformerDataPreparatorType, + data_preparator_type: tp.Type[TransformerDataPreparatorBase], transformer_layers_type: tp.Type[TransformerLayersBase] = PreLNTransformerLayers, n_blocks: int = 2, n_heads: int = 4, @@ -238,7 +243,12 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals lightning_module_type: tp.Type[TransformerLightningModuleBase] = TransformerLightningModule, get_val_mask_func: tp.Optional[ValMaskCallable] = None, get_trainer_func: tp.Optional[TrainerCallable] = None, - init_kwargs: tp.Optional[InitKwargs] = None, + data_preparator_kwargs: tp.Optional[InitKwargs] = None, + transformer_layers_kwargs: tp.Optional[InitKwargs] = None, + item_net_block_kwargs: tp.Optional[InitKwargs] = None, + item_net_constructor_kwargs: tp.Optional[InitKwargs] = None, + pos_encoding_kwargs: tp.Optional[InitKwargs] = None, + lightning_module_kwargs: tp.Optional[InitKwargs] = None, **kwargs: tp.Any, ) -> None: super().__init__(verbose=verbose) @@ -271,7 +281,12 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals self.lightning_module_type = lightning_module_type self.get_val_mask_func = get_val_mask_func self.get_trainer_func = get_trainer_func - self.init_kwargs = init_kwargs + self.data_preparator_kwargs = data_preparator_kwargs + self.transformer_layers_kwargs = transformer_layers_kwargs + self.item_net_block_kwargs = item_net_block_kwargs + self.item_net_constructor_kwargs = item_net_constructor_kwargs + self.pos_encoding_kwargs = pos_encoding_kwargs + self.lightning_module_kwargs = lightning_module_kwargs self._init_data_preparator() self._init_trainer() @@ -280,15 +295,23 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals self.data_preparator: TransformerDataPreparatorBase self.fit_trainer: tp.Optional[Trainer] = None + @staticmethod + def _get_kwargs(actual_kwargs: tp.Optional[InitKwargs]) -> InitKwargs: + kwargs = {} + if actual_kwargs is not None: + kwargs = actual_kwargs + return kwargs + def _init_data_preparator(self) -> None: self.data_preparator = self.data_preparator_type( session_max_len=self.session_max_len, - n_negatives=self.n_negatives if self.loss != "softmax" else None, batch_size=self.batch_size, dataloader_num_workers=self.dataloader_num_workers, train_min_user_interactions=self.train_min_user_interactions, + n_negatives=self.n_negatives if self.loss != "softmax" else None, get_val_mask_func=self.get_val_mask_func, - init_kwargs=self.init_kwargs, + shuffle_train=True, + **self._get_kwargs(self.data_preparator_kwargs), ) def _init_trainer(self) -> None: @@ -312,7 +335,7 @@ def _construct_item_net(self, dataset: Dataset) -> ItemNetBase: self.n_factors, self.dropout_rate, self.item_net_block_types, - init_kwargs=self.init_kwargs, + **self._get_kwargs(self.item_net_constructor_kwargs), ) def _construct_item_net_from_dataset_schema(self, dataset_schema: DatasetSchema) -> ItemNetBase: @@ -321,7 +344,7 @@ def _construct_item_net_from_dataset_schema(self, dataset_schema: DatasetSchema) self.n_factors, self.dropout_rate, self.item_net_block_types, - init_kwargs=self.init_kwargs, + **self._get_kwargs(self.item_net_constructor_kwargs), ) def _init_pos_encoding_layer(self) -> PositionalEncodingBase: @@ -329,7 +352,7 @@ def _init_pos_encoding_layer(self) -> PositionalEncodingBase: self.use_pos_emb, self.session_max_len, self.n_factors, - init_kwargs=self.init_kwargs, + **self._get_kwargs(self.pos_encoding_kwargs), ) def _init_transformer_layers(self) -> TransformerLayersBase: @@ -338,7 +361,7 @@ def _init_transformer_layers(self) -> TransformerLayersBase: n_factors=self.n_factors, n_heads=self.n_heads, dropout_rate=self.dropout_rate, - init_kwargs=self.init_kwargs, + **self._get_kwargs(self.transformer_layers_kwargs), ) def _init_torch_model(self, item_model: ItemNetBase) -> TransformerTorchBackbone: @@ -352,7 +375,6 @@ def _init_torch_model(self, item_model: ItemNetBase) -> TransformerTorchBackbone transformer_layers=transformer_layers, use_causal_attn=self.use_causal_attn, use_key_padding_mask=self.use_key_padding_mask, - init_kwargs=self.init_kwargs, ) def _init_lightning_model( @@ -374,7 +396,8 @@ def _init_lightning_model( verbose=self.verbose, train_loss_name=self.train_loss_name, val_loss_name=self.val_loss_name, - init_kwargs=self.init_kwargs, + adam_betas=(0.9, 0.98), + **self._get_kwargs(self.lightning_module_kwargs), ) def _fit( diff --git a/rectools/models/nn/transformer_data_preparator.py b/rectools/models/nn/transformer_data_preparator.py index ab9568c8..80f8870b 100644 --- a/rectools/models/nn/transformer_data_preparator.py +++ b/rectools/models/nn/transformer_data_preparator.py @@ -28,7 +28,7 @@ from rectools.dataset.features import DenseFeatures, Features, SparseFeatures from rectools.dataset.identifiers import IdMap -from .constants import PADDING_VALUE, InitKwargs +from .constants import PADDING_VALUE class SequenceDataset(TorchDataset): @@ -119,7 +119,6 @@ def __init__( train_min_user_interactions: int = 2, n_negatives: tp.Optional[int] = None, get_val_mask_func: tp.Optional[tp.Callable] = None, - init_kwargs: tp.Optional[InitKwargs] = None, **kwargs: tp.Any, ) -> None: self.item_id_map: IdMap @@ -133,7 +132,6 @@ def __init__( self.train_min_user_interactions = train_min_user_interactions self.shuffle_train = shuffle_train self.get_val_mask_func = get_val_mask_func - self.init_kwargs = init_kwargs def get_known_items_sorted_internal_ids(self) -> np.ndarray: """Return internal item ids from processed dataset in sorted order.""" diff --git a/rectools/models/nn/transformer_lightning.py b/rectools/models/nn/transformer_lightning.py index 382b4485..24b8e0af 100644 --- a/rectools/models/nn/transformer_lightning.py +++ b/rectools/models/nn/transformer_lightning.py @@ -25,7 +25,6 @@ from rectools.models.rank import Distance, ImplicitRanker, Ranker, TorchRanker from rectools.types import InternalIdsArray -from .constants import InitKwargs from .transformer_backbone import TransformerTorchBackbone from .transformer_data_preparator import TransformerDataPreparatorBase @@ -71,7 +70,7 @@ def __init__( train_loss_name: str = "train_loss", val_loss_name: str = "val_loss", adam_betas: tp.Tuple[float, float] = (0.9, 0.98), - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ): super().__init__() self.torch_model = torch_model @@ -87,7 +86,6 @@ def __init__( self.train_loss_name = train_loss_name self.val_loss_name = val_loss_name self.item_embs: torch.Tensor - self.init_kwargs = init_kwargs self.save_hyperparameters(ignore=["torch_model", "data_preparator"]) diff --git a/rectools/models/nn/transformer_net_blocks.py b/rectools/models/nn/transformer_net_blocks.py index 66bf3396..7e56256a 100644 --- a/rectools/models/nn/transformer_net_blocks.py +++ b/rectools/models/nn/transformer_net_blocks.py @@ -17,8 +17,6 @@ import torch from torch import nn -from .constants import InitKwargs - class PointWiseFeedForward(nn.Module): """ @@ -197,7 +195,7 @@ def __init__( n_heads: int, dropout_rate: float, ff_factors_multiplier: int = 4, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ): super().__init__() self.n_blocks = n_blocks @@ -212,7 +210,6 @@ def __init__( for _ in range(self.n_blocks) ] ) - self.init_kwargs = init_kwargs def forward( self, @@ -273,11 +270,10 @@ def __init__( use_pos_emb: bool, session_max_len: int, n_factors: int, - init_kwargs: tp.Optional[InitKwargs] = None, + **kwargs: tp.Any, ): super().__init__() self.pos_emb = torch.nn.Embedding(session_max_len, n_factors) if use_pos_emb else None - self.init_kwargs = init_kwargs def forward(self, sessions: torch.Tensor) -> torch.Tensor: """ diff --git a/tests/models/nn/test_bert4rec.py b/tests/models/nn/test_bert4rec.py index 2c756099..bc04cae9 100644 --- a/tests/models/nn/test_bert4rec.py +++ b/tests/models/nn/test_bert4rec.py @@ -757,6 +757,12 @@ def initial_config(self) -> tp.Dict[str, tp.Any]: "mask_prob": 0.15, "get_val_mask_func": leave_one_out_mask, "get_trainer_func": None, + "data_preparator_kwargs": None, + "transformer_layers_kwargs": None, + "item_net_block_kwargs": None, + "item_net_constructor_kwargs": None, + "pos_encoding_kwargs": None, + "lightning_module_kwargs": None, } return config diff --git a/tests/models/nn/test_sasrec.py b/tests/models/nn/test_sasrec.py index d9e4cf84..210e973e 100644 --- a/tests/models/nn/test_sasrec.py +++ b/tests/models/nn/test_sasrec.py @@ -926,6 +926,12 @@ def initial_config(self) -> tp.Dict[str, tp.Any]: "lightning_module_type": TransformerLightningModule, "get_val_mask_func": leave_one_out_mask, "get_trainer_func": None, + "data_preparator_kwargs": None, + "transformer_layers_kwargs": None, + "item_net_block_kwargs": None, + "item_net_constructor_kwargs": None, + "pos_encoding_kwargs": None, + "lightning_module_kwargs": None, } return config From 1562e6433c3e2f42967bf8f87419c097a4b71584 Mon Sep 17 00:00:00 2001 From: blondered Date: Fri, 14 Feb 2025 13:00:04 +0300 Subject: [PATCH 03/15] tested guide --- .../transformers_customization_guide.ipynb | 291 +++++++----------- 1 file changed, 108 insertions(+), 183 deletions(-) diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb index 0b277673..2adb2b54 100644 --- a/examples/tutorials/transformers_customization_guide.ipynb +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -79,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -95,7 +95,7 @@ "60" ] }, - "execution_count": 9, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -108,7 +108,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -147,7 +147,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -220,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -253,7 +253,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -266,12 +266,12 @@ "\n", " | Name | Type | Params | Mode \n", "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 3.1 M | train\n", + "0 | torch_model | TransformerTorchBackbone | 5.5 M | train\n", "-----------------------------------------------------------------\n", - "3.1 M Trainable params\n", + "5.5 M Trainable params\n", "0 Non-trainable params\n", - "3.1 M Total params\n", - "12.211 Total estimated model params size (MB)\n", + "5.5 M Total params\n", + "22.040 Total estimated model params size (MB)\n", "37 Modules in train mode\n", "0 Modules in eval mode\n" ] @@ -279,7 +279,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ac4c71895c35445ea26e0adfaa09a418", + "model_id": "49d0297542c7483dba27e03cd4ed2e84", "version_major": 2, "version_minor": 0 }, @@ -301,17 +301,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 24.6 s, sys: 1.18 s, total: 25.8 s\n", - "Wall time: 20.4 s\n" + "CPU times: user 30.9 s, sys: 3.3 s, total: 34.2 s\n", + "Wall time: 27.8 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 25, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -323,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -336,12 +336,12 @@ "\n", " | Name | Type | Params | Mode \n", "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 2.3 M | train\n", + "0 | torch_model | TransformerTorchBackbone | 4.7 M | train\n", "-----------------------------------------------------------------\n", - "2.3 M Trainable params\n", + "4.7 M Trainable params\n", "0 Non-trainable params\n", - "2.3 M Total params\n", - "9.061 Total estimated model params size (MB)\n", + "4.7 M Total params\n", + "18.890 Total estimated model params size (MB)\n", "34 Modules in train mode\n", "0 Modules in eval mode\n" ] @@ -349,7 +349,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "364cad6f60104b03a6d9deeff09c8634", + "model_id": "4ef281146a32431bb36965190283983d", "version_major": 2, "version_minor": 0 }, @@ -371,17 +371,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 24.7 s, sys: 1.33 s, total: 26 s\n", - "Wall time: 22.2 s\n" + "CPU times: user 28.9 s, sys: 3.31 s, total: 32.2 s\n", + "Wall time: 27 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 26, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -403,7 +403,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -414,15 +414,13 @@ " def __init__(\n", " self,\n", " n_items: int,\n", - " emb_factors: int,\n", " n_factors: int,\n", " item_net_blocks: tp.Sequence[ItemNetBase],\n", - " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " emb_factors: int = 16, # accept new kwargs\n", " ) -> None:\n", " super().__init__(\n", " n_items=n_items,\n", " item_net_blocks=item_net_blocks,\n", - " init_kwargs=init_kwargs\n", " )\n", " self.item_emb_proj = nn.Linear(emb_factors, n_factors)\n", "\n", @@ -433,13 +431,9 @@ " n_factors: int,\n", " dropout_rate: float,\n", " item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]],\n", - " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " emb_factors: int, # accept new kwargs\n", " ) -> tpe.Self:\n", " n_items = dataset.item_id_map.size\n", - " \n", - " if init_kwargs is None or \"albert_emb_factors\" not in init_kwargs:\n", - " raise ValueError(\"Please specify `albert_emb_factors` in `init_kwargs` for Albert-like Embeddings\")\n", - " emb_factors = init_kwargs[\"albert_emb_factors\"]\n", "\n", " item_net_blocks: tp.List[ItemNetBase] = []\n", " for item_net in item_net_block_types:\n", @@ -447,7 +441,7 @@ " if item_net_block is not None:\n", " item_net_blocks.append(item_net_block)\n", "\n", - " return cls(n_items, emb_factors, n_factors, item_net_blocks, init_kwargs)\n", + " return cls(n_items, n_factors, item_net_blocks, emb_factors)\n", "\n", " @classmethod\n", " def from_dataset_schema(\n", @@ -456,13 +450,9 @@ " n_factors: int,\n", " dropout_rate: float,\n", " item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]],\n", - " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " emb_factors: int, # accept new kwargs\n", " ) -> tpe.Self:\n", " n_items = dataset_schema.items.n_hot\n", - " \n", - " if init_kwargs is None or \"albert_emb_factors\" not in init_kwargs:\n", - " raise ValueError(\"Please specify `albert_emb_factors` in `init_kwargs` for Albert-like Embeddings\")\n", - " emb_factors = init_kwargs[\"albert_emb_factors\"]\n", "\n", " item_net_blocks: tp.List[ItemNetBase] = []\n", " for item_net in item_net_block_types:\n", @@ -470,15 +460,20 @@ " if item_net_block is not None:\n", " item_net_blocks.append(item_net_block)\n", "\n", - " return cls(n_items, emb_factors, n_factors, item_net_blocks, init_kwargs)\n", + " return cls(n_items, n_factors, item_net_blocks, emb_factors)\n", "\n", " def forward(self, items: torch.Tensor) -> torch.Tensor:\n", " item_embs = super().forward(items)\n", " item_embs = self.item_emb_proj(item_embs)\n", - " return item_embs\n", - " \n", - "\n", - "\n", + " return item_embs" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ "# ### ---------- Special Albert logic for Transfromer Layers ---------- ### #\n", " \n", "class AlBERT4RecPreLNTransformerLayers(TransformerLayersBase):\n", @@ -490,22 +485,16 @@ " n_heads: int,\n", " dropout_rate: float,\n", " ff_factors_multiplier: int = 4,\n", - " init_kwargs: tp.Optional[InitKwargs] = None,\n", + " n_hidden_groups: int=1, # accept new kwarg\n", + " n_inner_groups: int=1, # accept new kwarg\n", + " \n", " ):\n", " super().__init__()\n", " \n", - " if init_kwargs is None or \"albert_n_hidden_groups\" not in init_kwargs:\n", - " raise ValueError(\"Please specify `albert_n_hidden_groups` in `init_kwargs` for Albert-like Embeddings\")\n", - " albert_n_hidden_groups = init_kwargs[\"albert_n_hidden_groups\"]\n", - " \n", - " if init_kwargs is None or \"albert_n_inner_groups\" not in init_kwargs:\n", - " raise ValueError(\"Please specify `albert_n_inner_groups` in `init_kwargs` for Albert-like Embeddings\")\n", - " albert_n_inner_groups = init_kwargs[\"albert_n_inner_groups\"]\n", - " \n", " self.n_blocks = n_blocks\n", - " self.n_hidden_groups = albert_n_hidden_groups\n", - " self.n_inner_groups = albert_n_inner_groups\n", - " n_fitted_blocks = int(albert_n_hidden_groups * albert_n_inner_groups)\n", + " self.n_hidden_groups = n_hidden_groups\n", + " self.n_inner_groups = n_inner_groups\n", + " n_fitted_blocks = int(n_hidden_groups * n_inner_groups)\n", " self.transformer_blocks = nn.ModuleList(\n", " [\n", " PreLNTransformerLayer(\n", @@ -520,8 +509,7 @@ " for _ in range(n_fitted_blocks)\n", " ]\n", " )\n", - " self.n_layers_per_group = n_blocks / albert_n_hidden_groups\n", - " self.init_kwargs = init_kwargs\n", + " self.n_layers_per_group = n_blocks / n_hidden_groups\n", "\n", " def forward(\n", " self,\n", @@ -540,20 +528,23 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ - "ALBERT_INIT_KWARGS = { # these arguments are obligatory for our custom model\n", - " \"albert_emb_factors\": 32,\n", - " \"albert_n_hidden_groups\": 2,\n", - " \"albert_n_inner_groups\": 1,\n", + "ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS = { # these arguments are obligatory for our custom classes\n", + " \"emb_factors\": 32,\n", + "}\n", + "\n", + "ALBERT_TRANSFORMER_LAYERS_KWARGS = { # these arguments are obligatory for our custom classes\n", + " \"n_hidden_groups\": 2,\n", + " \"n_inner_groups\": 1,\n", "}" ] }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -569,15 +560,16 @@ "source": [ "albert_model = BERT4RecModel(\n", " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " item_net_constructor_kwargs=ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " init_kwargs = ALBERT_INIT_KWARGS,\n", + " transformer_layers_kwargs=ALBERT_TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", " get_trainer_func = get_debug_trainer,\n", ")" ] }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -590,12 +582,12 @@ "\n", " | Name | Type | Params | Mode \n", "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", "-----------------------------------------------------------------\n", - "1.8 M Trainable params\n", + "2.1 M Trainable params\n", "0 Non-trainable params\n", - "1.8 M Total params\n", - "7.178 Total estimated model params size (MB)\n", + "2.1 M Total params\n", + "8.407 Total estimated model params size (MB)\n", "38 Modules in train mode\n", "0 Modules in eval mode\n" ] @@ -603,7 +595,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d3f01369da5741d1924cd475934b6c99", + "model_id": "aebe81c85dbb482180d21be64d4d7411", "version_major": 2, "version_minor": 0 }, @@ -625,17 +617,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 21.5 s, sys: 773 ms, total: 22.3 s\n", - "Wall time: 17 s\n" + "CPU times: user 31.6 s, sys: 4.36 s, total: 36 s\n", + "Wall time: 29.3 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 68, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -647,7 +639,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -663,15 +655,16 @@ "source": [ "alsasrec = SASRecModel(\n", " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " item_net_constructor_kwargs=ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " init_kwargs = ALBERT_INIT_KWARGS,\n", + " transformer_layers_kwargs=ALBERT_TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", " get_trainer_func = get_debug_trainer,\n", ")" ] }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -684,12 +677,12 @@ "\n", " | Name | Type | Params | Mode \n", "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", "-----------------------------------------------------------------\n", - "1.8 M Trainable params\n", + "2.1 M Trainable params\n", "0 Non-trainable params\n", - "1.8 M Total params\n", - "7.178 Total estimated model params size (MB)\n", + "2.1 M Total params\n", + "8.407 Total estimated model params size (MB)\n", "38 Modules in train mode\n", "0 Modules in eval mode\n" ] @@ -697,7 +690,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "17e867574bd94d9199fa92ae2c64a86b", + "model_id": "0be15dcd434342a0b73efe8153e18774", "version_major": 2, "version_minor": 0 }, @@ -719,17 +712,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 31.2 s, sys: 1.97 s, total: 33.2 s\n", - "Wall time: 27.2 s\n" + "CPU times: user 57 s, sys: 5.67 s, total: 1min 2s\n", + "Wall time: 39.9 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 70, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -749,7 +742,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -765,18 +758,19 @@ "source": [ "next_action_albert_causal = BERT4RecModel(\n", " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " item_net_constructor_kwargs=ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " transformer_layers_kwargs=ALBERT_TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", " data_preparator_type=NextItemDataPreparator, # custom data preparator\n", " lightning_module_type=NextItemLightningModule, # custom lightning module\n", " use_causal_attn=True, # Apply causal attention mask\n", " get_trainer_func = get_debug_trainer,\n", - " init_kwargs = ALBERT_INIT_KWARGS,\n", ")" ] }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -789,12 +783,12 @@ "\n", " | Name | Type | Params | Mode \n", "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", + "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", "-----------------------------------------------------------------\n", - "1.8 M Trainable params\n", + "2.1 M Trainable params\n", "0 Non-trainable params\n", - "1.8 M Total params\n", - "7.178 Total estimated model params size (MB)\n", + "2.1 M Total params\n", + "8.407 Total estimated model params size (MB)\n", "38 Modules in train mode\n", "0 Modules in eval mode\n" ] @@ -802,7 +796,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6470804a48f24a28b58ab33baf74b0c0", + "model_id": "ab2068caad4b406d86f674af208d2a53", "version_major": 2, "version_minor": 0 }, @@ -824,17 +818,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 26.1 s, sys: 1.62 s, total: 27.7 s\n", - "Wall time: 22.6 s\n" + "CPU times: user 47.5 s, sys: 4.16 s, total: 51.6 s\n", + "Wall time: 37.5 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 72, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -848,12 +842,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Whan about configs?" + "## What about configs?" ] }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -862,7 +856,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -900,13 +894,17 @@ " 'lightning_module_type': '__main__.NextItemLightningModule',\n", " 'get_val_mask_func': None,\n", " 'get_trainer_func': '__main__.get_debug_trainer',\n", - " 'init_kwargs.albert_emb_factors': 32,\n", - " 'init_kwargs.albert_n_hidden_groups': 2,\n", - " 'init_kwargs.albert_n_inner_groups': 1,\n", + " 'data_preparator_kwargs': None,\n", + " 'transformer_layers_kwargs.n_hidden_groups': 2,\n", + " 'transformer_layers_kwargs.n_inner_groups': 1,\n", + " 'item_net_block_kwargs': None,\n", + " 'item_net_constructor_kwargs.emb_factors': 32,\n", + " 'pos_encoding_kwargs': None,\n", + " 'lightning_module_kwargs': None,\n", " 'mask_prob': 0.15}" ] }, - "execution_count": 82, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -917,7 +915,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -936,29 +934,7 @@ }, { "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'albert_emb_factors': 32,\n", - " 'albert_n_hidden_groups': 2,\n", - " 'albert_n_inner_groups': 1}" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.init_kwargs" - ] - }, - { - "cell_type": "code", - "execution_count": 80, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -966,59 +942,8 @@ "output_type": "stream", "text": [ "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 1.8 M | train\n", - "-----------------------------------------------------------------\n", - "1.8 M Trainable params\n", - "0 Non-trainable params\n", - "1.8 M Total params\n", - "7.178 Total estimated model params size (MB)\n", - "38 Modules in train mode\n", - "0 Modules in eval mode\n" + " unq_values = pd.unique(values)\n" ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e38d87109fc74b40b29b484af4056a73", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ From aefdeab8870cca8691b7faf455a5209967fcd486 Mon Sep 17 00:00:00 2001 From: blondered Date: Fri, 14 Feb 2025 13:47:43 +0300 Subject: [PATCH 04/15] guide up --- .../transformers_customization_guide.ipynb | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb index 2adb2b54..4d70099c 100644 --- a/examples/tutorials/transformers_customization_guide.ipynb +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -934,7 +934,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -942,8 +942,59 @@ "output_type": "stream", "text": [ "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n" + " unq_values = pd.unique(values)\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", + "-----------------------------------------------------------------\n", + "2.1 M Trainable params\n", + "0 Non-trainable params\n", + "2.1 M Total params\n", + "8.407 Total estimated model params size (MB)\n", + "38 Modules in train mode\n", + "0 Modules in eval mode\n" ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cb5a9bb492a04e1fa58ea29faa47d69f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ From 5e9e12ad3851b8b0dbe8a78842f95b0949ec68b1 Mon Sep 17 00:00:00 2001 From: blondered Date: Fri, 14 Feb 2025 22:36:39 +0300 Subject: [PATCH 05/15] training guide --- ...transformers_advanced_training_guide.ipynb | 94 ++++++++++++------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/examples/tutorials/transformers_advanced_training_guide.ipynb b/examples/tutorials/transformers_advanced_training_guide.ipynb index 8a60feba..3354bf89 100644 --- a/examples/tutorials/transformers_advanced_training_guide.ipynb +++ b/examples/tutorials/transformers_advanced_training_guide.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -53,7 +53,7 @@ "from rectools.metrics import NDCG, Recall, Serendipity, calc_metrics\n", "from rectools.models import BERT4RecModel, SASRecModel, load_model\n", "from rectools.models.nn.item_net import IdEmbeddingsItemNet\n", - "from rectools.models.nn.transformer_base import TransformerModelBase\n", + "from rectools.models.nn.transformers.base import TransformerModelBase\n", "\n", "# Enable deterministic behaviour with CUDA >= 10.2\n", "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n", @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -147,7 +147,7 @@ "1 699317 1659 2021-05-29 8317 100.0" ] }, - "execution_count": 3, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -167,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -176,7 +176,7 @@ "(962179, 15706)" ] }, - "execution_count": 4, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -187,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -210,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -226,7 +226,7 @@ "60" ] }, - "execution_count": 6, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -261,7 +261,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -765,7 +765,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -860,7 +860,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -1179,7 +1179,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -1192,6 +1192,31 @@ "HPU available: False, using: 0 HPUs\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n" ] + }, + { + "ename": "AttributeError", + "evalue": "'TransformerLightningModule' object has no attribute 'data_preparator'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[16], line 54\u001b[0m\n\u001b[1;32m 38\u001b[0m model \u001b[38;5;241m=\u001b[39m SASRecModel(\n\u001b[1;32m 39\u001b[0m n_factors\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m256\u001b[39m,\n\u001b[1;32m 40\u001b[0m n_blocks\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 49\u001b[0m get_trainer_func\u001b[38;5;241m=\u001b[39mget_custom_trainer, \u001b[38;5;66;03m# pass function to initialize our custom trainer\u001b[39;00m\n\u001b[1;32m 50\u001b[0m )\n\u001b[1;32m 53\u001b[0m \u001b[38;5;66;03m# Fit model. Everything will happen under the hood\u001b[39;00m\n\u001b[0;32m---> 54\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m)\u001b[49m;\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/rectools/models/base.py:326\u001b[0m, in \u001b[0;36mModelBase.fit\u001b[0;34m(self, dataset, *args, **kwargs)\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m: T, dataset: Dataset, \u001b[38;5;241m*\u001b[39margs: tp\u001b[38;5;241m.\u001b[39mAny, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: tp\u001b[38;5;241m.\u001b[39mAny) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 314\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 315\u001b[0m \u001b[38;5;124;03m Fit model.\u001b[39;00m\n\u001b[1;32m 316\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 324\u001b[0m \u001b[38;5;124;03m self\u001b[39;00m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 326\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_fitted \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/rectools/models/nn/transformers/base.py:381\u001b[0m, in \u001b[0;36mTransformerModelBase._fit\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_lightning_model(\n\u001b[1;32m 374\u001b[0m torch_model\u001b[38;5;241m=\u001b[39mtorch_model,\n\u001b[1;32m 375\u001b[0m dataset_schema\u001b[38;5;241m=\u001b[39mdataset_schema,\n\u001b[1;32m 376\u001b[0m item_external_ids\u001b[38;5;241m=\u001b[39mitem_external_ids,\n\u001b[1;32m 377\u001b[0m model_config\u001b[38;5;241m=\u001b[39mmodel_config,\n\u001b[1;32m 378\u001b[0m )\n\u001b[1;32m 380\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfit_trainer \u001b[38;5;241m=\u001b[39m deepcopy(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trainer)\n\u001b[0;32m--> 381\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_trainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlightning_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_dataloader\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:538\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m=\u001b[39m TrainerStatus\u001b[38;5;241m.\u001b[39mRUNNING\n\u001b[1;32m 537\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 538\u001b[0m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_and_handle_interrupt\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 539\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit_impl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_dataloaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdatamodule\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mckpt_path\u001b[49m\n\u001b[1;32m 540\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py:47\u001b[0m, in \u001b[0;36m_call_and_handle_interrupt\u001b[0;34m(trainer, trainer_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mlauncher \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mlauncher\u001b[38;5;241m.\u001b[39mlaunch(trainer_fn, \u001b[38;5;241m*\u001b[39margs, trainer\u001b[38;5;241m=\u001b[39mtrainer, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 47\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtrainer_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m _TunerExitException:\n\u001b[1;32m 50\u001b[0m _call_teardown_hook(trainer)\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:574\u001b[0m, in \u001b[0;36mTrainer._fit_impl\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mfn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 568\u001b[0m ckpt_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_checkpoint_connector\u001b[38;5;241m.\u001b[39m_select_ckpt_path(\n\u001b[1;32m 569\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mfn,\n\u001b[1;32m 570\u001b[0m ckpt_path,\n\u001b[1;32m 571\u001b[0m model_provided\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 572\u001b[0m model_connected\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlightning_module \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 573\u001b[0m )\n\u001b[0;32m--> 574\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mckpt_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mckpt_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 576\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mstopped\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:981\u001b[0m, in \u001b[0;36mTrainer._run\u001b[0;34m(self, model, ckpt_path)\u001b[0m\n\u001b[1;32m 976\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_signal_connector\u001b[38;5;241m.\u001b[39mregister_signal_handlers()\n\u001b[1;32m 978\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[1;32m 979\u001b[0m \u001b[38;5;66;03m# RUN THE TRAINER\u001b[39;00m\n\u001b[1;32m 980\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[0;32m--> 981\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run_stage\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 983\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[1;32m 984\u001b[0m \u001b[38;5;66;03m# POST-Training CLEAN UP\u001b[39;00m\n\u001b[1;32m 985\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[1;32m 986\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: trainer tearing down\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1023\u001b[0m, in \u001b[0;36mTrainer._run_stage\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining:\n\u001b[1;32m 1022\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m isolate_rng():\n\u001b[0;32m-> 1023\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run_sanity_check\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1024\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mset_detect_anomaly(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_detect_anomaly):\n\u001b[1;32m 1025\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfit_loop\u001b[38;5;241m.\u001b[39mrun()\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1052\u001b[0m, in \u001b[0;36mTrainer._run_sanity_check\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1049\u001b[0m call\u001b[38;5;241m.\u001b[39m_call_callback_hooks(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_sanity_check_start\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1051\u001b[0m \u001b[38;5;66;03m# run eval step\u001b[39;00m\n\u001b[0;32m-> 1052\u001b[0m \u001b[43mval_loop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1054\u001b[0m call\u001b[38;5;241m.\u001b[39m_call_callback_hooks(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_sanity_check_end\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1056\u001b[0m \u001b[38;5;66;03m# reset logger connector\u001b[39;00m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/loops/utilities.py:178\u001b[0m, in \u001b[0;36m_no_grad_context.._decorator\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 176\u001b[0m context_manager \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mno_grad\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m context_manager():\n\u001b[0;32m--> 178\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloop_run\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/loops/evaluation_loop.py:135\u001b[0m, in \u001b[0;36m_EvaluationLoop.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_progress\u001b[38;5;241m.\u001b[39mis_last_batch \u001b[38;5;241m=\u001b[39m data_fetcher\u001b[38;5;241m.\u001b[39mdone\n\u001b[1;32m 134\u001b[0m \u001b[38;5;66;03m# run step hooks\u001b[39;00m\n\u001b[0;32m--> 135\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_evaluation_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdataloader_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdataloader_iter\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;66;03m# this needs to wrap the `*_step` call too (not just `next`) for `dataloader_iter` support\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/loops/evaluation_loop.py:410\u001b[0m, in \u001b[0;36m_EvaluationLoop._evaluation_step\u001b[0;34m(self, batch, batch_idx, dataloader_idx, dataloader_iter)\u001b[0m\n\u001b[1;32m 405\u001b[0m hook_kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_kwargs(\n\u001b[1;32m 406\u001b[0m batch, batch_idx, dataloader_idx \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_sequential \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_dataloaders \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 407\u001b[0m )\n\u001b[1;32m 409\u001b[0m hook_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_test_batch_end\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mtesting \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_validation_batch_end\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 410\u001b[0m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_callback_hooks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrainer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhook_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhook_kwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 411\u001b[0m call\u001b[38;5;241m.\u001b[39m_call_lightning_module_hook(trainer, hook_name, output, \u001b[38;5;241m*\u001b[39mhook_kwargs\u001b[38;5;241m.\u001b[39mvalues())\n\u001b[1;32m 413\u001b[0m trainer\u001b[38;5;241m.\u001b[39m_logger_connector\u001b[38;5;241m.\u001b[39mon_batch_end()\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py:218\u001b[0m, in \u001b[0;36m_call_callback_hooks\u001b[0;34m(trainer, hook_name, monitoring_callbacks, *args, **kwargs)\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcallable\u001b[39m(fn):\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mprofile(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[Callback]\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcallback\u001b[38;5;241m.\u001b[39mstate_key\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhook_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 218\u001b[0m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrainer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlightning_module\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pl_module:\n\u001b[1;32m 221\u001b[0m \u001b[38;5;66;03m# restore current_fx when nested context\u001b[39;00m\n\u001b[1;32m 222\u001b[0m pl_module\u001b[38;5;241m.\u001b[39m_current_fx_name \u001b[38;5;241m=\u001b[39m prev_fx_name\n", + "Cell \u001b[0;32mIn[13], line 57\u001b[0m, in \u001b[0;36mValidationMetrics.on_validation_batch_end\u001b[0;34m(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx)\u001b[0m\n\u001b[1;32m 48\u001b[0m batch_recos_df[Columns\u001b[38;5;241m.\u001b[39mRank] \u001b[38;5;241m=\u001b[39m batch_recos_df\u001b[38;5;241m.\u001b[39mgroupby(Columns\u001b[38;5;241m.\u001b[39mUser, sort\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\u001b[38;5;241m.\u001b[39mcumcount() \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 50\u001b[0m interactions \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(\n\u001b[1;32m 51\u001b[0m {\n\u001b[1;32m 52\u001b[0m Columns\u001b[38;5;241m.\u001b[39mUser: batch_target_users,\n\u001b[1;32m 53\u001b[0m Columns\u001b[38;5;241m.\u001b[39mItem: \u001b[38;5;28mlist\u001b[39m(itertools\u001b[38;5;241m.\u001b[39mchain\u001b[38;5;241m.\u001b[39mfrom_iterable(targets)),\n\u001b[1;32m 54\u001b[0m }\n\u001b[1;32m 55\u001b[0m )\n\u001b[0;32m---> 57\u001b[0m prev_interactions \u001b[38;5;241m=\u001b[39m \u001b[43mpl_module\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata_preparator\u001b[49m\u001b[38;5;241m.\u001b[39mtrain_dataset\u001b[38;5;241m.\u001b[39minteractions\u001b[38;5;241m.\u001b[39mdf\n\u001b[1;32m 58\u001b[0m catalog \u001b[38;5;241m=\u001b[39m prev_interactions[Columns\u001b[38;5;241m.\u001b[39mItem]\u001b[38;5;241m.\u001b[39munique()\n\u001b[1;32m 60\u001b[0m batch_metrics \u001b[38;5;241m=\u001b[39m calc_metrics(\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mval_metrics, \n\u001b[1;32m 62\u001b[0m batch_recos_df,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 65\u001b[0m catalog\n\u001b[1;32m 66\u001b[0m )\n", + "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1931\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1929\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1930\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1931\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[1;32m 1932\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1933\u001b[0m )\n", + "\u001b[0;31mAttributeError\u001b[0m: 'TransformerLightningModule' object has no attribute 'data_preparator'" + ] } ], "source": [ @@ -1770,7 +1795,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -1787,7 +1812,7 @@ "text/plain": [ "{'cls': 'SASRecModel',\n", " 'verbose': 0,\n", - " 'data_preparator_type': 'rectools.models.nn.sasrec.SASRecDataPreparator',\n", + " 'data_preparator_type': 'rectools.models.nn.transformers.sasrec.SASRecDataPreparator',\n", " 'n_blocks': 1,\n", " 'n_heads': 1,\n", " 'n_factors': 64,\n", @@ -1805,21 +1830,21 @@ " 'epochs': 2,\n", " 'deterministic': False,\n", " 'recommend_batch_size': 256,\n", - " 'recommend_accelerator': 'auto',\n", - " 'recommend_devices': 1,\n", + " 'recommend_device': None,\n", " 'recommend_n_threads': 0,\n", - " 'recommend_use_gpu_ranking': True,\n", + " 'recommend_use_torch_ranking': True,\n", " 'train_min_user_interactions': 2,\n", " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n", " 'rectools.models.nn.item_net.CatFeaturesItemNet'],\n", - " 'pos_encoding_type': 'rectools.models.nn.transformer_net_blocks.LearnableInversePositionalEncoding',\n", - " 'transformer_layers_type': 'rectools.models.nn.sasrec.SASRecTransformerLayers',\n", - " 'lightning_module_type': 'rectools.models.nn.transformer_base.TransformerLightningModule',\n", + " 'item_net_constructor_type': 'rectools.models.nn.item_net.SumOfEmbeddingsConstructor',\n", + " 'pos_encoding_type': 'rectools.models.nn.transformers.net_blocks.LearnableInversePositionalEncoding',\n", + " 'transformer_layers_type': 'rectools.models.nn.transformers.sasrec.SASRecTransformerLayers',\n", + " 'lightning_module_type': 'rectools.models.nn.transformers.lightning.TransformerLightningModule',\n", " 'get_val_mask_func': None,\n", " 'get_trainer_func': None}" ] }, - "execution_count": 35, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -1851,7 +1876,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1868,7 +1893,7 @@ "text/plain": [ "{'cls': 'SASRecModel',\n", " 'verbose': 0,\n", - " 'data_preparator_type': 'rectools.models.nn.sasrec.SASRecDataPreparator',\n", + " 'data_preparator_type': 'rectools.models.nn.transformers.sasrec.SASRecDataPreparator',\n", " 'n_blocks': 2,\n", " 'n_heads': 4,\n", " 'n_factors': 256,\n", @@ -1886,21 +1911,21 @@ " 'epochs': 3,\n", " 'deterministic': False,\n", " 'recommend_batch_size': 256,\n", - " 'recommend_accelerator': 'auto',\n", - " 'recommend_devices': 1,\n", + " 'recommend_device': None,\n", " 'recommend_n_threads': 0,\n", - " 'recommend_use_gpu_ranking': True,\n", + " 'recommend_use_torch_ranking': True,\n", " 'train_min_user_interactions': 2,\n", " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n", " 'rectools.models.nn.item_net.CatFeaturesItemNet'],\n", - " 'pos_encoding_type': 'rectools.models.nn.transformer_net_blocks.LearnableInversePositionalEncoding',\n", - " 'transformer_layers_type': 'rectools.models.nn.sasrec.SASRecTransformerLayers',\n", - " 'lightning_module_type': 'rectools.models.nn.transformer_base.TransformerLightningModule',\n", + " 'item_net_constructor_type': 'rectools.models.nn.item_net.SumOfEmbeddingsConstructor',\n", + " 'pos_encoding_type': 'rectools.models.nn.transformers.net_blocks.LearnableInversePositionalEncoding',\n", + " 'transformer_layers_type': 'rectools.models.nn.transformers.sasrec.SASRecTransformerLayers',\n", + " 'lightning_module_type': 'rectools.models.nn.transformers.lightning.TransformerLightningModule',\n", " 'get_val_mask_func': '__main__.get_val_mask_func',\n", " 'get_trainer_func': '__main__.get_custom_trainer'}" ] }, - "execution_count": 36, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -1909,7 +1934,8 @@ "config = {\n", " \"get_val_mask_func\": get_val_mask_func, # function to get validation mask\n", " \"get_trainer_func\": get_custom_trainer, # function to get custom trainer\n", - " \"transformer_layers_type\": \"rectools.models.nn.sasrec.SASRecTransformerLayers\", # path to transformer layers class\n", + " # path to transformer layers class:\n", + " \"transformer_layers_type\": \"rectools.models.nn.transformers.sasrec.SASRecTransformerLayers\",\n", "}\n", "\n", "model = SASRecModel.from_config(config)\n", @@ -1944,9 +1970,9 @@ ], "metadata": { "kernelspec": { - "display_name": "rectools-sasrec", + "display_name": "rectools", "language": "python", - "name": "rectools-sasrec" + "name": "rectools" }, "language_info": { "codemirror_mode": { From 65b03fe7a96c1583a34400fe5827fac25a12cb1e Mon Sep 17 00:00:00 2001 From: blondered Date: Fri, 14 Feb 2025 22:55:37 +0300 Subject: [PATCH 06/15] fixed kwargs and docs --- rectools/models/nn/transformers/base.py | 5 +---- rectools/models/nn/transformers/bert4rec.py | 18 ++++++++++++++++-- rectools/models/nn/transformers/constants.py | 3 +-- rectools/models/nn/transformers/sasrec.py | 19 ++++++++++++++++--- tests/models/nn/transformers/test_bert4rec.py | 1 - tests/models/nn/transformers/test_sasrec.py | 1 - 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/rectools/models/nn/transformers/base.py b/rectools/models/nn/transformers/base.py index 04d8b738..283e75fc 100644 --- a/rectools/models/nn/transformers/base.py +++ b/rectools/models/nn/transformers/base.py @@ -31,7 +31,6 @@ from rectools.types import InternalIdsArray from rectools.utils.misc import get_class_or_function_full_path, import_object -from .constants import InitKwargs from ..item_net import ( CatFeaturesItemNet, IdEmbeddingsItemNet, @@ -39,6 +38,7 @@ ItemNetConstructorBase, SumOfEmbeddingsConstructor, ) +from .constants import InitKwargs from .data_preparator import TransformerDataPreparatorBase from .lightning import TransformerLightningModule, TransformerLightningModuleBase from .net_blocks import ( @@ -188,7 +188,6 @@ class TransformerModelConfig(ModelConfig): get_trainer_func: tp.Optional[TrainerCallableSerialized] = None data_preparator_kwargs: tp.Optional[InitKwargs] = None transformer_layers_kwargs: tp.Optional[InitKwargs] = None - item_net_block_kwargs: tp.Optional[InitKwargs] = None item_net_constructor_kwargs: tp.Optional[InitKwargs] = None pos_encoding_kwargs: tp.Optional[InitKwargs] = None lightning_module_kwargs: tp.Optional[InitKwargs] = None @@ -245,7 +244,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals get_trainer_func: tp.Optional[TrainerCallable] = None, data_preparator_kwargs: tp.Optional[InitKwargs] = None, transformer_layers_kwargs: tp.Optional[InitKwargs] = None, - item_net_block_kwargs: tp.Optional[InitKwargs] = None, item_net_constructor_kwargs: tp.Optional[InitKwargs] = None, pos_encoding_kwargs: tp.Optional[InitKwargs] = None, lightning_module_kwargs: tp.Optional[InitKwargs] = None, @@ -283,7 +281,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals self.get_trainer_func = get_trainer_func self.data_preparator_kwargs = data_preparator_kwargs self.transformer_layers_kwargs = transformer_layers_kwargs - self.item_net_block_kwargs = item_net_block_kwargs self.item_net_constructor_kwargs = item_net_constructor_kwargs self.pos_encoding_kwargs = pos_encoding_kwargs self.lightning_module_kwargs = lightning_module_kwargs diff --git a/rectools/models/nn/transformers/bert4rec.py b/rectools/models/nn/transformers/bert4rec.py index 446d300c..0094e40b 100644 --- a/rectools/models/nn/transformers/bert4rec.py +++ b/rectools/models/nn/transformers/bert4rec.py @@ -19,7 +19,6 @@ import numpy as np import torch -from .constants import MASKING_VALUE, PADDING_VALUE, InitKwargs from ..item_net import ( CatFeaturesItemNet, IdEmbeddingsItemNet, @@ -36,7 +35,7 @@ TransformerModelConfig, ValMaskCallable, ) -from .constants import MASKING_VALUE, PADDING_VALUE +from .constants import MASKING_VALUE, PADDING_VALUE, InitKwargs from .data_preparator import TransformerDataPreparatorBase from .net_blocks import ( LearnableInversePositionalEncoding, @@ -279,6 +278,21 @@ class BERT4RecModel(TransformerModelBase[BERT4RecModelConfig]): set to ``True`` (default). If you want to change this parameter after model is initialized, you can manually assign new value to model `recommend_n_threads` attribute. + data_preparator_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `data_preparator_type` initialization. + Make sure all dict values have JSON serializable types. + transformer_layers_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `transformer_layers_type` initialization. + Make sure all dict values have JSON serializable types. + item_net_constructor_kwargs optional(dict), default ``None`` + Additional keyword arguments to pass during `item_net_constructor_type` initialization. + Make sure all dict values have JSON serializable types. + pos_encoding_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `pos_encoding_type` initialization. + Make sure all dict values have JSON serializable types. + lightning_module_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `lightning_module_type` initialization. + Make sure all dict values have JSON serializable types. """ config_class = BERT4RecModelConfig diff --git a/rectools/models/nn/transformers/constants.py b/rectools/models/nn/transformers/constants.py index 5cdf119f..c6aa45c8 100644 --- a/rectools/models/nn/transformers/constants.py +++ b/rectools/models/nn/transformers/constants.py @@ -16,5 +16,4 @@ PADDING_VALUE = "PAD" MASKING_VALUE = "MASK" -SimpleHashable = tp.Union[str, bool, int, float] -InitKwargs = tp.Dict[str, tp.Optional[tp.Union[SimpleHashable, tp.Tuple[SimpleHashable]]]] +InitKwargs = tp.Dict[str, tp.Any] diff --git a/rectools/models/nn/transformers/sasrec.py b/rectools/models/nn/transformers/sasrec.py index 2e60fe1a..d5feb207 100644 --- a/rectools/models/nn/transformers/sasrec.py +++ b/rectools/models/nn/transformers/sasrec.py @@ -36,6 +36,7 @@ TransformerModelConfig, ValMaskCallable, ) +from .constants import InitKwargs from .data_preparator import TransformerDataPreparatorBase from .net_blocks import ( LearnableInversePositionalEncoding, @@ -43,7 +44,6 @@ PositionalEncodingBase, TransformerLayersBase, ) -from .constants import InitKwargs class SASRecDataPreparator(TransformerDataPreparatorBase): @@ -364,6 +364,21 @@ class SASRecModel(TransformerModelBase[SASRecModelConfig]): set to ``True`` (default). If you want to change this parameter after model is initialized, you can manually assign new value to model `recommend_n_threads` attribute. + data_preparator_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `data_preparator_type` initialization. + Make sure all dict values have JSON serializable types. + transformer_layers_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `transformer_layers_type` initialization. + Make sure all dict values have JSON serializable types. + item_net_constructor_kwargs optional(dict), default ``None`` + Additional keyword arguments to pass during `item_net_constructor_type` initialization. + Make sure all dict values have JSON serializable types. + pos_encoding_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `pos_encoding_type` initialization. + Make sure all dict values have JSON serializable types. + lightning_module_kwargs: optional(dict), default ``None`` + Additional keyword arguments to pass during `lightning_module_type` initialization. + Make sure all dict values have JSON serializable types. """ config_class = SASRecModelConfig @@ -402,7 +417,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals recommend_n_threads: int = 0, data_preparator_kwargs: tp.Optional[InitKwargs] = None, transformer_layers_kwargs: tp.Optional[InitKwargs] = None, - item_net_block_kwargs: tp.Optional[InitKwargs] = None, item_net_constructor_kwargs: tp.Optional[InitKwargs] = None, pos_encoding_kwargs: tp.Optional[InitKwargs] = None, lightning_module_kwargs: tp.Optional[InitKwargs] = None, @@ -440,7 +454,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals get_trainer_func=get_trainer_func, data_preparator_kwargs=data_preparator_kwargs, transformer_layers_kwargs=transformer_layers_kwargs, - item_net_block_kwargs=item_net_block_kwargs, item_net_constructor_kwargs=item_net_constructor_kwargs, pos_encoding_kwargs=pos_encoding_kwargs, lightning_module_kwargs=lightning_module_kwargs, diff --git a/tests/models/nn/transformers/test_bert4rec.py b/tests/models/nn/transformers/test_bert4rec.py index ed6e510f..406cc8c8 100644 --- a/tests/models/nn/transformers/test_bert4rec.py +++ b/tests/models/nn/transformers/test_bert4rec.py @@ -759,7 +759,6 @@ def initial_config(self) -> tp.Dict[str, tp.Any]: "get_trainer_func": None, "data_preparator_kwargs": None, "transformer_layers_kwargs": None, - "item_net_block_kwargs": None, "item_net_constructor_kwargs": None, "pos_encoding_kwargs": None, "lightning_module_kwargs": None, diff --git a/tests/models/nn/transformers/test_sasrec.py b/tests/models/nn/transformers/test_sasrec.py index 5d98f55f..6992d0dc 100644 --- a/tests/models/nn/transformers/test_sasrec.py +++ b/tests/models/nn/transformers/test_sasrec.py @@ -925,7 +925,6 @@ def initial_config(self) -> tp.Dict[str, tp.Any]: "get_trainer_func": None, "data_preparator_kwargs": None, "transformer_layers_kwargs": None, - "item_net_block_kwargs": None, "item_net_constructor_kwargs": None, "pos_encoding_kwargs": None, "lightning_module_kwargs": None, From 775268e86d8e24a2b002ec21536bd52c39413dea Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 00:24:06 +0300 Subject: [PATCH 07/15] customization guide finished --- .../transformers_customization_guide.ipynb | 740 ++++++------------ 1 file changed, 244 insertions(+), 496 deletions(-) diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb index 4d70099c..2cbe95e2 100644 --- a/examples/tutorials/transformers_customization_guide.ipynb +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -1,50 +1,76 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Transformer Models Customization Guide\n", + "\n", + "RecTools provides many options to change any part of the model with custom modules: from training objective to special transformer layers logic. Current guide provides just a few examples of the various customizations that can be done.\n", + "\n", + "\n", + "### Table of Contents\n", + "\n", + "* Prepare data\n", + "* \"Next Action\" training objective from Pinnerformer\n", + " - Custom data preparator and lightning module\n", + " - Create `NextActionTransformer`\n", + " - Enable unidirectional attention\n", + "* Albert\n", + " - Custom transformer layers and item net constructor\n", + " - Pass Albert modules to `BERT4RecModel`\n", + " - Pass Albert modules to `SASRecModel`\n", + "* How about `NextActionTransformer` with Albert logic and causal attention?\n", + " - Combining custom modules together\n", + "* Configs support for custom models\n", + "* Full list of customization options" + ] + }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", "import os\n", - "import pandas as pd\n", - "import torch\n", "import typing as tp\n", + "import typing_extensions as tpe\n", "import warnings\n", "from pathlib import Path\n", "\n", "import torch.nn as nn\n", - "import typing_extensions as tpe\n", - "\n", + "import pandas as pd\n", + "import torch\n", + "import numpy as np\n", "from lightning_fabric import seed_everything\n", "from pytorch_lightning import Trainer\n", + "\n", "from rectools import Columns\n", "from rectools.dataset import Dataset\n", "from rectools.models import BERT4RecModel, SASRecModel\n", - "\n", - "\n", - "from rectools.dataset.dataset import Dataset, DatasetSchema\n", + "from rectools.dataset.dataset import DatasetSchema\n", "from rectools.models.nn.item_net import (\n", " ItemNetBase,\n", " SumOfEmbeddingsConstructor,\n", ")\n", - "from rectools.models.nn.transformer_net_blocks import (\n", + "from rectools.models.nn.transformers.net_blocks import (\n", " PreLNTransformerLayer,\n", " TransformerLayersBase,\n", ")\n", - "from rectools.models.nn.constants import InitKwargs\n", + "from rectools.models.nn.transformers.constants import InitKwargs, MASKING_VALUE\n", + "from rectools.models.nn.transformers.bert4rec import BERT4RecDataPreparator\n", + "from rectools.models.nn.transformers.lightning import TransformerLightningModule\n", "\n", "# Enable deterministic behaviour with CUDA >= 10.2\n", "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n", - "warnings.simplefilter(\"ignore\", UserWarning)" + "warnings.simplefilter(\"ignore\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Load data" + "## Prepare data" ] }, { @@ -54,9 +80,9 @@ "outputs": [], "source": [ "# %%time\n", - "# !wget -q https://github.com/irsafilo/KION_DATASET/raw/f69775be31fa5779907cf0a92ddedb70037fb5ae/data_original.zip -O data_original.zip\n", - "# !unzip -o data_original.zip\n", - "# !rm data_original.zip" + "!wget -q https://github.com/irsafilo/KION_DATASET/raw/f69775be31fa5779907cf0a92ddedb70037fb5ae/data_original.zip -O data_original.zip\n", + "!unzip -o data_original.zip\n", + "!rm data_original.zip" ] }, { @@ -72,7 +98,7 @@ " .rename(columns={\"last_watch_dt\": \"datetime\"})\n", ")\n", "interactions[Columns.Weight] = np.where(interactions['watched_pct'] > 10, 3, 1)\n", - "dataset_no_features = Dataset.construct(\n", + "dataset = Dataset.construct(\n", " interactions_df=interactions,\n", ")" ] @@ -108,19 +134,21 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "# Function to get custom trainer for quick debugging\n", + "# Function to get custom trainer\n", "def get_debug_trainer() -> Trainer:\n", " return Trainer(\n", - " accelerator=\"gpu\",\n", - " devices=[1],\n", + " accelerator=\"cpu\",\n", + " devices=1,\n", " min_epochs=1,\n", " max_epochs=1,\n", " deterministic=True,\n", - " limit_train_batches=2,\n", + " enable_model_summary=False,\n", + " enable_progress_bar=False,\n", + " limit_train_batches=2, # limit train batches for quick debug runs\n", " )" ] }, @@ -128,43 +156,38 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# **Training Objective**" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "https://arxiv.org/pdf/2205.04507" + "## \"Next Action\" training objective from Pinnerformer\n", + "[PinnerFormer: Sequence Modeling for User Representation at Pinterest](https://arxiv.org/pdf/2205.04507)\n", + "\n", + "This training objective aims to predict the most recent action for each user. Thus only one target should be taken from each user sequence.\n", + "\n", + "We will take BERT4RecModel as our base class and just change one single detail in data preparation: let's put \"MASK\" token replacing the last position of each user sequence. Everything else will work out of the box.\n", + "\n", + "For computational efficiency we will return `y` and `yw` (and `negatives`) in the shape of `(batch_size, 1)` instead of `(batch_size, session_max_len)`.\n", + "To process this reshaped batch correctly during training we will also rewrite training step in lightning module.\n", + "\n", + "We could have filled `y` and `yw` with zeros except for the last target item. This way trainig step should have been left unchanged. But it's less efficient." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## **Next Action**" + "### Custom data preparator and lightning module" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "from typing import Dict, List, Tuple\n", - "\n", - "from rectools.models.nn.bert4rec import BERT4RecDataPreparator\n", - "from rectools.models.nn.constants import MASKING_VALUE\n", - "from rectools.models.nn.transformer_lightning import TransformerLightningModule\n", - "\n", - "\n", - "# \"MASK\" token is used for predicting one next token.\n", - "class NextItemDataPreparator(BERT4RecDataPreparator):\n", + "class NextActionDataPreparator(BERT4RecDataPreparator):\n", " \n", " def _collate_fn_train(\n", " self,\n", - " batch: List[Tuple[List[int], List[float]]],\n", - " ) -> Dict[str, torch.Tensor]:\n", + " batch: tp.List[tp.Tuple[tp.List[int], tp.List[float]]],\n", + " ) -> tp.Dict[str, torch.Tensor]:\n", " \"\"\"\n", " Truncate each session from right to keep `session_max_len` items.\n", " Do left padding until `session_max_len` is reached.\n", @@ -176,10 +199,10 @@ " yw = np.zeros((batch_size, 1))\n", " for i, (ses, ses_weights) in enumerate(batch):\n", " session = ses.copy()\n", - " session[-1] = self.extra_token_ids[MASKING_VALUE]\n", - " x[i, -len(ses) :] = session # ses: [session_len] -> x[i]: [session_max_len]\n", - " y[i] = ses[-1] # ses: [session_len] -> y[i]: [1]\n", - " yw[i] = ses_weights[-1] # ses_weights: [session_len] -> yw[i]: [1]\n", + " session[-1] = self.extra_token_ids[MASKING_VALUE] # Replace last token with \"MASK\"\n", + " x[i, -len(ses) :] = session\n", + " y[i] = ses[-1]\n", + " yw[i] = ses_weights[-1]\n", "\n", " batch_dict = {\"x\": torch.LongTensor(x), \"y\": torch.LongTensor(y), \"yw\": torch.FloatTensor(yw)}\n", " if self.n_negatives is not None:\n", @@ -187,28 +210,26 @@ " low=self.n_item_extra_tokens,\n", " high=self.item_id_map.size,\n", " size=(batch_size, 1, self.n_negatives),\n", - " ) # [batch_size, 1, n_negatives]\n", + " )\n", " batch_dict[\"negatives\"] = negatives\n", " return batch_dict\n", "\n", "\n", - "# Last logits are used for reducing the number of calculations on training step.\n", - "# You could also fill the y with zeros except for the last item in `_collate_fn_train`` and not change the training step \n", - "class NextItemLightningModule(TransformerLightningModule):\n", + "class NextActionLightningModule(TransformerLightningModule):\n", "\n", " def training_step(self, batch: tp.Dict[str, torch.Tensor], batch_idx: int) -> torch.Tensor:\n", " \"\"\"Training step.\"\"\"\n", " x, y, w = batch[\"x\"], batch[\"y\"], batch[\"yw\"]\n", " if self.loss == \"softmax\":\n", - " logits = self._get_full_catalog_logits(x)[:, -1: :]\n", + " logits = self._get_full_catalog_logits(x)[:, -1: :] # take only token last hidden state\n", " loss = self._calc_softmax_loss(logits, y, w)\n", " elif self.loss == \"BCE\":\n", " negatives = batch[\"negatives\"]\n", - " logits = self._get_pos_neg_logits(x, y, negatives)[:, -1: :]\n", + " logits = self._get_pos_neg_logits(x, y, negatives)[:, -1: :] # take only last token hidden state\n", " loss = self._calc_bce_loss(logits, y, w)\n", " elif self.loss == \"gBCE\":\n", " negatives = batch[\"negatives\"]\n", - " logits = self._get_pos_neg_logits(x, y, negatives)[:, -1: :]\n", + " logits = self._get_pos_neg_logits(x, y, negatives)[:, -1: :] # take only last token hidden state\n", " loss = self._calc_gbce_loss(logits, y, w, negatives)\n", " else:\n", " loss = self._calc_custom_loss(batch, batch_idx)\n", @@ -219,195 +240,120 @@ ] }, { - "cell_type": "code", - "execution_count": 7, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "GPU available: True (cuda), used: True\n", - "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n", - "GPU available: True (cuda), used: True\n", - "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n" - ] - } - ], "source": [ - "nextitem_transformer_bidirectional = BERT4RecModel(\n", - " data_preparator_type=NextItemDataPreparator, # \"NextItem\" training objective data preparator\n", - " lightning_module_type=NextItemLightningModule, # \"NextItem\" lightning module\n", - " get_trainer_func = get_debug_trainer,\n", - ")\n", - "\n", - "nextitem_transformer_unidirectional = SASRecModel(\n", - " data_preparator_type=NextItemDataPreparator, # \"NextItem\" training objective data preparator\n", - " lightning_module_type=NextItemLightningModule, # \"NextItem\" lightning module\n", - " use_causal_attn=True, # Apply causal attention mask\n", - " get_trainer_func = get_debug_trainer,\n", - ")" + "### Create `NextActionTransformer`" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 5.5 M | train\n", - "-----------------------------------------------------------------\n", - "5.5 M Trainable params\n", - "0 Non-trainable params\n", - "5.5 M Total params\n", - "22.040 Total estimated model params size (MB)\n", - "37 Modules in train mode\n", - "0 Modules in eval mode\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "49d0297542c7483dba27e03cd4ed2e84", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" + "" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "nextitem_transformer_bidirectional.fit(dataset_no_features)" + "next_action_model = BERT4RecModel(\n", + " data_preparator_type=NextActionDataPreparator,\n", + " lightning_module_type=NextActionLightningModule,\n", + " get_trainer_func = get_debug_trainer,\n", + ")\n", + "\n", + "next_action_model.fit(dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Enable unidirectional attention" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 4.7 M | train\n", - "-----------------------------------------------------------------\n", - "4.7 M Trainable params\n", - "0 Non-trainable params\n", - "4.7 M Total params\n", - "18.890 Total estimated model params size (MB)\n", - "34 Modules in train mode\n", - "0 Modules in eval mode\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4ef281146a32431bb36965190283983d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" + "" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "nextitem_transformer_unidirectional.fit(dataset_no_features)" + "next_action_model_causal = BERT4RecModel(\n", + " data_preparator_type=NextActionDataPreparator,\n", + " lightning_module_type=NextActionLightningModule,\n", + " get_trainer_func = get_debug_trainer,\n", + " use_causal_attn = True, # simple flag\n", + ")\n", + "\n", + "next_action_model_causal.fit(dataset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# ALBERT\n", - "Albert has 2 main innovations which can be used together or separately:\n", - "1. Learning embeddings of smaller size and then projecting them to the required size through a Liner projection\n", - "2. Sharing weights between transformer layers" + "## ALBERT\n", + "[ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942)\n", + "\n", + "Albert has two parameter-reduction techniques to lower memory consumption and increase the training speed which can actually be used together or separately:\n", + "1. Learning embeddings of smaller size and then projecting them to the required size through a Liner projection (\"Factorized embedding parameterization\")\n", + "2. Sharing weights between transformer layers (\"Cross-layer parameter sharing\")\n", + "\n", + "We will implement both techiques in custom classes for transformer layers and for item net constructor." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Custom item net constructor and transformer layers" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "# ### ---------- Special Albert logic for Embeddings ---------- ### #\n", + "# Special Albert logic for Embeddings - Factorized embedding parameterization\n", "\n", "class AlBERT4RecSumOfEmbeddingsConstructor(SumOfEmbeddingsConstructor):\n", "\n", @@ -416,13 +362,13 @@ " n_items: int,\n", " n_factors: int,\n", " item_net_blocks: tp.Sequence[ItemNetBase],\n", - " emb_factors: int = 16, # accept new kwargs\n", + " emb_factors: int = 16, # accept new kwarg for lower dimensional space size\n", " ) -> None:\n", " super().__init__(\n", " n_items=n_items,\n", " item_net_blocks=item_net_blocks,\n", " )\n", - " self.item_emb_proj = nn.Linear(emb_factors, n_factors)\n", + " self.item_emb_proj = nn.Linear(emb_factors, n_factors) # Project to actual required hidden space\n", "\n", " @classmethod\n", " def from_dataset(\n", @@ -431,12 +377,13 @@ " n_factors: int,\n", " dropout_rate: float,\n", " item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]],\n", - " emb_factors: int, # accept new kwargs\n", + " emb_factors: int, # accept new kwarg for lower dimensional space size\n", " ) -> tpe.Self:\n", " n_items = dataset.item_id_map.size\n", "\n", " item_net_blocks: tp.List[ItemNetBase] = []\n", " for item_net in item_net_block_types:\n", + " # Item net blocks will work in lower dimensional space\n", " item_net_block = item_net.from_dataset(dataset, emb_factors, dropout_rate)\n", " if item_net_block is not None:\n", " item_net_blocks.append(item_net_block)\n", @@ -450,7 +397,7 @@ " n_factors: int,\n", " dropout_rate: float,\n", " item_net_block_types: tp.Sequence[tp.Type[ItemNetBase]],\n", - " emb_factors: int, # accept new kwargs\n", + " emb_factors: int, # accept new kwarg for lower dimensional space size\n", " ) -> tpe.Self:\n", " n_items = dataset_schema.items.n_hot\n", "\n", @@ -463,18 +410,18 @@ " return cls(n_items, n_factors, item_net_blocks, emb_factors)\n", "\n", " def forward(self, items: torch.Tensor) -> torch.Tensor:\n", - " item_embs = super().forward(items)\n", - " item_embs = self.item_emb_proj(item_embs)\n", + " item_embs = super().forward(items) # Create embeddings in lower dimensional space\n", + " item_embs = self.item_emb_proj(item_embs) # Project to actual required hidden space\n", " return item_embs" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "# ### ---------- Special Albert logic for Transfromer Layers ---------- ### #\n", + "# Special Albert logic for Transformer Layers - Cross-layer parameter sharing\n", " \n", "class AlBERT4RecPreLNTransformerLayers(TransformerLayersBase):\n", "\n", @@ -527,336 +474,164 @@ ] }, { - "cell_type": "code", - "execution_count": 30, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS = { # these arguments are obligatory for our custom classes\n", - " \"emb_factors\": 32,\n", - "}\n", - "\n", - "ALBERT_TRANSFORMER_LAYERS_KWARGS = { # these arguments are obligatory for our custom classes\n", - " \"n_hidden_groups\": 2,\n", - " \"n_inner_groups\": 1,\n", - "}" + "### Pass Albert modules to `BERT4RecModel`" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "GPU available: True (cuda), used: True\n", + "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n" - ] - } - ], - "source": [ - "albert_model = BERT4RecModel(\n", - " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", - " item_net_constructor_kwargs=ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", - " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " transformer_layers_kwargs=ALBERT_TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", - " get_trainer_func = get_debug_trainer,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", - "-----------------------------------------------------------------\n", - "2.1 M Trainable params\n", - "0 Non-trainable params\n", - "2.1 M Total params\n", - "8.407 Total estimated model params size (MB)\n", - "38 Modules in train mode\n", - "0 Modules in eval mode\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "aebe81c85dbb482180d21be64d4d7411", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" + "" ] }, - "execution_count": 32, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "albert_model.fit(dataset_no_features)" + "CONSTRUCTOR_KWARGS = {\n", + " \"emb_factors\": 32,\n", + "}\n", + "\n", + "TRANSFORMER_LAYERS_KWARGS = {\n", + " \"n_hidden_groups\": 2,\n", + " \"n_inner_groups\": 1,\n", + "}\n", + "\n", + "albert_model = BERT4RecModel(\n", + " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", + " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", + " get_trainer_func = get_debug_trainer,\n", + ")\n", + "albert_model.fit(dataset)" ] }, { - "cell_type": "code", - "execution_count": 33, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "GPU available: True (cuda), used: True\n", - "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n" - ] - } - ], "source": [ - "alsasrec = SASRecModel(\n", - " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", - " item_net_constructor_kwargs=ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", - " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " transformer_layers_kwargs=ALBERT_TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", - " get_trainer_func = get_debug_trainer,\n", - ")" + "### Pass Albert modules to `SASRecModel`" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", - "-----------------------------------------------------------------\n", - "2.1 M Trainable params\n", - "0 Non-trainable params\n", - "2.1 M Total params\n", - "8.407 Total estimated model params size (MB)\n", - "38 Modules in train mode\n", - "0 Modules in eval mode\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0be15dcd434342a0b73efe8153e18774", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" + "" ] }, - "execution_count": 34, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "alsasrec.fit(dataset_no_features)" + "alsasrec_model = SASRecModel(\n", + " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", + " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", + " get_trainer_func = get_debug_trainer,\n", + ")\n", + "alsasrec_model.fit(dataset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# How about NextTokenTransformer with Albert logic and causal attention?\n", - "# Just because we can!" + "## How about `NextActionTransformer` with Albert logic and causal attention?\n", + "Just because we can!\n", + "### Combining custom modules together" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "GPU available: True (cuda), used: True\n", + "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n" - ] - } - ], - "source": [ - "next_action_albert_causal = BERT4RecModel(\n", - " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", - " item_net_constructor_kwargs=ALBERT_ITEM_NET_CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", - " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " transformer_layers_kwargs=ALBERT_TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", - " data_preparator_type=NextItemDataPreparator, # custom data preparator\n", - " lightning_module_type=NextItemLightningModule, # custom lightning module\n", - " use_causal_attn=True, # Apply causal attention mask\n", - " get_trainer_func = get_debug_trainer,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", - "-----------------------------------------------------------------\n", - "2.1 M Trainable params\n", - "0 Non-trainable params\n", - "2.1 M Total params\n", - "8.407 Total estimated model params size (MB)\n", - "38 Modules in train mode\n", - "0 Modules in eval mode\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ab2068caad4b406d86f674af208d2a53", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" + "" ] }, - "execution_count": 37, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "next_action_albert_causal.fit(dataset_no_features)" + "next_action_albert_causal = BERT4RecModel(\n", + " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", + " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", + " data_preparator_type=NextActionDataPreparator, # custom data preparator\n", + " lightning_module_type=NextActionLightningModule, # custom lightning module\n", + " use_causal_attn=True, # Apply causal attention mask\n", + " get_trainer_func = get_debug_trainer,\n", + ")\n", + "next_action_albert_causal.fit(dataset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## What about configs?" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "params = next_action_albert_causal.get_params(simple_types=True)" + "## Configs support for custom models\n", + "All custom models fully support initialization from configs and other RecTools benefits. For models with keyword arguments we suggest to use `from_params` method that accepts configs in a flat dict form. See example below:\n", + "\n", + "**Important: only JSON serializable custom keyword argument values are accepted during customization**" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -864,7 +639,7 @@ "text/plain": [ "{'cls': 'BERT4RecModel',\n", " 'verbose': 0,\n", - " 'data_preparator_type': '__main__.NextItemDataPreparator',\n", + " 'data_preparator_type': '__main__.NextActionDataPreparator',\n", " 'n_blocks': 2,\n", " 'n_heads': 4,\n", " 'n_factors': 256,\n", @@ -889,117 +664,90 @@ " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n", " 'rectools.models.nn.item_net.CatFeaturesItemNet'],\n", " 'item_net_constructor_type': '__main__.AlBERT4RecSumOfEmbeddingsConstructor',\n", - " 'pos_encoding_type': 'rectools.models.nn.transformer_net_blocks.LearnableInversePositionalEncoding',\n", + " 'pos_encoding_type': 'rectools.models.nn.transformers.net_blocks.LearnableInversePositionalEncoding',\n", " 'transformer_layers_type': '__main__.AlBERT4RecPreLNTransformerLayers',\n", - " 'lightning_module_type': '__main__.NextItemLightningModule',\n", + " 'lightning_module_type': '__main__.NextActionLightningModule',\n", " 'get_val_mask_func': None,\n", " 'get_trainer_func': '__main__.get_debug_trainer',\n", " 'data_preparator_kwargs': None,\n", " 'transformer_layers_kwargs.n_hidden_groups': 2,\n", " 'transformer_layers_kwargs.n_inner_groups': 1,\n", - " 'item_net_block_kwargs': None,\n", " 'item_net_constructor_kwargs.emb_factors': 32,\n", " 'pos_encoding_kwargs': None,\n", " 'lightning_module_kwargs': None,\n", " 'mask_prob': 0.15}" ] }, - "execution_count": 39, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "params" + "params = next_action_albert_causal.get_params(simple_types=True)\n", + "params\n", + "# params include \"transformer_layers_kwargs.n_hidden_groups\", \n", + "# \"transformer_layers_kwargs.n_inner_groups\"\n", + "# and \"item_net_constructor_kwargs.emb_factors\"" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "GPU available: True (cuda), used: True\n", + "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n" - ] - } - ], - "source": [ - "model = BERT4RecModel.from_params(params)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/data/home/dmtikhono1/RecTools/rectools/dataset/identifiers.py:60: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n", - " unq_values = pd.unique(values)\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", - "\n", - " | Name | Type | Params | Mode \n", - "-----------------------------------------------------------------\n", - "0 | torch_model | TransformerTorchBackbone | 2.1 M | train\n", - "-----------------------------------------------------------------\n", - "2.1 M Trainable params\n", - "0 Non-trainable params\n", - "2.1 M Total params\n", - "8.407 Total estimated model params size (MB)\n", - "38 Modules in train mode\n", - "0 Modules in eval mode\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "cb5a9bb492a04e1fa58ea29faa47d69f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Training: | | 0/? [00:00" + "" ] }, - "execution_count": 41, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "model.fit(dataset_no_features)" + "model = BERT4RecModel.from_params(params)\n", + "model.fit(dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Full list of customization options\n", + "\n", + "These blocks of RecTools transfofmer models can be replaced with custom classes (WITH an option to add required keyword arguments for initialization):\n", + "- data preparator (`data_preparator_type`, `data_preparator_kwargs`)\n", + " - forming training objectives\n", + " - providing train, val and recommend dataloaders preparation\n", + "- lightning module (`lightning_module_type`, `lightning_module_kwargs`)\n", + " - tying of user session latent represenation and candidate embeddings\n", + " - training, validation and recommending logic\n", + " - losses computation\n", + " - weights initialization\n", + " - optimizer configuration\n", + "- item net constructor (`item_net_constructor_type`, `item_net_constructor_kwargs`)\n", + " - way for aggregating outputs from item net blocks\n", + "- transformer layers (`transformer_layers_type`, `transformer_layers_kwargs`)\n", + "- positional encoding (`pos_encoding_type`, `pos_encoding_kwargs`)\n", + "\n", + "These blocks of RecTools transfofmer models can be replaced with custom classes (WITHOUT an option to add keyword arguments):\n", + "- item net blocks (`item_net_block_types`)\n", + "\n", + "These keyword model arguments have great effect on model architecture:\n", + "- `use_causal_attn` (applies unidirectional attention instead of bidirectional when set to ``True``)" ] } ], From 406d390316dcef9736453e60149ff26eff7cfad8 Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 12:00:18 +0300 Subject: [PATCH 08/15] data preparator back --- rectools/models/nn/transformers/base.py | 1 + rectools/models/nn/transformers/lightning.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rectools/models/nn/transformers/base.py b/rectools/models/nn/transformers/base.py index 283e75fc..2bcb5673 100644 --- a/rectools/models/nn/transformers/base.py +++ b/rectools/models/nn/transformers/base.py @@ -386,6 +386,7 @@ def _init_lightning_model( dataset_schema=dataset_schema, item_external_ids=item_external_ids, item_extra_tokens=self.data_preparator.item_extra_tokens, + data_preparator=self.data_preparator, model_config=model_config, lr=self.lr, loss=self.loss, diff --git a/rectools/models/nn/transformers/lightning.py b/rectools/models/nn/transformers/lightning.py index c2e5f6fd..25b360a0 100644 --- a/rectools/models/nn/transformers/lightning.py +++ b/rectools/models/nn/transformers/lightning.py @@ -26,6 +26,7 @@ from rectools.models.rank import Distance, ImplicitRanker, Ranker, TorchRanker from rectools.types import InternalIdsArray +from .data_preparator import TransformerDataPreparatorBase from .torch_backbone import TransformerTorchBackbone # #### -------------- Lightning Base Model -------------- #### # @@ -63,6 +64,7 @@ def __init__( dataset_schema: DatasetSchemaDict, item_external_ids: ExternalIds, item_extra_tokens: tp.Sequence[Hashable], + data_preparator: TransformerDataPreparatorBase, lr: float, gbce_t: float, loss: str, @@ -78,6 +80,7 @@ def __init__( self.dataset_schema = dataset_schema self.item_external_ids = item_external_ids self.item_extra_tokens = item_extra_tokens + self.data_preparator = data_preparator self.lr = lr self.loss = loss self.adam_betas = adam_betas @@ -87,7 +90,7 @@ def __init__( self.val_loss_name = val_loss_name self.item_embs: torch.Tensor - self.save_hyperparameters(ignore=["torch_model"]) + self.save_hyperparameters(ignore=["torch_model", "data_preparator"]) def configure_optimizers(self) -> torch.optim.Adam: """Choose what optimizers and learning-rate schedulers to use in optimization""" From f0aa7b258472f29aa805ed8575c18e68124efd79 Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 12:00:30 +0300 Subject: [PATCH 09/15] readme --- README.md | 8 ++++---- docs/source/tutorials.rst | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9b63eb1f..740630fe 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ faster than ever before. **BERT4Rec and SASRec are now available in RecTools:** - Fully compatible with our `fit` / `recommend` paradigm and require NO special data processing - Explicitly described in our [Transformers Theory & Practice Tutorial](examples/tutorials/transformers_tutorial.ipynb): loss options, item embedding options, category features utilization and more! -- Configurable, customizable, callback-friendly, checkpoints-included, logs-out-of-the-box, custom-validation-ready, multi-gpu-compatible! See our [Transformers Advanced Training User Guide](examples/tutorials/transformers_advanced_training_guide.ipynb) -- Public benchmarks which compare of RecTools models to other open-source implementations following BERT4Rec replicability paper show that RecTools implementations achieve highest scores on multiple datasets: [Performance on public transformers benchmarks](https://github.com/blondered/bert4rec_repro?tab=readme-ov-file#rectools-transformers-benchmark-results) +- Configurable, customizable, callback-friendly, checkpoints-included, logs-out-of-the-box, custom-validation-ready, multi-gpu-compatible! See [Transformers Advanced Training User Guide](examples/tutorials/transformers_advanced_training_guide.ipynb) and [Transformers Customization Guide](examples/tutorials/transformers_customization_guide.ipynb) +- Public benchmarks which compare RecTools models to other open-source implementations following BERT4Rec replicability paper show that RecTools implementations achieve highest scores on multiple datasets: [Performance on public transformers benchmarks](https://github.com/blondered/bert4rec_repro?tab=readme-ov-file#rectools-transformers-benchmark-results) @@ -109,8 +109,8 @@ See [recommender baselines extended tutorial](https://github.com/MobileTeleSyste | Model | Type | Description (🎏 for user/item features, 🔆 for warm inference, ❄️ for cold inference support) | Tutorials & Benchmarks | |----|----|---------|--------| -| SASRec | Neural Network | `rectools.models.SASRecModel` - Transformer-based sequential model with unidirectional attention mechanism and "Shifted Sequence" training objective
🎏| 📕 [Transformers Theory & Practice](examples/tutorials/transformers_tutorial.ipynb)
📗 [Transformers advanced training](examples/tutorials/transformers_advanced_training_guide.ipynb)
🚀 [Top performance on public benchmarks](https://github.com/blondered/bert4rec_repro?tab=readme-ov-file#rectools-transformers-benchmark-results) | -| BERT4Rec | Neural Network | `rectools.models.BERT4RecModel` - Transformer-based sequential model with bidirectional attention mechanism and "MLM" (masked item) training objective
🎏| 📕 [Transformers Theory & Practice](examples/tutorials/transformers_tutorial.ipynb)
📗 [Transformers advanced training](examples/tutorials/transformers_advanced_training_guide.ipynb)
🚀 [Top performance on public benchmarks](https://github.com/blondered/bert4rec_repro?tab=readme-ov-file#rectools-transformers-benchmark-results) | +| SASRec | Neural Network | `rectools.models.SASRecModel` - Transformer-based sequential model with unidirectional attention mechanism and "Shifted Sequence" training objective
🎏| 📕 [Transformers Theory & Practice](examples/tutorials/transformers_tutorial.ipynb)
📗 [Advanced training guide](examples/tutorials/transformers_advanced_training_guide.ipynb)
📘 [Customization guide](examples/tutorials/transformers_customization_guide.ipynb)
🚀 [Top performance on public benchmarks](https://github.com/blondered/bert4rec_repro?tab=readme-ov-file#rectools-transformers-benchmark-results) | +| BERT4Rec | Neural Network | `rectools.models.BERT4RecModel` - Transformer-based sequential model with bidirectional attention mechanism and "MLM" (masked item) training objective
🎏| 📕 [Transformers Theory & Practice](examples/tutorials/transformers_tutorial.ipynb)
📗 [Advanced training guide](examples/tutorials/transformers_advanced_training_guide.ipynb)
📘 [Customization guide](examples/tutorials/transformers_customization_guide.ipynb)
🚀 [Top performance on public benchmarks](https://github.com/blondered/bert4rec_repro?tab=readme-ov-file#rectools-transformers-benchmark-results) | | [implicit](https://github.com/benfred/implicit) ALS Wrapper | Matrix Factorization | `rectools.models.ImplicitALSWrapperModel` - Alternating Least Squares Matrix Factorizattion algorithm for implicit feedback.
🎏| 📙 [Theory & Practice](https://rectools.readthedocs.io/en/latest/examples/tutorials/baselines_extended_tutorial.html#Implicit-ALS)
🚀 [50% boost to metrics with user & item features](examples/5_benchmark_iALS_with_features.ipynb) | | [implicit](https://github.com/benfred/implicit) BPR-MF Wrapper | Matrix Factorization | `rectools.models.ImplicitBPRWrapperModel` - Bayesian Personalized Ranking Matrix Factorization algorithm. | 📙 [Theory & Practice](https://rectools.readthedocs.io/en/latest/examples/tutorials/baselines_extended_tutorial.html#Bayesian-Personalized-Ranking-Matrix-Factorization-(BPR-MF)) | | [implicit](https://github.com/benfred/implicit) ItemKNN Wrapper | Nearest Neighbours | `rectools.models.ImplicitItemKNNWrapperModel` - Algorithm that calculates item-item similarity matrix using distances between item vectors in user-item interactions matrix | 📙 [Theory & Practice](https://rectools.readthedocs.io/en/latest/examples/tutorials/baselines_extended_tutorial.html#ItemKNN) | diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index 1e85dca3..d7a30ee5 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -10,3 +10,4 @@ See tutorials here: https://github.com/MobileTeleSystems/RecTools/tree/main/exam examples/tutorials/baselines_extended_tutorial examples/tutorials/transformers_tutorial examples/tutorials/transformers_advanced_training_guide + examples/tutorials/transformers_customization_guide From 761051e20c0b71a391959d3d9d15c777d93043de Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 12:00:41 +0300 Subject: [PATCH 10/15] guids up --- ...transformers_advanced_training_guide.ipynb | 467 +++++++++++------- .../transformers_customization_guide.ipynb | 65 +-- 2 files changed, 312 insertions(+), 220 deletions(-) diff --git a/examples/tutorials/transformers_advanced_training_guide.ipynb b/examples/tutorials/transformers_advanced_training_guide.ipynb index 3354bf89..7b98bdf2 100644 --- a/examples/tutorials/transformers_advanced_training_guide.ipynb +++ b/examples/tutorials/transformers_advanced_training_guide.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -147,7 +147,7 @@ "1 699317 1659 2021-05-29 8317 100.0" ] }, - "execution_count": 6, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -167,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -176,7 +176,7 @@ "(962179, 15706)" ] }, - "execution_count": 7, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -187,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -210,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -226,7 +226,7 @@ "60" ] }, - "execution_count": 9, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -261,7 +261,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -289,6 +289,36 @@ " return leave_one_out_mask_for_users(interactions, val_users = VAL_USERS)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this guide we are going to use custom Lighhning trainers. We need to implement function that return desired Lightining trainer and pass it to model during initialization." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Function to get custom trainer\n", + "\n", + "def get_debug_trainer() -> Trainer:\n", + " return Trainer(\n", + " accelerator=\"gpu\",\n", + " devices=1,\n", + " min_epochs=2,\n", + " max_epochs=2,\n", + " deterministic=True,\n", + " enable_model_summary=False,\n", + " enable_progress_bar=False,\n", + " enable_checkpointing=False,\n", + " limit_train_batches=2, # limit train batches for quick debug runs\n", + " logger = CSVLogger(\"test_logs\"), # We use CSV logging for this guide but there are many other options\n", + " )" + ] + }, { "cell_type": "code", "execution_count": 8, @@ -316,6 +346,7 @@ " deterministic=True,\n", " item_net_block_types=(IdEmbeddingsItemNet,),\n", " get_val_mask_func=get_val_mask_func, # pass our custom `get_val_mask_func`\n", + " get_trainer_func=get_debug_trainer, # pass our custom trainer func\n", ")" ] }, @@ -325,46 +356,24 @@ "source": [ "### Validation loss\n", "\n", - "Let's check how the validation loss is being logged.\n", - "We just want to quickly check functionality for now so let's create a custom Lightning trainer and use it replace the default one.\n", - "\n", - "Right now we will just assign new trainer to model `_trainer` attribute but later in this tutorial a clean way for passing custom trainer will be shown." + "Let's check how the validation loss is being logged." ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "GPU available: True (cuda), used: True\n", - "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", "`Trainer.fit` stopped: `max_epochs=2` reached.\n" ] } ], "source": [ - "trainer = Trainer(\n", - " accelerator='gpu',\n", - " devices=1,\n", - " min_epochs=2,\n", - " max_epochs=2, \n", - " deterministic=True,\n", - " limit_train_batches=2, # use only 2 batches for each epoch for a test run\n", - " enable_checkpointing=False,\n", - " logger = CSVLogger(\"test_logs\"), # We use CSV logging for this guide but there are many other options\n", - " enable_progress_bar=False,\n", - " enable_model_summary=False,\n", - ")\n", - "\n", - "# Replace default trainer with our custom one\n", - "model._trainer = trainer\n", - "\n", "# Fit model. Validation fold and validation loss computation will be done under the hood.\n", "model.fit(dataset);" ] @@ -378,7 +387,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -396,22 +405,22 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "epoch,step,train_loss,val_loss\r\n", + "epoch,step,train_loss,val_loss\r", "\r\n", - "0,1,,22.39907455444336\r\n", + "0,1,,22.365339279174805\r", "\r\n", - "0,1,22.390357971191406,\r\n", + "0,1,22.38391876220703,\r", "\r\n", - "1,3,,22.25874137878418\r\n", + "1,3,,22.189851760864258\r", "\r\n", - "1,3,22.909526824951172,\r\n", + "1,3,22.898216247558594,\r", "\r\n" ] } @@ -430,26 +439,14 @@ "### Callback for Early Stopping\n", "\n", "By default RecTools transfomers train for exact amount of epochs (specified in `epochs` argument).\n", + "When `get_trainer_func` is provided, number of model training epochs depends on Lightning trainer arguments instead.\n", "\n", - "But now that we have validation loss logged, let's use it for model Early Stopping. It will ensure that model will not resume training if validation loss (or any other custom metric) doesn't impove. We have Lightning Callbacks for that." + "Now that we have validation loss logged, let's use it for model Early Stopping. It will ensure that model will not resume training if validation loss (or any other custom metric) doesn't impove. We have Lightning Callbacks for that." ] }, { "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "early_stopping_callback = EarlyStopping(\n", - " monitor=SASRecModel.val_loss_name, # or just pass \"val_loss\" here\n", - " mode=\"min\",\n", - " min_delta=1. # just for a quick test of functionality\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -458,12 +455,17 @@ "text": [ "GPU available: True (cuda), used: True\n", "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n" + "HPU available: False, using: 0 HPUs\n" ] } ], "source": [ + "early_stopping_callback = EarlyStopping(\n", + " monitor=SASRecModel.val_loss_name, # or just pass \"val_loss\" here\n", + " mode=\"min\",\n", + " min_delta=1. # just for a quick test of functionality\n", + ")\n", + "\n", "trainer = Trainer(\n", " accelerator='gpu',\n", " devices=1,\n", @@ -476,9 +478,32 @@ " callbacks=early_stopping_callback, # pass our callback\n", " enable_progress_bar=False,\n", " enable_model_summary=False,\n", - ")\n", - "\n", - "# Replace default trainer with our custom one\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We want to pass our new trainer to model. \n", + "We just want to quickly check functionality for now and we already have model initialized. So let's just assign new trainer to model `_trainer` attribute." + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n" + ] + } + ], + "source": [ + "# Replace trainer with our custom one\n", "model._trainer = trainer\n", "\n", "# Fit model. Everything will happen under the hood\n", @@ -494,30 +519,30 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "epoch,step,train_loss,val_loss\r\n", + "epoch,step,train_loss,val_loss\r", "\r\n", - "0,1,,22.363222122192383\r\n", + "0,1,,22.343637466430664\r", "\r\n", - "0,1,22.359580993652344,\r\n", + "0,1,22.36273765563965,\r", "\r\n", - "1,3,,22.194488525390625\r\n", + "1,3,,22.159835815429688\r", "\r\n", - "1,3,22.31987190246582,\r\n", + "1,3,22.33755874633789,\r", "\r\n", - "2,5,,21.974754333496094\r\n", + "2,5,,21.94308853149414\r", "\r\n", - "2,5,22.225738525390625,\r\n", + "2,5,22.244243621826172,\r", "\r\n", - "3,7,,21.718231201171875\r\n", + "3,7,,21.702259063720703\r", "\r\n", - "3,7,22.150163650512695,\r\n", + "3,7,22.196012496948242,\r", "\r\n" ] } @@ -537,7 +562,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -555,7 +580,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -584,7 +609,7 @@ " enable_model_summary=False,\n", ")\n", "\n", - "# Replace default trainer with our custom one\n", + "# Replace trainer with our custom one\n", "model._trainer = trainer\n", "\n", "# Fit model. Everything will happen under the hood\n", @@ -600,14 +625,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 42, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "epoch=4-val_loss=21.53.ckpt epoch=5-val_loss=21.22.ckpt last_epoch.ckpt\r\n" + "epoch=4-val_loss=21.52.ckpt epoch=5-val_loss=21.24.ckpt last_epoch.ckpt\r\n" ] } ], @@ -627,13 +652,112 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Loading checkpoints is very simple with `load_from_checkpoint` method.\n", + "Loading checkpoints is very simple with `load_from_weights_from_checkpoint` method." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_iditem_idscorerank
0176549152970.6759641
117654926570.6614442
2176549104400.5629423
317654944950.5572084
417654964430.5461085
\n", + "
" + ], + "text/plain": [ + " user_id item_id score rank\n", + "0 176549 15297 0.675964 1\n", + "1 176549 2657 0.661444 2\n", + "2 176549 10440 0.562942 3\n", + "3 176549 4495 0.557208 4\n", + "4 176549 6443 0.546108 5" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ckpt_path = os.path.join(model.fit_trainer.log_dir, \"checkpoints\", \"last_epoch.ckpt\")\n", + "model.load_weights_from_checkpoint(ckpt_path)\n", + "model.recommend(users=VAL_USERS[:1], dataset=dataset, filter_viewed=True, k=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also load both model and its weights from checkpoint using `load_from_checkpoint` class method.\n", "Note that there is an important limitation: **loaded model will not have `fit_trainer` and can't be saved again. But it is fully ready for recommendations.**" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 45, "metadata": {}, "outputs": [ { @@ -642,27 +766,9 @@ "text": [ "GPU available: True (cuda), used: True\n", "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n", - "GPU available: True (cuda), used: True\n", - "TPU available: False, using: 0 TPU cores\n", - "HPU available: False, using: 0 HPUs\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n" + "HPU available: False, using: 0 HPUs\n" ] }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c85988c886f245ed8573b00a92e6260c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Predicting: | | 0/? [00:000\n", " 176549\n", " 15297\n", - " 0.646373\n", + " 0.675964\n", " 1\n", " \n", " \n", " 1\n", " 176549\n", - " 8636\n", - " 0.609171\n", + " 2657\n", + " 0.661444\n", " 2\n", " \n", " \n", " 2\n", " 176549\n", - " 12259\n", - " 0.597595\n", + " 10440\n", + " 0.562942\n", " 3\n", " \n", " \n", " 3\n", " 176549\n", - " 12356\n", - " 0.544033\n", + " 4495\n", + " 0.557208\n", " 4\n", " \n", " \n", " 4\n", " 176549\n", - " 3734\n", - " 0.541580\n", + " 6443\n", + " 0.546108\n", " 5\n", " \n", " \n", @@ -732,14 +838,14 @@ ], "text/plain": [ " user_id item_id score rank\n", - "0 176549 15297 0.646373 1\n", - "1 176549 8636 0.609171 2\n", - "2 176549 12259 0.597595 3\n", - "3 176549 12356 0.544033 4\n", - "4 176549 3734 0.541580 5" + "0 176549 15297 0.675964 1\n", + "1 176549 2657 0.661444 2\n", + "2 176549 10440 0.562942 3\n", + "3 176549 4495 0.557208 4\n", + "4 176549 6443 0.546108 5" ] }, - "execution_count": 18, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -765,7 +871,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -860,7 +966,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -886,7 +992,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -903,10 +1009,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 21, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -925,7 +1031,7 @@ " enable_model_summary=False,\n", ")\n", "\n", - "# Replace default trainer with our custom one\n", + "# Replace trainer with our custom one\n", "model._trainer = trainer\n", "\n", "# Fit model. Everything will happen under the hood\n", @@ -941,7 +1047,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -965,7 +1071,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -998,32 +1104,32 @@ " \n", " 0\n", " 0\n", - " 22.361748\n", - " 22.401196\n", + " 22.383919\n", + " 22.365339\n", " \n", " \n", " 1\n", " 1\n", - " 21.989809\n", - " 22.256557\n", + " 22.898216\n", + " 22.189852\n", " \n", " \n", " 2\n", " 2\n", - " 22.994307\n", - " 22.055750\n", + " 22.218102\n", + " 21.964468\n", " \n", " \n", " 3\n", " 3\n", - " 22.510998\n", - " 21.802269\n", + " 22.875019\n", + " 21.701391\n", " \n", " \n", " 4\n", " 4\n", - " 21.606628\n", - " 21.510941\n", + " 21.739164\n", + " 21.417864\n", " \n", " \n", "\n", @@ -1031,14 +1137,14 @@ ], "text/plain": [ " epoch train_loss val_loss\n", - "0 0 22.361748 22.401196\n", - "1 1 21.989809 22.256557\n", - "2 2 22.994307 22.055750\n", - "3 3 22.510998 21.802269\n", - "4 4 21.606628 21.510941" + "0 0 22.383919 22.365339\n", + "1 1 22.898216 22.189852\n", + "2 2 22.218102 21.964468\n", + "3 3 22.875019 21.701391\n", + "4 4 21.739164 21.417864" ] }, - "execution_count": 24, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -1065,7 +1171,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -1101,39 +1207,39 @@ " 0\n", " 0.000052\n", " 0.000657\n", - " 0.000003\n", + " 0.000004\n", " 0\n", " 1\n", " \n", " \n", " 1\n", - " 0.000322\n", - " 0.004602\n", - " 0.000003\n", + " 0.002204\n", + " 0.024984\n", + " 0.000006\n", " 1\n", " 3\n", " \n", " \n", " 2\n", - " 0.002595\n", - " 0.029586\n", - " 0.000002\n", + " 0.006865\n", + " 0.071006\n", + " 0.000004\n", " 2\n", " 5\n", " \n", " \n", " 3\n", - " 0.004564\n", - " 0.041420\n", - " 0.000004\n", + " 0.009856\n", + " 0.097304\n", + " 0.000003\n", " 3\n", " 7\n", " \n", " \n", " 4\n", - " 0.011301\n", - " 0.094017\n", - " 0.000004\n", + " 0.010442\n", + " 0.107824\n", + " 0.000002\n", " 4\n", " 9\n", " \n", @@ -1143,14 +1249,14 @@ ], "text/plain": [ " NDCG@10 Recall@10 Serendipity@10 epoch step\n", - "0 0.000052 0.000657 0.000003 0 1\n", - "1 0.000322 0.004602 0.000003 1 3\n", - "2 0.002595 0.029586 0.000002 2 5\n", - "3 0.004564 0.041420 0.000004 3 7\n", - "4 0.011301 0.094017 0.000004 4 9" + "0 0.000052 0.000657 0.000004 0 1\n", + "1 0.002204 0.024984 0.000006 1 3\n", + "2 0.006865 0.071006 0.000004 2 5\n", + "3 0.009856 0.097304 0.000003 3 7\n", + "4 0.010442 0.107824 0.000002 4 9" ] }, - "execution_count": 25, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -1161,7 +1267,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -1179,7 +1285,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1192,31 +1298,6 @@ "HPU available: False, using: 0 HPUs\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n" ] - }, - { - "ename": "AttributeError", - "evalue": "'TransformerLightningModule' object has no attribute 'data_preparator'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[16], line 54\u001b[0m\n\u001b[1;32m 38\u001b[0m model \u001b[38;5;241m=\u001b[39m SASRecModel(\n\u001b[1;32m 39\u001b[0m n_factors\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m256\u001b[39m,\n\u001b[1;32m 40\u001b[0m n_blocks\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 49\u001b[0m get_trainer_func\u001b[38;5;241m=\u001b[39mget_custom_trainer, \u001b[38;5;66;03m# pass function to initialize our custom trainer\u001b[39;00m\n\u001b[1;32m 50\u001b[0m )\n\u001b[1;32m 53\u001b[0m \u001b[38;5;66;03m# Fit model. Everything will happen under the hood\u001b[39;00m\n\u001b[0;32m---> 54\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m)\u001b[49m;\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/rectools/models/base.py:326\u001b[0m, in \u001b[0;36mModelBase.fit\u001b[0;34m(self, dataset, *args, **kwargs)\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m: T, dataset: Dataset, \u001b[38;5;241m*\u001b[39margs: tp\u001b[38;5;241m.\u001b[39mAny, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: tp\u001b[38;5;241m.\u001b[39mAny) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 314\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 315\u001b[0m \u001b[38;5;124;03m Fit model.\u001b[39;00m\n\u001b[1;32m 316\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 324\u001b[0m \u001b[38;5;124;03m self\u001b[39;00m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 326\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_fitted \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/rectools/models/nn/transformers/base.py:381\u001b[0m, in \u001b[0;36mTransformerModelBase._fit\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_init_lightning_model(\n\u001b[1;32m 374\u001b[0m torch_model\u001b[38;5;241m=\u001b[39mtorch_model,\n\u001b[1;32m 375\u001b[0m dataset_schema\u001b[38;5;241m=\u001b[39mdataset_schema,\n\u001b[1;32m 376\u001b[0m item_external_ids\u001b[38;5;241m=\u001b[39mitem_external_ids,\n\u001b[1;32m 377\u001b[0m model_config\u001b[38;5;241m=\u001b[39mmodel_config,\n\u001b[1;32m 378\u001b[0m )\n\u001b[1;32m 380\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfit_trainer \u001b[38;5;241m=\u001b[39m deepcopy(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trainer)\n\u001b[0;32m--> 381\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_trainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlightning_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_dataloader\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:538\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m=\u001b[39m TrainerStatus\u001b[38;5;241m.\u001b[39mRUNNING\n\u001b[1;32m 537\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 538\u001b[0m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_and_handle_interrupt\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 539\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fit_impl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mval_dataloaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdatamodule\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mckpt_path\u001b[49m\n\u001b[1;32m 540\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py:47\u001b[0m, in \u001b[0;36m_call_and_handle_interrupt\u001b[0;34m(trainer, trainer_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mlauncher \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mstrategy\u001b[38;5;241m.\u001b[39mlauncher\u001b[38;5;241m.\u001b[39mlaunch(trainer_fn, \u001b[38;5;241m*\u001b[39margs, trainer\u001b[38;5;241m=\u001b[39mtrainer, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 47\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtrainer_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m _TunerExitException:\n\u001b[1;32m 50\u001b[0m _call_teardown_hook(trainer)\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:574\u001b[0m, in \u001b[0;36mTrainer._fit_impl\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mfn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 568\u001b[0m ckpt_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_checkpoint_connector\u001b[38;5;241m.\u001b[39m_select_ckpt_path(\n\u001b[1;32m 569\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mfn,\n\u001b[1;32m 570\u001b[0m ckpt_path,\n\u001b[1;32m 571\u001b[0m model_provided\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 572\u001b[0m model_connected\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlightning_module \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 573\u001b[0m )\n\u001b[0;32m--> 574\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mckpt_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mckpt_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 576\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mstopped\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:981\u001b[0m, in \u001b[0;36mTrainer._run\u001b[0;34m(self, model, ckpt_path)\u001b[0m\n\u001b[1;32m 976\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_signal_connector\u001b[38;5;241m.\u001b[39mregister_signal_handlers()\n\u001b[1;32m 978\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[1;32m 979\u001b[0m \u001b[38;5;66;03m# RUN THE TRAINER\u001b[39;00m\n\u001b[1;32m 980\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[0;32m--> 981\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run_stage\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 983\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[1;32m 984\u001b[0m \u001b[38;5;66;03m# POST-Training CLEAN UP\u001b[39;00m\n\u001b[1;32m 985\u001b[0m \u001b[38;5;66;03m# ----------------------------\u001b[39;00m\n\u001b[1;32m 986\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: trainer tearing down\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1023\u001b[0m, in \u001b[0;36mTrainer._run_stage\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining:\n\u001b[1;32m 1022\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m isolate_rng():\n\u001b[0;32m-> 1023\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_run_sanity_check\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1024\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mset_detect_anomaly(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_detect_anomaly):\n\u001b[1;32m 1025\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfit_loop\u001b[38;5;241m.\u001b[39mrun()\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1052\u001b[0m, in \u001b[0;36mTrainer._run_sanity_check\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1049\u001b[0m call\u001b[38;5;241m.\u001b[39m_call_callback_hooks(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_sanity_check_start\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1051\u001b[0m \u001b[38;5;66;03m# run eval step\u001b[39;00m\n\u001b[0;32m-> 1052\u001b[0m \u001b[43mval_loop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1054\u001b[0m call\u001b[38;5;241m.\u001b[39m_call_callback_hooks(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_sanity_check_end\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 1056\u001b[0m \u001b[38;5;66;03m# reset logger connector\u001b[39;00m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/loops/utilities.py:178\u001b[0m, in \u001b[0;36m_no_grad_context.._decorator\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 176\u001b[0m context_manager \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mno_grad\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m context_manager():\n\u001b[0;32m--> 178\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloop_run\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/loops/evaluation_loop.py:135\u001b[0m, in \u001b[0;36m_EvaluationLoop.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_progress\u001b[38;5;241m.\u001b[39mis_last_batch \u001b[38;5;241m=\u001b[39m data_fetcher\u001b[38;5;241m.\u001b[39mdone\n\u001b[1;32m 134\u001b[0m \u001b[38;5;66;03m# run step hooks\u001b[39;00m\n\u001b[0;32m--> 135\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_evaluation_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdataloader_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdataloader_iter\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 137\u001b[0m \u001b[38;5;66;03m# this needs to wrap the `*_step` call too (not just `next`) for `dataloader_iter` support\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/loops/evaluation_loop.py:410\u001b[0m, in \u001b[0;36m_EvaluationLoop._evaluation_step\u001b[0;34m(self, batch, batch_idx, dataloader_idx, dataloader_iter)\u001b[0m\n\u001b[1;32m 405\u001b[0m hook_kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_kwargs(\n\u001b[1;32m 406\u001b[0m batch, batch_idx, dataloader_idx \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_sequential \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_dataloaders \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 407\u001b[0m )\n\u001b[1;32m 409\u001b[0m hook_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_test_batch_end\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mtesting \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mon_validation_batch_end\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 410\u001b[0m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_callback_hooks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrainer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhook_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhook_kwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalues\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 411\u001b[0m call\u001b[38;5;241m.\u001b[39m_call_lightning_module_hook(trainer, hook_name, output, \u001b[38;5;241m*\u001b[39mhook_kwargs\u001b[38;5;241m.\u001b[39mvalues())\n\u001b[1;32m 413\u001b[0m trainer\u001b[38;5;241m.\u001b[39m_logger_connector\u001b[38;5;241m.\u001b[39mon_batch_end()\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py:218\u001b[0m, in \u001b[0;36m_call_callback_hooks\u001b[0;34m(trainer, hook_name, monitoring_callbacks, *args, **kwargs)\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcallable\u001b[39m(fn):\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m trainer\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mprofile(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[Callback]\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcallback\u001b[38;5;241m.\u001b[39mstate_key\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhook_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 218\u001b[0m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrainer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlightning_module\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pl_module:\n\u001b[1;32m 221\u001b[0m \u001b[38;5;66;03m# restore current_fx when nested context\u001b[39;00m\n\u001b[1;32m 222\u001b[0m pl_module\u001b[38;5;241m.\u001b[39m_current_fx_name \u001b[38;5;241m=\u001b[39m prev_fx_name\n", - "Cell \u001b[0;32mIn[13], line 57\u001b[0m, in \u001b[0;36mValidationMetrics.on_validation_batch_end\u001b[0;34m(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx)\u001b[0m\n\u001b[1;32m 48\u001b[0m batch_recos_df[Columns\u001b[38;5;241m.\u001b[39mRank] \u001b[38;5;241m=\u001b[39m batch_recos_df\u001b[38;5;241m.\u001b[39mgroupby(Columns\u001b[38;5;241m.\u001b[39mUser, sort\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\u001b[38;5;241m.\u001b[39mcumcount() \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 50\u001b[0m interactions \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(\n\u001b[1;32m 51\u001b[0m {\n\u001b[1;32m 52\u001b[0m Columns\u001b[38;5;241m.\u001b[39mUser: batch_target_users,\n\u001b[1;32m 53\u001b[0m Columns\u001b[38;5;241m.\u001b[39mItem: \u001b[38;5;28mlist\u001b[39m(itertools\u001b[38;5;241m.\u001b[39mchain\u001b[38;5;241m.\u001b[39mfrom_iterable(targets)),\n\u001b[1;32m 54\u001b[0m }\n\u001b[1;32m 55\u001b[0m )\n\u001b[0;32m---> 57\u001b[0m prev_interactions \u001b[38;5;241m=\u001b[39m \u001b[43mpl_module\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata_preparator\u001b[49m\u001b[38;5;241m.\u001b[39mtrain_dataset\u001b[38;5;241m.\u001b[39minteractions\u001b[38;5;241m.\u001b[39mdf\n\u001b[1;32m 58\u001b[0m catalog \u001b[38;5;241m=\u001b[39m prev_interactions[Columns\u001b[38;5;241m.\u001b[39mItem]\u001b[38;5;241m.\u001b[39munique()\n\u001b[1;32m 60\u001b[0m batch_metrics \u001b[38;5;241m=\u001b[39m calc_metrics(\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mval_metrics, \n\u001b[1;32m 62\u001b[0m batch_recos_df,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 65\u001b[0m catalog\n\u001b[1;32m 66\u001b[0m )\n", - "File \u001b[0;32m/data/home/dmtikhono1/RecTools/.venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1931\u001b[0m, in \u001b[0;36mModule.__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 1929\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01min\u001b[39;00m modules:\n\u001b[1;32m 1930\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m modules[name]\n\u001b[0;32m-> 1931\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[1;32m 1932\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(\u001b[38;5;28mself\u001b[39m)\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m object has no attribute \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1933\u001b[0m )\n", - "\u001b[0;31mAttributeError\u001b[0m: 'TransformerLightningModule' object has no attribute 'data_preparator'" - ] } ], "source": [ @@ -1795,7 +1876,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -1841,10 +1922,15 @@ " 'transformer_layers_type': 'rectools.models.nn.transformers.sasrec.SASRecTransformerLayers',\n", " 'lightning_module_type': 'rectools.models.nn.transformers.lightning.TransformerLightningModule',\n", " 'get_val_mask_func': None,\n", - " 'get_trainer_func': None}" + " 'get_trainer_func': None,\n", + " 'data_preparator_kwargs': None,\n", + " 'transformer_layers_kwargs': None,\n", + " 'item_net_constructor_kwargs': None,\n", + " 'pos_encoding_kwargs': None,\n", + " 'lightning_module_kwargs': None}" ] }, - "execution_count": 3, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -1876,7 +1962,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -1922,10 +2008,15 @@ " 'transformer_layers_type': 'rectools.models.nn.transformers.sasrec.SASRecTransformerLayers',\n", " 'lightning_module_type': 'rectools.models.nn.transformers.lightning.TransformerLightningModule',\n", " 'get_val_mask_func': '__main__.get_val_mask_func',\n", - " 'get_trainer_func': '__main__.get_custom_trainer'}" + " 'get_trainer_func': '__main__.get_custom_trainer',\n", + " 'data_preparator_kwargs': None,\n", + " 'transformer_layers_kwargs': None,\n", + " 'item_net_constructor_kwargs': None,\n", + " 'pos_encoding_kwargs': None,\n", + " 'lightning_module_kwargs': None}" ] }, - "execution_count": 17, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb index 2cbe95e2..7b129451 100644 --- a/examples/tutorials/transformers_customization_guide.ipynb +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -57,7 +57,7 @@ " PreLNTransformerLayer,\n", " TransformerLayersBase,\n", ")\n", - "from rectools.models.nn.transformers.constants import InitKwargs, MASKING_VALUE\n", + "from rectools.models.nn.transformers.constants import MASKING_VALUE\n", "from rectools.models.nn.transformers.bert4rec import BERT4RecDataPreparator\n", "from rectools.models.nn.transformers.lightning import TransformerLightningModule\n", "\n", @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -87,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -105,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -121,7 +121,7 @@ "60" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -134,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -178,7 +178,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -248,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -264,10 +264,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 10, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -291,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -307,10 +307,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 11, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -349,11 +349,11 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "# Special Albert logic for Embeddings - Factorized embedding parameterization\n", + "# Special Albert logic for embeddings - Factorized embedding parameterization\n", "\n", "class AlBERT4RecSumOfEmbeddingsConstructor(SumOfEmbeddingsConstructor):\n", "\n", @@ -417,11 +417,11 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "# Special Albert logic for Transformer Layers - Cross-layer parameter sharing\n", + "# Special Albert logic for transformer layers - Cross-layer parameter sharing\n", " \n", "class AlBERT4RecPreLNTransformerLayers(TransformerLayersBase):\n", "\n", @@ -482,7 +482,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -498,10 +498,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 14, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -535,7 +535,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -551,10 +551,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 16, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -581,7 +581,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -597,10 +597,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 17, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -631,7 +631,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -678,7 +678,7 @@ " 'mask_prob': 0.15}" ] }, - "execution_count": 18, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -686,14 +686,15 @@ "source": [ "params = next_action_albert_causal.get_params(simple_types=True)\n", "params\n", - "# params include \"transformer_layers_kwargs.n_hidden_groups\", \n", + "# see below that model params include our custom keyword arguments:\n", + "# \"transformer_layers_kwargs.n_hidden_groups\", \n", "# \"transformer_layers_kwargs.n_inner_groups\"\n", "# and \"item_net_constructor_kwargs.emb_factors\"" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -709,10 +710,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 19, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } From ff92e7d9b7e9565dc3f447e0e727f31e6fe85d69 Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 12:02:18 +0300 Subject: [PATCH 11/15] updated links --- examples/tutorials/transformers_tutorial.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/tutorials/transformers_tutorial.ipynb b/examples/tutorials/transformers_tutorial.ipynb index 7bb3e049..9da6bcbf 100644 --- a/examples/tutorials/transformers_tutorial.ipynb +++ b/examples/tutorials/transformers_tutorial.ipynb @@ -19,8 +19,8 @@ "- Item features are added to item embedding net.\n", "- Multiple loss options.\n", "\n", - "- Advanced training options like custom validation, logging, checkpoints and early stopping are available. See [Advanced training guide](https://github.com/MobileTeleSystems/RecTools/blob/feature/transformer_ckpt/examples/tutorials/transformers_advanced_training_guide.ipynb).\n", - "- You can customize models architecture any way you like, keeping all of the above benefits." + "- Advanced training options like custom validation, logging, checkpoints and early stopping are available. See [Advanced training guide](https://github.com/MobileTeleSystems/RecTools/blob/master/examples/tutorials/transformers_advanced_training_guide.ipynb).\n", + "- You can customize models architecture any way you like, keeping all of the above benefits. See [Customization guide](https://github.com/MobileTeleSystems/RecTools/blob/master/examples/tutorials/transformers_customization_guide.ipynb)." ] }, { @@ -2151,9 +2151,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "rectools", "language": "python", - "name": "python3" + "name": "rectools" }, "language_info": { "codemirror_mode": { @@ -2165,7 +2165,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.12" } }, "nbformat": 4, From 45c9468d69100bf9740e70a2a393dc83f9c9dba6 Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 14:30:00 +0300 Subject: [PATCH 12/15] links and changelog --- CHANGELOG.md | 3 +-- examples/tutorials/transformers_tutorial.ipynb | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb7e1c93..03360c3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `SASRecModel` and `BERT4RecModel` - models based on transformer architecture ([#220](https://github.com/MobileTeleSystems/RecTools/pull/220)) -- Transfomers extended theory & practice tutorial ([#220](https://github.com/MobileTeleSystems/RecTools/pull/220)) -- Transfomers advanced training guide ([#220](https://github.com/MobileTeleSystems/RecTools/pull/220)) +- Transfomers extended theory & practice tutorial, advanced training guide and customization guide ([#220](https://github.com/MobileTeleSystems/RecTools/pull/220)) - `use_gpu` for PureSVD ([#229](https://github.com/MobileTeleSystems/RecTools/pull/229)) - `from_params` method for models and `model_from_params` function ([#252](https://github.com/MobileTeleSystems/RecTools/pull/252)) - `TorchRanker` ranker which calculates scores using torch. Supports GPU. [#251](https://github.com/MobileTeleSystems/RecTools/pull/251) diff --git a/examples/tutorials/transformers_tutorial.ipynb b/examples/tutorials/transformers_tutorial.ipynb index 9da6bcbf..ea8ea89a 100644 --- a/examples/tutorials/transformers_tutorial.ipynb +++ b/examples/tutorials/transformers_tutorial.ipynb @@ -19,8 +19,8 @@ "- Item features are added to item embedding net.\n", "- Multiple loss options.\n", "\n", - "- Advanced training options like custom validation, logging, checkpoints and early stopping are available. See [Advanced training guide](https://github.com/MobileTeleSystems/RecTools/blob/master/examples/tutorials/transformers_advanced_training_guide.ipynb).\n", - "- You can customize models architecture any way you like, keeping all of the above benefits. See [Customization guide](https://github.com/MobileTeleSystems/RecTools/blob/master/examples/tutorials/transformers_customization_guide.ipynb)." + "- Advanced training options like custom validation, logging, checkpoints and early stopping are available. See [Advanced training guide](https://github.com/MobileTeleSystems/RecTools/blob/main/examples/tutorials/transformers_advanced_training_guide.ipynb).\n", + "- You can customize models architecture any way you like, keeping all of the above benefits. See [Customization guide](https://github.com/MobileTeleSystems/RecTools/blob/main/examples/tutorials/transformers_customization_guide.ipynb)." ] }, { From c1a62f5a42fbd0f8635c23508e3ed6bd5b5255f2 Mon Sep 17 00:00:00 2001 From: blondered Date: Sat, 15 Feb 2025 15:12:48 +0300 Subject: [PATCH 13/15] customized happy path --- tests/models/nn/transformers/test_bert4rec.py | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/tests/models/nn/transformers/test_bert4rec.py b/tests/models/nn/transformers/test_bert4rec.py index 406cc8c8..fbf1bac5 100644 --- a/tests/models/nn/transformers/test_bert4rec.py +++ b/tests/models/nn/transformers/test_bert4rec.py @@ -32,7 +32,7 @@ TrainerCallable, TransformerLightningModule, ) -from rectools.models.nn.transformers.bert4rec import BERT4RecDataPreparator +from rectools.models.nn.transformers.bert4rec import MASKING_VALUE, BERT4RecDataPreparator, ValMaskCallable from tests.models.data import DATASET from tests.models.utils import ( assert_default_config_and_default_model_params_are_the_same, @@ -535,6 +535,95 @@ def test_recommend_for_cold_user_with_hot_item( actual, ) + def test_customized_happy_path(self, dataset_devices: Dataset, get_trainer_func: TrainerCallable) -> None: + class NextActionDataPreparator(BERT4RecDataPreparator): + def __init__( + self, + session_max_len: int, + n_negatives: tp.Optional[int], + batch_size: int, + dataloader_num_workers: int, + train_min_user_interactions: int, + mask_prob: float = 0.15, + shuffle_train: bool = True, + get_val_mask_func: tp.Optional[ValMaskCallable] = None, + n_last_targets: int = 1, # custom kwarg + ) -> None: + super().__init__( + session_max_len=session_max_len, + n_negatives=n_negatives, + batch_size=batch_size, + dataloader_num_workers=dataloader_num_workers, + train_min_user_interactions=train_min_user_interactions, + shuffle_train=shuffle_train, + get_val_mask_func=get_val_mask_func, + mask_prob=mask_prob, + ) + self.n_last_targets = n_last_targets + + def _collate_fn_train( + self, + batch: tp.List[tp.Tuple[tp.List[int], tp.List[float]]], + ) -> tp.Dict[str, torch.Tensor]: + batch_size = len(batch) + x = np.zeros((batch_size, self.session_max_len)) + y = np.zeros((batch_size, self.session_max_len)) + yw = np.zeros((batch_size, self.session_max_len)) + for i, (ses, ses_weights) in enumerate(batch): + y[i, -self.n_last_targets] = ses[-self.n_last_targets] + yw[i, -self.n_last_targets] = ses_weights[-self.n_last_targets] + x[i, -len(ses) :] = ses + x[i, -self.n_last_targets] = self.extra_token_ids[MASKING_VALUE] # Replace last tokens with "MASK" + batch_dict = {"x": torch.LongTensor(x), "y": torch.LongTensor(y), "yw": torch.FloatTensor(yw)} + if self.n_negatives is not None: + negatives = torch.randint( + low=self.n_item_extra_tokens, + high=self.item_id_map.size, + size=(batch_size, self.session_max_len, self.n_negatives), + ) + batch_dict["negatives"] = negatives + return batch_dict + + model = BERT4RecModel( + n_factors=32, + n_blocks=2, + n_heads=1, + session_max_len=4, + lr=0.001, + batch_size=4, + epochs=2, + deterministic=True, + item_net_block_types=(IdEmbeddingsItemNet,), + get_trainer_func=get_trainer_func, + data_preparator_type=NextActionDataPreparator, + data_preparator_kwargs={"n_last_targets": 1}, + ) + model.fit(dataset=dataset_devices) + + assert model.data_preparator.n_last_targets == 1 # type: ignore + + users = np.array([10, 30, 40]) + items_to_recommend = np.array([11, 13, 17]) + actual = model.recommend( + users=users, + dataset=dataset_devices, + k=3, + filter_viewed=False, + items_to_recommend=items_to_recommend, + ) + expected = pd.DataFrame( + { + Columns.User: [10, 10, 30, 30, 40, 40], + Columns.Item: [13, 11, 13, 11, 13, 11], + Columns.Rank: [1, 2, 1, 2, 1, 2], + } + ) + pd.testing.assert_frame_equal(actual.drop(columns=Columns.Score), expected) + pd.testing.assert_frame_equal( + actual.sort_values([Columns.User, Columns.Score], ascending=[True, False]).reset_index(drop=True), + actual, + ) + class TestBERT4RecDataPreparator: From 3d646c3ae05426951b20a88b65c544db299bee35 Mon Sep 17 00:00:00 2001 From: blondered Date: Sun, 16 Feb 2025 09:59:36 +0300 Subject: [PATCH 14/15] validation in customization --- .../transformers_customization_guide.ipynb | 559 +++++++++++++++--- .../tutorials/transformers_tutorial.ipynb | 2 +- 2 files changed, 488 insertions(+), 73 deletions(-) diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb index 7b129451..0a63c47b 100644 --- a/examples/tutorials/transformers_customization_guide.ipynb +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -16,19 +16,20 @@ " - Custom data preparator and lightning module\n", " - Create `NextActionTransformer`\n", " - Enable unidirectional attention\n", - "* Albert\n", + "* ALBERT\n", " - Custom transformer layers and item net constructor\n", - " - Pass Albert modules to `BERT4RecModel`\n", - " - Pass Albert modules to `SASRecModel`\n", - "* How about `NextActionTransformer` with Albert logic and causal attention?\n", + " - Pass ALBERT modules to `BERT4RecModel`\n", + " - Pass ALBERT modules to `SASRecModel`\n", + "* How about `NextActionTransformer` with ALBERT modules and causal attention?\n", " - Combining custom modules together\n", + "* Cross-validation\n", "* Configs support for custom models\n", "* Full list of customization options" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -47,8 +48,17 @@ "\n", "from rectools import Columns\n", "from rectools.dataset import Dataset\n", - "from rectools.models import BERT4RecModel, SASRecModel\n", + "from rectools.models import BERT4RecModel, SASRecModel, PopularModel\n", "from rectools.dataset.dataset import DatasetSchema\n", + "from rectools.model_selection import TimeRangeSplitter, cross_validate\n", + "from rectools.metrics import (\n", + " MAP,\n", + " CoveredUsers,\n", + " AvgRecPopularity,\n", + " Intersection,\n", + " HitRate,\n", + " Serendipity,\n", + ")\n", "from rectools.models.nn.item_net import (\n", " ItemNetBase,\n", " SumOfEmbeddingsConstructor,\n", @@ -87,17 +97,17 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "DATA_PATH = Path(\"data_original\")\n", + "DATA_PATH = Path(\"data_en\")\n", "\n", "interactions = (\n", " pd.read_csv(DATA_PATH / 'interactions.csv', parse_dates=[\"last_watch_dt\"])\n", " .rename(columns={\"last_watch_dt\": \"datetime\"})\n", ")\n", - "interactions[Columns.Weight] = np.where(interactions['watched_pct'] > 10, 3, 1)\n", + "interactions[Columns.Weight] = 1\n", "dataset = Dataset.construct(\n", " interactions_df=interactions,\n", ")" @@ -105,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -121,7 +131,7 @@ "60" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -134,11 +144,12 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# Function to get custom trainer\n", + "\n", "def get_debug_trainer() -> Trainer:\n", " return Trainer(\n", " accelerator=\"cpu\",\n", @@ -146,7 +157,7 @@ " min_epochs=1,\n", " max_epochs=1,\n", " deterministic=True,\n", - " enable_model_summary=False,\n", + " enable_model_summary=True,\n", " enable_progress_bar=False,\n", " limit_train_batches=2, # limit train batches for quick debug runs\n", " )" @@ -178,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -248,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -258,16 +269,27 @@ "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 5.5 M | train\n", + "-----------------------------------------------------------------\n", + "5.5 M Trainable params\n", + "0 Non-trainable params\n", + "5.5 M Total params\n", + "22.040 Total estimated model params size (MB)\n", + "37 Modules in train mode\n", + "0 Modules in eval mode\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -291,7 +313,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -301,16 +323,27 @@ "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 5.5 M | train\n", + "-----------------------------------------------------------------\n", + "5.5 M Trainable params\n", + "0 Non-trainable params\n", + "5.5 M Total params\n", + "22.040 Total estimated model params size (MB)\n", + "37 Modules in train mode\n", + "0 Modules in eval mode\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -322,8 +355,7 @@ " get_trainer_func = get_debug_trainer,\n", " use_causal_attn = True, # simple flag\n", ")\n", - "\n", - "next_action_model_causal.fit(dataset)" + "next_action_model.fit(dataset)" ] }, { @@ -333,7 +365,7 @@ "## ALBERT\n", "[ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942)\n", "\n", - "Albert has two parameter-reduction techniques to lower memory consumption and increase the training speed which can actually be used together or separately:\n", + "ALBERT has two parameter-reduction techniques to lower memory consumption and increase the training speed which can actually be used together or separately:\n", "1. Learning embeddings of smaller size and then projecting them to the required size through a Liner projection (\"Factorized embedding parameterization\")\n", "2. Sharing weights between transformer layers (\"Cross-layer parameter sharing\")\n", "\n", @@ -349,13 +381,13 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "# Special Albert logic for embeddings - Factorized embedding parameterization\n", + "# Special ALBERT logic for embeddings - Factorized embedding parameterization\n", "\n", - "class AlBERT4RecSumOfEmbeddingsConstructor(SumOfEmbeddingsConstructor):\n", + "class AlbertSumConstructor(SumOfEmbeddingsConstructor):\n", "\n", " def __init__(\n", " self,\n", @@ -417,13 +449,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "# Special Albert logic for transformer layers - Cross-layer parameter sharing\n", + "# Special ALBERT logic for transformer layers - Cross-layer parameter sharing\n", " \n", - "class AlBERT4RecPreLNTransformerLayers(TransformerLayersBase):\n", + "class AlbertLayers(TransformerLayersBase):\n", "\n", " def __init__(\n", " self,\n", @@ -477,12 +509,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Pass Albert modules to `BERT4RecModel`" + "### Pass ALBERT modules to `BERT4RecModel`\n", + "Now we need to pass both our custom classes and their keyword arguments for initialization." ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -492,50 +525,65 @@ "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 4.2 M | train\n", + "-----------------------------------------------------------------\n", + "4.2 M Trainable params\n", + "0 Non-trainable params\n", + "4.2 M Total params\n", + "16.710 Total estimated model params size (MB)\n", + "64 Modules in train mode\n", + "0 Modules in eval mode\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "CONSTRUCTOR_KWARGS = {\n", - " \"emb_factors\": 32,\n", + " \"emb_factors\": 64,\n", "}\n", "\n", "TRANSFORMER_LAYERS_KWARGS = {\n", " \"n_hidden_groups\": 2,\n", - " \"n_inner_groups\": 1,\n", + " \"n_inner_groups\": 2,\n", "}\n", "\n", "albert_model = BERT4RecModel(\n", - " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", - " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", - " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", + " item_net_constructor_type=AlbertSumConstructor, # type\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs\n", + " transformer_layers_type=AlbertLayers, # type\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs\n", " get_trainer_func = get_debug_trainer,\n", ")\n", - "albert_model.fit(dataset)" + "\n", + "albert_model.fit(dataset)\n", + "# See that with Albert modules we have 4.2 M trainable params instead of 5.5 M previously" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Pass Albert modules to `SASRecModel`" + "### Pass ALBERT modules to `SASRecModel`\n", + "We are not limited to BERT4Rec when we just changed embedding and transformer layers logic.\n", + "Why not create ALSASRec?" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -545,26 +593,37 @@ "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 4.2 M | train\n", + "-----------------------------------------------------------------\n", + "4.2 M Trainable params\n", + "0 Non-trainable params\n", + "4.2 M Total params\n", + "16.711 Total estimated model params size (MB)\n", + "64 Modules in train mode\n", + "0 Modules in eval mode\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "alsasrec_model = SASRecModel(\n", - " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", - " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", - " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", + " item_net_constructor_type=AlbertSumConstructor,\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS,\n", + " transformer_layers_type=AlbertLayers,\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS,\n", " get_trainer_func = get_debug_trainer,\n", ")\n", "alsasrec_model.fit(dataset)" @@ -574,14 +633,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## How about `NextActionTransformer` with Albert logic and causal attention?\n", + "## How about `NextActionTransformer` with ALBERT modules and causal attention?\n", "Just because we can!\n", "### Combining custom modules together" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -591,34 +650,377 @@ "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 4.2 M | train\n", + "-----------------------------------------------------------------\n", + "4.2 M Trainable params\n", + "0 Non-trainable params\n", + "4.2 M Total params\n", + "16.710 Total estimated model params size (MB)\n", + "64 Modules in train mode\n", + "0 Modules in eval mode\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "next_action_albert_causal = BERT4RecModel(\n", - " item_net_constructor_type=AlBERT4RecSumOfEmbeddingsConstructor, # custom item net constructor\n", - " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS, # kwargs for custom constructor\n", - " transformer_layers_type=AlBERT4RecPreLNTransformerLayers, # custom transformer layers\n", - " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS, # kwargs for custom transformer layers\n", - " data_preparator_type=NextActionDataPreparator, # custom data preparator\n", - " lightning_module_type=NextActionLightningModule, # custom lightning module\n", - " use_causal_attn=True, # Apply causal attention mask\n", + " item_net_constructor_type=AlbertSumConstructor,\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS,\n", + " transformer_layers_type=AlbertLayers,\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS,\n", + " data_preparator_type=NextActionDataPreparator,\n", + " lightning_module_type=NextActionLightningModule,\n", + " use_causal_attn=True,\n", " get_trainer_func = get_debug_trainer,\n", ")\n", "next_action_albert_causal.fit(dataset)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cross-validation\n", + "Let's validate our custom models compared to vanilla SASRec" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n", + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + } + ], + "source": [ + "# Initialize models for cross-validation\n", + "\n", + "def get_trainer() -> Trainer:\n", + " return Trainer(\n", + " accelerator=\"gpu\",\n", + " devices=[1],\n", + " min_epochs=3,\n", + " max_epochs=3,\n", + " deterministic=True,\n", + " enable_model_summary=False,\n", + " enable_progress_bar=False,\n", + " )\n", + "\n", + "next_action_bidirectional = BERT4RecModel(\n", + " data_preparator_type=NextActionDataPreparator,\n", + " lightning_module_type=NextActionLightningModule,\n", + " deterministic=True,\n", + " get_trainer_func=get_trainer,\n", + ")\n", + "\n", + "next_action_unidirectional = BERT4RecModel(\n", + " data_preparator_type=NextActionDataPreparator,\n", + " lightning_module_type=NextActionLightningModule,\n", + " deterministic=True,\n", + " use_causal_attn=True,\n", + " get_trainer_func=get_trainer,\n", + ")\n", + "\n", + "CONSTRUCTOR_KWARGS = {\n", + " \"emb_factors\": 64,\n", + "}\n", + "TRANSFORMER_LAYERS_KWARGS = {\n", + " \"n_hidden_groups\": 2,\n", + " \"n_inner_groups\": 2,\n", + "}\n", + "\n", + "albert = BERT4RecModel(\n", + " item_net_constructor_type=AlbertSumConstructor,\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS,\n", + " transformer_layers_type=AlbertLayers,\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS,\n", + " deterministic=True,\n", + " get_trainer_func=get_trainer,\n", + ")\n", + "\n", + "alsasrec = SASRecModel(\n", + " item_net_constructor_type=AlbertSumConstructor,\n", + " item_net_constructor_kwargs=CONSTRUCTOR_KWARGS,\n", + " transformer_layers_type=AlbertLayers,\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS,\n", + " deterministic=True,\n", + " get_trainer_func=get_trainer,\n", + ")\n", + "\n", + "sasrec_albert_layers = SASRecModel(\n", + " transformer_layers_type=AlbertLayers,\n", + " transformer_layers_kwargs=TRANSFORMER_LAYERS_KWARGS,\n", + " deterministic=True,\n", + " get_trainer_func=get_trainer,\n", + ")\n", + "\n", + "\n", + "models = {\n", + " \"popular\": PopularModel(),\n", + " \"sasrec\": SASRecModel(deterministic=True),\n", + " \"next_action_bidirectional\": next_action_bidirectional,\n", + " \"next_action_unidirectional\": next_action_unidirectional,\n", + " \"albert\": albert,\n", + " \"alsasrec\": alsasrec,\n", + " \"sasrec_albert_layers\": sasrec_albert_layers,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "`Trainer.fit` stopped: `max_epochs=3` reached.\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "`Trainer.fit` stopped: `max_epochs=3` reached.\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "`Trainer.fit` stopped: `max_epochs=3` reached.\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "`Trainer.fit` stopped: `max_epochs=3` reached.\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "`Trainer.fit` stopped: `max_epochs=3` reached.\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n", + "`Trainer.fit` stopped: `max_epochs=3` reached.\n" + ] + } + ], + "source": [ + "# Validate models\n", + "\n", + "metrics = {\n", + " \"HitRate@10\": HitRate(k=10),\n", + " \"MAP@10\": MAP(k=10),\n", + " \"Serendipity@10\": Serendipity(k=10),\n", + " \"CoveredUsers@10\": CoveredUsers(k=10), # how many test users received recommendations\n", + " \"AvgRecPopularity@10\": AvgRecPopularity(k=10), # average popularity of recommended items\n", + " \"Intersection@10\": Intersection(k=10), # intersection with recommendations from reference model\n", + "}\n", + "\n", + "splitter = TimeRangeSplitter(\n", + " test_size=\"7D\",\n", + " n_splits=1, # 1 fold\n", + " filter_already_seen=True,\n", + ")\n", + "\n", + "K_RECS = 10\n", + "\n", + "# For each fold generate train and test part of dataset\n", + "# Then fit every model, generate recommendations and calculate metrics\n", + "\n", + "cv_results = cross_validate(\n", + " dataset=dataset,\n", + " splitter=splitter,\n", + " models=models,\n", + " metrics=metrics,\n", + " k=K_RECS,\n", + " filter_viewed=True,\n", + " ref_models=[\"popular\"], # pass reference model to calculate recommendations intersection\n", + " validate_ref_models=True,\n", + ")\n", + "\n", + "pivot_results = (\n", + " pd.DataFrame(cv_results[\"metrics\"])\n", + " .drop(columns=\"i_split\")\n", + " .groupby([\"model\"], sort=False)\n", + " .agg([\"mean\"])\n", + ")\n", + "pivot_results.columns = pivot_results.columns.droplevel(1)\n", + "pivot_results.to_csv(\"rectools_custom_transformers_cv_en.csv\", index=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
HitRate@10MAP@10AvgRecPopularity@10Serendipity@10Intersection@10_popularCoveredUsers@10
model
popular0.2743650.08011482236.7617830.0000021.0000001.0
sasrec0.3169170.09223670526.2435310.0000290.6211301.0
next_action_bidirectional0.3477690.09948857260.7008780.0000990.4615271.0
next_action_unidirectional0.3429540.10058154372.5091360.0001070.4450711.0
albert0.3325520.09570262428.8685900.0000500.5140521.0
alsasrec0.3469510.09855450137.4045800.0001990.3934411.0
sasrec_albert_layers0.3474870.10007950387.7822160.0002500.4240361.0
\n", + "
" + ], + "text/plain": [ + " HitRate@10 MAP@10 AvgRecPopularity@10 \\\n", + "model \n", + "popular 0.274365 0.080114 82236.761783 \n", + "sasrec 0.316917 0.092236 70526.243531 \n", + "next_action_bidirectional 0.347769 0.099488 57260.700878 \n", + "next_action_unidirectional 0.342954 0.100581 54372.509136 \n", + "albert 0.332552 0.095702 62428.868590 \n", + "alsasrec 0.346951 0.098554 50137.404580 \n", + "sasrec_albert_layers 0.347487 0.100079 50387.782216 \n", + "\n", + " Serendipity@10 Intersection@10_popular \\\n", + "model \n", + "popular 0.000002 1.000000 \n", + "sasrec 0.000029 0.621130 \n", + "next_action_bidirectional 0.000099 0.461527 \n", + "next_action_unidirectional 0.000107 0.445071 \n", + "albert 0.000050 0.514052 \n", + "alsasrec 0.000199 0.393441 \n", + "sasrec_albert_layers 0.000250 0.424036 \n", + "\n", + " CoveredUsers@10 \n", + "model \n", + "popular 1.0 \n", + "sasrec 1.0 \n", + "next_action_bidirectional 1.0 \n", + "next_action_unidirectional 1.0 \n", + "albert 1.0 \n", + "alsasrec 1.0 \n", + "sasrec_albert_layers 1.0 " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pivot_results" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -631,7 +1033,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -663,22 +1065,22 @@ " 'train_min_user_interactions': 2,\n", " 'item_net_block_types': ['rectools.models.nn.item_net.IdEmbeddingsItemNet',\n", " 'rectools.models.nn.item_net.CatFeaturesItemNet'],\n", - " 'item_net_constructor_type': '__main__.AlBERT4RecSumOfEmbeddingsConstructor',\n", + " 'item_net_constructor_type': '__main__.AlbertSumConstructor',\n", " 'pos_encoding_type': 'rectools.models.nn.transformers.net_blocks.LearnableInversePositionalEncoding',\n", - " 'transformer_layers_type': '__main__.AlBERT4RecPreLNTransformerLayers',\n", + " 'transformer_layers_type': '__main__.AlbertLayers',\n", " 'lightning_module_type': '__main__.NextActionLightningModule',\n", " 'get_val_mask_func': None,\n", " 'get_trainer_func': '__main__.get_debug_trainer',\n", " 'data_preparator_kwargs': None,\n", " 'transformer_layers_kwargs.n_hidden_groups': 2,\n", - " 'transformer_layers_kwargs.n_inner_groups': 1,\n", - " 'item_net_constructor_kwargs.emb_factors': 32,\n", + " 'transformer_layers_kwargs.n_inner_groups': 2,\n", + " 'item_net_constructor_kwargs.emb_factors': 64,\n", " 'pos_encoding_kwargs': None,\n", " 'lightning_module_kwargs': None,\n", " 'mask_prob': 0.15}" ] }, - "execution_count": 13, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -686,7 +1088,7 @@ "source": [ "params = next_action_albert_causal.get_params(simple_types=True)\n", "params\n", - "# see below that model params include our custom keyword arguments:\n", + "# See below that model params include our custom keyword arguments:\n", "# \"transformer_layers_kwargs.n_hidden_groups\", \n", "# \"transformer_layers_kwargs.n_inner_groups\"\n", "# and \"item_net_constructor_kwargs.emb_factors\"" @@ -694,8 +1096,10 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": {}, + "execution_count": 20, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stderr", @@ -704,16 +1108,27 @@ "GPU available: True (cuda), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", + "\n", + " | Name | Type | Params | Mode \n", + "-----------------------------------------------------------------\n", + "0 | torch_model | TransformerTorchBackbone | 4.2 M | train\n", + "-----------------------------------------------------------------\n", + "4.2 M Trainable params\n", + "0 Non-trainable params\n", + "4.2 M Total params\n", + "16.710 Total estimated model params size (MB)\n", + "64 Modules in train mode\n", + "0 Modules in eval mode\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 14, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -729,7 +1144,7 @@ "source": [ "## Full list of customization options\n", "\n", - "These blocks of RecTools transfofmer models can be replaced with custom classes (WITH an option to add required keyword arguments for initialization):\n", + "These blocks of RecTools transformer models can be replaced with custom classes (WITH an option to add required keyword arguments for initialization):\n", "- data preparator (`data_preparator_type`, `data_preparator_kwargs`)\n", " - forming training objectives\n", " - providing train, val and recommend dataloaders preparation\n", @@ -744,7 +1159,7 @@ "- transformer layers (`transformer_layers_type`, `transformer_layers_kwargs`)\n", "- positional encoding (`pos_encoding_type`, `pos_encoding_kwargs`)\n", "\n", - "These blocks of RecTools transfofmer models can be replaced with custom classes (WITHOUT an option to add keyword arguments):\n", + "These blocks of RecTools transformer models can be replaced with custom classes (WITHOUT an option to add keyword arguments):\n", "- item net blocks (`item_net_block_types`)\n", "\n", "These keyword model arguments have great effect on model architecture:\n", diff --git a/examples/tutorials/transformers_tutorial.ipynb b/examples/tutorials/transformers_tutorial.ipynb index ea8ea89a..569fac00 100644 --- a/examples/tutorials/transformers_tutorial.ipynb +++ b/examples/tutorials/transformers_tutorial.ipynb @@ -1408,7 +1408,7 @@ } ], "source": [ - "models_metrics = pivot_results[[\"model\", \"MAP@10\", \"Serendipity@10\"]]\n", + "models_metrics = pivot_results.reset_index()[[\"model\", \"MAP@10\", \"Serendipity@10\"]]\n", "\n", "models_to_skip_meta = [\"popular\", \"ease\", \"bert4rec_softmax_ids_and_cat\"]\n", "models_metadata = [\n", From a78f5684fb2aaf4e1f217e035011217495e468f7 Mon Sep 17 00:00:00 2001 From: blondered Date: Sun, 16 Feb 2025 10:39:59 +0300 Subject: [PATCH 15/15] plot --- .../transformers_customization_guide.ipynb | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/examples/tutorials/transformers_customization_guide.ipynb b/examples/tutorials/transformers_customization_guide.ipynb index 0a63c47b..9d9fb048 100644 --- a/examples/tutorials/transformers_customization_guide.ipynb +++ b/examples/tutorials/transformers_customization_guide.ipynb @@ -70,6 +70,7 @@ "from rectools.models.nn.transformers.constants import MASKING_VALUE\n", "from rectools.models.nn.transformers.bert4rec import BERT4RecDataPreparator\n", "from rectools.models.nn.transformers.lightning import TransformerLightningModule\n", + "from rectools.visuals import MetricsApp\n", "\n", "# Enable deterministic behaviour with CUDA >= 10.2\n", "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n", @@ -872,7 +873,9 @@ { "cell_type": "code", "execution_count": 16, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "data": { @@ -1021,6 +1024,38 @@ "pivot_results" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "models_metrics = pivot_results.reset_index()[[\"model\", \"MAP@10\", \"Serendipity@10\"]]\n", + "\n", + "app = MetricsApp.construct(\n", + " models_metrics=models_metrics,\n", + " scatter_kwargs={\n", + " \"symbol_sequence\": ['circle', 'square', 'diamond', 'cross', 'x', 'star', 'pentagon'],\n", + " }\n", + ")\n", + "fig = app.fig\n", + "fig.update_layout(title=\"Model CV metrics\", font={\"size\": 15})\n", + "fig.update_traces(marker={'size': 9})\n", + "fig.show(\"png\")" + ] + }, + { + "attachments": { + "image.png": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyAAAAH0CAYAAADFQEl4AAAgAElEQVR4Aey9a7MUVdr3eX+DeT1fYCbifjPzoufV0xF3dMwTHd0vOrpnup8OY+xoH6cP9oG+1Ucbx0OLEtKiKIqCIC0CiiAgiKAICoI0KoogQiMHAUWQ8/nMZsOa+C28ilW565BVWVVX1a7/ikgyK3OtlSv/V27W9ct1+regIAWkgBSQAlJACkgBKSAFpIAU6JAC/9ah++g2UkAKSAEpIAWkgBSQAlJACkiBIADRSyAFpIAUkAJSQApIASkgBaRAxxQQgHRMat1ICkgBKSAFpIAUkAJSQApIAQGI3gEpIAWkgBSQAlJACkgBKSAFOqaAAKRjUutGUkAKSAEpIAWkgBSQAlJACghA9A5IASkgBaSAFJACUkAKSAEp0DEFBCAdk1o3kgJSQApIASkgBaSAFJACUkAAondACkgBKSAFpIAUkAJSQApIgY4pIADpmNS6kRSQAlJACkgBKSAFpIAUkAICEL0DUkAKSAEpIAWkgBSQAlJACnRMAQFIx6TWjaSAFJACUkAKSAEpIAWkgBQQgOgdkAJSQApIASkgBaSAFJACUqBjCghAOia1biQFpIAUkAJSQApIASkgBaSAAETvgBSQAlJACkgBKSAFpIAUkAIdU0AA0jGpdSMpIAWkgBSQAlJACkgBKSAFBCB6B6SAFJACUkAKSAEpIAWkgBTomAICkI5JrRtJASkgBaSAFJACUkAKSAEpIADROyAFpIAUkAJSQApIASkgBaRAxxQQgHRMat1ICkgBKSAFpIAUkAJSQApIAQGI3gEpIAWkgBSQAlJACkgBKSAFOqaAAKRjUutGUkAKSAEpIAWkgBSQAlJACghA9A5IASkgBaSAFJACUkAKSAEp0DEFBCAdk1o3kgJSQApIASkgBaSAFJACUkAAondACkgBKSAFpIAUkAJSQApIgY4pIADpmNS6kRSQAlJACkgBKSAFpIAUkAICEL0DUkAKSAEpIAWkgBSQAlJACnRMAQFIx6TWjaSAFJACUkAKSAEpIAWkgBQQgOgdkAJSQApIASkgBaSAFJACUqBjCghAOia1biQFpIAUkAJSQApIASkgBaSAAETvgBSQAlJACkgBKSAFpIAUkAIdU0AA0jGpdSMpIAWkgBSQAlJACkgBKSAFBCB6B6SAFJACUkAKSAEpIAWkgBTomAICkAJSnz13IezYvS/s3X+4QC4hXL48EPPZ9dX+QvkosRSQAlJACkgBKSAFpIAU6HYFegJABgauhKmzloTnX14c95cuD1TV9YudX8d4xH1zxUdV47XiwrJVn4R//49bwi9//1Ch7Lbv+ibmQ17NhgOHj4fFyz8IT06ZG/5wz/i4PfL0S+GVhSvC4aMnY7YDVwZLOnLPemHL9j1Ry1kL3q0XdVhdX7FmQ0A73jkFKSAFpIAUkAJSQApIgdYq0BMAcvrs+ZKDjpM+f8nqqircesfYUlyO2xm6AUCuDA6Gl+YvLz0z+lTaRtz3dDh6/FT4+W8fjNdHPjKlrjSkIa+/jp5cN263RLhnzPOxzDPnLWu6SE9Mnhvz+Mmv7206DyWUAlJACkgBKSAFpIAUqKxATwLID2+6O+B4ZwNf7FPne7gDyLVr18JtI58oPfOvRowJc994L6zftD2s/OeGMHnmG+GmP44uXad1iBYR0+jc+YtZCUu/T546W4q3Zt3npfPdfnDXw8/Fck9/9e2mi7p89frw0JMzon5NZ6KEUkAKSAEpIAWkgBSQAhUV6DkA+d6PbosOZqXuVXeOmhSvWZw8AIIT30hI4zfSApKmy96v2S5YaGAw8Y9X3gyDg1ezWQfu+8bytQFNABBaQSxNJQ0tA1qZiEc6um61K9TSpdI968VvBYBUuq/OSQEpIAWkgBSQAlJACrRGgZ4DEMYj4BjTPebq1RvwsHf/oXj++z8dESa+uDAeVwOQZas+jmMkiEtetBzgwFdztHHsue/Nf34kxscpJ29zdquNAcHxZzyG3Yd4k6a/Hi5eulxmvWYAhAHwBlp0laoX6MZmY2coE89N60m1gCbEeWzi7GpRhpx/YfZb4c5RE8PaT7bEVpjb//ZsfHbKSTcuBuwTGK8yevzMaEOzJS0WqT3TzA8dOREeHj+z1H3sB7+4I/zp3qfC5i92p9HiPUzrH988MpaF8rBZK87qDzfF3zO+66L16ec7wqQZi6ItH580J+a3Zt3mGGfanLfK8rcfgCewyz14Nt4L0qbjangWxpJQTlrsiMc7S1neWrFuyDtgeWsvBaSAFJACUkAKSIHhrkDPAcjxk2cCDiiOK92MLDBomHM4lgwe5jgLIHw9HzVuerzG9exGd6VTp89ZlnEPLPzu7nFD4qZpswBCmr88MKGUBufTHGPS4YheuHgDQpoBkE8+21bKf/feA2VlrvcDALPy0yKSDfsPHi1d/9f2r7KXq/7G2Sbf9FntPnYepx490vN2XGnQNzCQxseZt/jsgTwL6fns8awF78RojA3hGtAAFKXxKDfBuqllwQ67AhBpmuyxlWXMhFk1423cstOiai8FpIAUkAJSQApIgb5SoOcA5MzZ8yUH0Rx/61aEo8q4hmoA8s7760tO4auLVkYIYCzJh+v/VXKax2a++FteOJo2o9T5C5didyYb8GzlsDeHGbiIj7OcfqXfuWdfCZ6mvLTYoscv5+bIlk7WOZi7eFW8B8/caAB+7H48UzbQksF1vvA3EgxASEsLysq1G8O+A0dCFiIoMwPnaRFBExvszvl0bM+pM+dK8MHsXgZttFQ9NXV+6fmJR6AVzMbEjJv8avh636HSRgsQwQDEnp9B+Qw6B4zIk1ANQGgpsXSMtWGcDO/bhs07S8BJ+m1f7i3FA4gpH61olIf785wCkCi1/pECUkAKSAEpIAX6UIGeBBAcUZw4nEHgwbpc4SASDBqyLSA41KQZ//y8Iaama445l3T5IeA42jkczmyoNAbk2InTpTQATzbQ1Ys8U+e+mRYQQIl8+JLfTHjgsWkxfRaeyMt0AkQaCQYgjz77ypDuVHQ9o7y0XtGKlQYgxXSme5YFoIPztBhlx37QGmFpaNGxYN3iqg1CNwDh/WGweaVQCUCOHDtZut/CpWsqJQubtu6K5xctWxvjco9K3coAWMqvIAWkgBSQAlJACkiBflSgJwEEQ1krw89uvb8EIydOXXdsKwEILSfmsPIlOhtwcK3rEF/sCXylJg2OJIsFZkMlAPno062l+2zYvCM6pTimtr325vul6/a1vxkAuffvU2M+jOdoJny88UYXrj1JF6706/2BQ8caytoAZM6ilUPSMe4BLbFXpWC2YSYzCzbmhu51pl+6B55Ih70t5AUQulJVC5UAhHEt9i6Y3aqlT+3J2BC6tClIASkgBaSAFJACUkAKXFegZwEkbZ3AMbQBxDxWJQBJHWsbjJ19CczhtW5JfOkmb7rpVAqVAISuXaTJs9n4i9RhrXSfSucMwOjm1UygS5AB13Mzr7cckY+1OmRbj/LcoxaA0B0LTWjNqBSsRWvzthsAYufqacl4CwvtAhCb/IBxQnmCvUtWdrQGegAZBSkgBaSAFJACUkAK9LMCPQsgGM2cZZy8g0dudN2pBCB8OTdnsNJ0teR3y+2Pxjg2Q9KLc5bG39Wc8UoAYuMn6GpEd59am41paAZA6D5kz2P5NPoiPzNtQcwDiKEFKIUSuhE1GmoByKoPPov3qtYCYrBhAEJZ7PlYk6OWjrTmWGgXgNg79Zu7Hrdb1dzTSgLIZgfN80zAibpg1ZRPF6WAFJACUkAKSIFhrEBPA8jhoyejM/fstIVlJjJnMQWHtA8/6SoFcxZtbIB1G0rHa6TpKgHIu2s+jY4zAJI3NAMgDN42B33260MHkue5966v9pfyYLB82i2LLmuNhlYCCPemtYRnrLVeSbaMBiDVxq/YGJBGu2AZ8DViVysb44KYBjiddcveMYujvRSQAlJACkgBKSAF+kWBngaQakaqBCB84bev7JUGlKcOPUBAsDEgOMEMlM4Gm842HcidOvXpl/ls2rQVphkAIT+bHpjnqgZVxKPLGV22vvn2cLYYpbU1GNRuUxTjKDcTWg0gBhOAZKXB3FbGVMv7x74QoWXCC6/Z5bJ9swCS2qjaDFYDA1fivSqNF7JCsDYK7xOTAChIASkgBaSAFJACUqAfFegbAMG4jHXA+aM/PsBhgfEk1v0qbTWhG421ijC96/kLF2MSHE2+ytsYihRAiGDTynKdhe7SwAKCdPFKx5Wkzm0at97x3v2H4/PwTEAIs26lzjjlZUC9zWrFSujZYAOuycM2W7QvG7fe71YDCAPSrUxjJrwcp7y1MgAkXAfCaKmyYDbmmZltyoLBQbMAAsDaO0LLDFP+WuA9QXtrHWFKX96XrN50lbN3Jp2G2fLRXgpIASkgBaSAFJAC/aBAXwEIDqkBBY4tM0ixorU5hZyz1g8zPgvdmRPM3roFpeeyAMJ4lDRP0uCcGwiQlnJYaBZASE83MGvZsTJRnvQ57Xz22Uhva6hYHPKqtiK8lbfavtUAwn1simXKR9mAACAxfeYUQHD67VmwAXEBg+xChI12waIsLPhoebNnLAfPnNqaeACIxePe3AsgsTLzHmgMSLW3SOelgBSQAlJACkiB4a5ATwAIrQbm0HFcL9hA8LQ1w9IwtsEWELQ82eO0V5qel3Q2G1Yan+5B1npQaWYk7mNrdaTpOGaRvnQFbxbkszhWzkb2TD9M9ylzcC0v9px7ePzMIV/j0/wBMUvzWGYhxjRevWNr+anUxc3WWak3CL3SyutMbUw6K6PteTZaRrLT3M5fsnpIXGYnIwAipAc8qwWmESYOq9lnA3AJUFgZbA9UsLgiAdBLx3tYHPZolE57nM1fv6WAFJACUkAKSAEpMNwV6AkAaYcR6A6Ds8taHadOX19Ju9Z96MKzdcdXcVyIdcWqFd+u0XWHAfDMwoXj2eyMVZZfvT3dyeiaxKDybw8eHbKAX7303XydVoMv9+wPzJRFyw3aVgu04gCUwB0rlrc60O2K/HmHbP2Z7D2Iw8KKjBmhHLYaezaefksBKSAFpIAUkAJSoJ8U6FsA6Scj61mlgBSQAlJACkgBKSAFpEC3KCAA6RZLqBxSQApIASkgBaSAFJACUqAPFBCA9IGR9YhSQApIASkgBaSAFJACUqBbFBCAdIslVA4pIAWkgBSQAlJACkgBKdAHCghA+sDIekQpIAWkgBSQAlJACkgBKdAtCghAusUSKocUkAJSQApIASkgBaSAFOgDBQQgfWBkPaIUkAJSQApIASkgBaSAFOgWBQQg3WIJlUMKSAEpIAWkgBSQAlJACvSBAgKQPjCyHlEKSAEpIAWkgBSQAlJACnSLAgKQbrGEyiEFpIAUkAJSQApIASkgBfpAAQFIHxhZjygFpIAUkAJSQApIASkgBbpFAQFIt1hC5ZACUkAKSAEpIAWkgBSQAn2ggACkD4ysR5QCUkAKSAEpIAWkgBSQAt2igACkWyyhckgBKSAFpIAUkAJSQApIgT5QQADSB0bWI0oBKSAFpIAUkAJSQApIgW5RQADSLZZQOaSAFJACUkAKSAEpIAWkQB8oIADpAyPrEaWAFJACUkAKSAEpIAWkQLcoIADpFkuoHFJACkgBKSAFpIAUkAJSoA8UEID0gZH1iFJACkgBKSAFpIAUkAJSoFsUEIB0iyVUDikgBaSAFJACUkAKSAEp0AcKCED6wMh6RCkgBaSAFJACUkAKSAEp0C0KCEC6xRIqhxSQAlJACkgBKSAFpIAU6AMFBCB9YGQ9ohSQAlJACkgBKSAFpIAU6BYFBCDdYgmVQwpIASkgBaSAFJACUkAK9IECApA+MLIeUQpIASkgBaSAFJACUkAKdIsCApBusYTKIQWkgBSQAlJACkgBKSAF+kABAUgfGFmPKAWkgBSQAlJACkgBKSAFukUBAUi3WELlkAJSQApIASkgBaSAFJACfaCAAKQPjKxHlAJSQApIASkgBaSAFJAC3aKAAKRbLKFySAEpIAWkgBSQAlJACkiBPlBAANIHRtYjSgEpIAWkgBSQAlJACkiBblFAANItllA5pIAUkAJSQApIASkgBaRAHyggAOkDI+sRpYAUkAJSQApIASkgBaRAtyggAOkWS6gcUkAKSAEpIAWkgBSQAlKgDxQQgPSBkfWIUkAKSAEpIAWkgBSQAlKgWxQQgHSLJVQOKSAFpIAUkAJSQApIASnQBwoIQPrAyHpEKSAFpIAUkAJSQApIASnQLQoIQLrFEiqHFJACUkAKSAEpIAWkgBToAwUEIH1gZD2iFJACUkAKSAEpIAWkgBToFgUEIN1iCZVDCkgBKSAFpIAUkAJSQAr0gQICkD4wsh5RCkgBKSAFpIAUkAJSQAp0iwICkG6xhMohBaSAFJACUkAKSAEpIAX6QAEBSB8YWY8oBaSAFJACUkAKSAEpIAW6RQEBSLdYQuWQAlJACkgBKSAFpIAUkAJ9oIAApA+MrEeUAlJACkgBKSAFpIAUkALdooAApFssoXJIASkgBaSAFJACUkAKSIE+UEAA0gdG1iNKASkgBaSAFJACUkAKSIFuUUAA0i2WUDmkgBSQAlJACkgBKSAFpEAfKCAA6QMj6xGlgBSQAlJACkgBKSAFpEC3KCAAKWiJb49dCO3eTp69HM5fGmz7fdr9HL2c//Ezl8PFy7KBpw2PnroULl+5qr+DDvyfU83Oh09eDAOD12QDRxscOnExDF7tbhsUrFaVXApIgT5QQABS0MjVKupWnheAtB/y6tlLAOJvAwGIvw0EIP42EIAUrLSVXApIga5QQABS0Az1HNdWXBeA+Ff6AhB/GwhA/G0gAPG3gQCkYKWt5FJACnSFAgKQgmZoBWDUy0MA4l/pC0D8bSAA8beBAMTfBgKQgpW2kksBKdAVCghACpqhHjy04roAxL/SF4D420AA4m8DAYi/DQQgBSttJZcCUqArFBCAFDRDKwCjXh4CEP9KXwDibwMBiL8NBCD+NhCAFKy0lVwKSIGuUEAAUtAM9eChFdcFIP6VvgDE3wYCEH8bCED8bSAAKVhpK7kUkAJdoYAApKAZWgEY9fIQgPhX+gIQfxsIQPxtIADxt4EApGClreRSQAp0hQICkIJmqAcPrbguAPGv9AUg/jYQgPjbQADibwMBSMFKW8mlgBToCgUEIAXN0ArAqJeHAMS/0heA+NtAAOJvAwGIvw0EIAUr7TrJFy5dEx56ckY4fPRknZiVLz/67CvhqanzK1/UWSkgBUoKCEBKUjR3UA8eWnFdAOJf6QtA/G0gAPG3gQDE3wYCkObq6ryp7hnzfPj3/7gl7NyzL2+Ssnik/f5PR5Sd0w8pIAWGKiAAGapJQ2daARj18hCA+Ff6AhB/GwhA/G0gAPG3gQCkoSq64cgCkIYlUwIp0JQCApCmZLuRqB48tOK6AMS/0heA+NtAAOJvAwGIvw0EIDfq33YcCUDaoarylAJDFRCADNWkoTOtAIx6eQhA/Ct9AYi/DQQg/jYQgPjbYDgCyIo1G8If7hkf1n6yJbww+63w898+GLtB3XL7o2H1h5tinbxs1SfhN3c9Hr73o9tiF6exE2eHCxcvD6mv16z7vBSPuLeNfCJ8+vmOIfE4sX7T9vC7u8fFPC3uj28eWbEL1tHjp+LYkJ/8+t54nf0Tk+eGc+cvluXdSBesSTMWxefe9dX+8NjE2cHy/uXvHwpr1m0uy3fX19+GO0dNinEoK9vPbr0/THxxYTh56mxZ3FbquXXHV+EvD0wIP/jFHfG5KdvcN94LV69eK7unfkiBRhUQgDSqWCZ+PXhoxXUBiH+lLwDxt4EAxN8GAhB/GwxHAJkxb1l0bnHe2X54090lCOE3IMIep/umP44uxZ2zaGVZjTz91bdL8XDW7xw1sRSXweVpWLRsbenazX9+JIy47+mSk8290jEgHHNvzgMo9z36j9JvYOnK4GApa+LkHQPyp3ufKpWBdACFARC/gQ4LgJjdHyDg+QwK0CQtQ6v0XLz8g1L5fjViTJmeAJOCFCiiQE8ByKkz5wI0/sln28K+A0cqPjdxKm3nL5R/pSAxXzQ2bN7R9GwX5NEKwKiXhwDEv9IXgPjbQADibwMBiL8NhjOA4Ehv3/VNqW6fOmtJdIBx/nGG7as7jjnOOK0bFr49eLQUd//Bo3Y60LpAXPI4cepMPH/85JkSQGzcsrMUl4Pb//ZsjG8Awj0BFPJ474ONpbgDVwZLcd9asa50nniNAgjdvtJZt2jdIR9aNyzQysEzpmFg4EqpbKluBiBF9MQ/QjO21N86c/Z8fD7Kly1PWjYdS4F6CvQMgED7vPDpRnMs/5GkIb2eHtO8a4FmW/tPxuLwH1klSLE01fb14KEV1wUg/pW+AMTfBgIQfxsIQPxtMJwB5M0VH5VVtTjV1NF88c8GWkm4hhNOmP36ivgbaMmGxyfNideWrrwOCvZl/9Y7xmajhuwYkN17D8S0dD06f+FS2bbknQ/jtXGTXy3lQ5kaBZADh46V0nMAFJEPZckGnnfzF7vDslUfh1cXrQw8A3HfXfNpKaoBSBE95y9ZHfN9csrcsmdGA6Yp5p6rPvisdE8dSIFGFegZABn5yJRA8yp9Ofnje/7lxfEPgCbWNPBHQZ9OvkikG60mFsZMmBXTLnjr/bBn74H4ZQXKv3/sCxYl974VgFEvDwGIf6UvAPG3gQDE3wYCEH8bGICcm7EynHrjk460wtero7LXc1eg30Ws5jB/8+3hWFfTPSobAALqe/twyPob/LYxI2n8N5Zf7241eeYb8TTrdBB35rxlabR4nAUQWj2IW2vjA6kF4hUFEGvhSfPlOR94bFrVcjBGxkIr9DRoq/XcAJCCFGhWgZ4BkEoPSN/L7B86fyy1+iYyYIw4jzz9UlmWUD7nrUUFMOHLwudbd5XFe2bagkj/djL7H287fgtA/Ct9AYi/DQQg/jYQgPjb4Ni/9oUwenYIv38mbpcfnR8O7DvZVSBi9WPefTWHma5U1Mt5AMS+yjOQPRuspcK6ND08fmbMlwHr2ZAFED5kUgZ6UdCCUmlLu3ERN+uXZO9hv20MSLYFBP+DfFIA4cMq5xgPs3LtxvD1vkNxADyD9jmfB0Ca0ZPWnUrPzLm9+w/Zo2gvBRpWoGcBhH6I/NHRNzMN1/9oJ4aPN26L8HD67Pn0cqk/KK0faaA5k7S0rhA2b9sTf2f/g+I/DMDHQjuAI5unAMS/0heA+NtAAOJvAwGIrw1OLt8Urv3nlBJ8GIRw7ugnu7sGQqx+zLtvBYBMn3t9ADpdsbLhuZmLYn1uA9H/8cqb8TezOWVDFkAYd4pvQN2fJxC31QBi/g49NRh7koZpc9oDIC/NXx6fG9soSIF2KNCzAGL/2cxbsqpMF/74sxtT3VmgKxbXAY40ABqctz6NAhDfijYLYd6/BSD+74MAxN8GAhA/GwAYBhzV9t3SEpLWrXmOWwEgW7Zf/2jI2JB0el4mpQEIqN/ti/2H6/8Vf/Mx8eKlG1P5MqiamaiIa4PQuW7paXnIBsapMJ2vBdK2GkAOHD4eywSApOXlAyutQ9xz+er1VoTQCj1t/E12EDo3uXbtWhyQnw72L91cB1IgpwI9CSAffbo1/sExLVw69RzPzBcOIMMGafEfDH+cNMESDDSYJzsN9h/SO+9f/yPOCyCXBgZDu7eBwath8Oq1tt+n3c/Ry/kPXLkaZ2Dp5Wfo9bJfxgbX9HfgacfLA7KBp/7h9qGtHwYj1x6e3TV1RFq35jluhcPMfe79+9RY37OeBuM7GDdqg9UZ02ABB9p8A+LSbZuB7vgKthmAkMb8Bq4Rj25PdNu26YHTge/EaTWAUAZbIwRAYv0RWmqAAytvq7tgcc/xz88r5U8XNz780n3dypJ2PTNttZcCeRXoOQBhTAZ/cMyVnV18p9JDE4f4DGInGFik0+Zxni8bxFu38YuyePW6YB07fTm0ezt34Uq4NHC17fdp93P0cv5nLlwJOMC9/Ay9XvbT5wbClcFrskEH/s+p9q6cPHc5DF4NsoGTDS5NXVa1FeTcm592jV1iJdrAP8AC9W+2XmZsBOcrzYJl64EwK5MFZoiy8Zyksw1AGOTFTQLT3loeFo9B3syIye8v9+xPYoewaeuuYAPfLT57xmZwzQLnWJ8jT7AxIAePHC+LzvgO8rnr4edK5xmYnq4RwnVaP0aNmx7j2sdTErRKT0CNGcNsvRHuaRuwl04dXCqoDqRATgV6CkCYAYuXH/pmjuq8ga8RNt0e81mTx4tzlpYltyn8duzeF88bqNQDkE50DdIYEL9uD2ZfdcHyt4G6YPnbQF2wfG1wfM22qgBy6MvDPTsGpKwybsGPy5cH4noitGJkx0xks6d7E923UpDJxkl/0wWKtUXozsV9Ohno8cG96eFx7MTpTt46rq9GtyxgKQtzHS2IbjZsFOgZAGFsBuDAFwhbTChrBQZqQexp4D8J0tmsV/zhpEBicfmSQHOm/YdiAJKOFSFvun1pELpvJWxQ0Mm9AMTf5gIQfxsIQPxtUGkQ+sCoV7oGPvh/WUEKSAEpUE+BngAQiBuIsJYLBp6nG1PWEWYteCcOIJuzaGWcBYvmXOv/CblbYCo+8mJKXZpOp7x0fU2RdPpeAxDiMWc4s2ZZc6kAxL8S7iR8cC8BiL/NBSD+NhCA+NuAaXivfrEvznrFwHS2bmr9EICYp6G9FJACtRToCQDZu//6YkQGIdk9fRQJa9ZtHtJXkb6L2YWJaJLNLujDGJF0dgkDEJthgnsCHgwAE4D4V8ICkP6zgQDE3+YCEH8b2EKEnf4/sJH71XI6dE0KSAEpgAI9ASCNmIpuUowPocWDAVLZLllpXkzVRx/RSn0/DUAYA0K3rGpjThr5T7nZuBoD4l/pqwXE3wYCEH8bCED8bSAASWtxHUsBKdCrCgw7AGmVIVIAqZVns1DRSDoBiH+lLwDxt4EAxFAsflcAACAASURBVN8GAhB/GwhAatXIuiYFpECvKCAAqWIpW9QoOwtWNnojINFsXAGIf6UvAPG3gQDE3wYCEH8bCECytbB+SwEp0IsKCEBqWI0p72p14SJps1DRSDoBiH+lLwDxt4EAxN8GAhB/GwhAalTauiQFpEDPKCAAKWiqRkCi2bgCEP9KXwDibwMBiL8NBCD+NhCAFKy0lVwKSIGuUEAAUtAMzUJFI+kEIP6VvgDE3wYCEH8bCED8bSAAKVhpK7kUkAJdoYAApKAZGgGJZuMKQPwrfQGIvw0EIP42EID420AAUrDSVnIpIAW6QgEBSEEzNAsVjaQTgPhX+gIQfxsIQPxtIADxt4EApGCl3eLki5atDS/NX17Kdf2m7eHJKXPDpcsDpXM6kAJSYKgCApChmjR0phGQaDauAMS/0heA+NtAAOJvAwGIvw0EIDeq6PMXQnh39dXw+DNX4jZnwWA4evzG9U4csVgxCxRbmD737cDCxWfOnrdTLd9Pm/NWGDtxdsvzVYZSoJMKCEAKqt0sVDSSTgDiX+kLQPxtIADxt4EAxN8GApDrlfbe/dfCyIcGwm/+s3wbcc9A2L7zWsGaPX9yDwC5c9TE8JNf35u/kIopBbpQAQFIQaM0AhLNxhWA+Ff6AhB/GwhA/G0gAPG3gQDkeqU9auyVIfBhMAKE0DpSNCx558Pwy98/FL7/0xHhez+6LbZ0zFrwbtn0/NUAZOXajeF3d4+L6cgju6bY+QuXwrjJr0aQIO9b7xgbNmzeWSrynr0H4rlNW3eFpSvXhb+Onhz+cM/4QOsH8WllIQ3bczMXldLpQAr0igICkIKWahYqGkknAPGv9AUg/jYQgPjbQADibwMBSIgtHAYb1fZr110tWLuH8Oizr0QAeWzi7DBpxqJw858fiY7//CWrS3lXAxAAgfgPPDatBAw7du+L6Vhj7Oe/fTDmBVhMeOG1CDmk+XrfoRhn87Y98Trww3mgg23ektXhB7+4Ix6TN9vcN94rlUcHUqBXFBCAFLRUIyDRbFwBiH+lLwDxt4EAxN8GAhB/GwhAQgAuqoGHnV+0dLBg7R7CwMCVsjwAB5z/vzwwoXS+GoCkLR5f7Pw6QsQjT78U072xfG38vfrDTaV8Dh89Gc89PmlOPGcAAsQYuFhkdcEyJbTvZQUEIAWt1yxUNJJOAOJf6QtA/G0gAPG3gQDE3wYCkHwtIO+sKt4CgnsACEydtSR2gaK7E60RN/1xdMlzqAYg6SD0q1evxRaLX40YE9PdP/aFmM/CpWsCMGIbLRx0s7L7cq8UZOymAhBTQvteVkAAUtB6jYBEs3EFIP6VvgDE3wYCEH8bCED8bSAACXF8x4iR5YPPreXD9q2YDeupqfMjKAAGf7r3qTBmwsuxBaRRAMHNuOX2R8OPbx4ZPQ5ABLige1V2s9YVawERgBR00pS8axUQgBQ0TbNQ0Ug6AYh/pS8A8beBAMTfBgIQfxsIQK5X2hs/v1a1G1Yrul8dPX4qQgKtDRcvXS55CreNfKLhFhASMxAdCCHc/rdnI8iUMq1wUA9ADGYqJNUpKdATCghACpqpEZBoNq4AxL/SF4D420AA4m8DAYi/DQQgNyptpttNZ8MaOWogACatCCwoSCsF3aQsXLt2LQ4sb7QFxMZ32NodDGgnb2a4yobjJ8/EU7UA5J4xz8eWk2xa/ZYCvaSAAKSgtZqFikbSCUD8K30BiL8NBCD+NhCA+NtAAFK50m7FtLtpzgxAp+sViwwyDe5bK9bFaXUBhzwAMuWlxREw1qzbXJrx6ptvD8dbnDh1JuZN9ytm1AJEmPKX8SQPPTkjxqkFIORNOV6cszR8vHFbWPvJlrToOpYCPaGAAKSgmRoBiWbjCkD8K30BiL8NBCD+NhCA+NtAAFKw0m4g+ZsrPopdpXD22QAEFgBkZioLjNlIV0KfMW9ZjPvDm+6Oe9IBMtmxHMxsRZcsy5s9M2xxT8KW7den4c2m4xoAw/S9lpZjBSnQawoIQAparFmoaCSdAMS/0heA+NtAAOJvAwGIvw0EIAUr7QaTDw5eDTv37AvWNSpvcma+2nfgSNi990Bg+t5qgfElrP1x6vS5alGqnj93/mI4dORE4F4KUqDXFBCAFLRYIyDRbFwBiH+lLwDxt4EAxN8GAhB/GwhAClbaSi4FpEBXKCAAKWiGZqGikXQCEP9KXwDibwMBiL8NBCD+NhCAFKy0lVwKSIGuUEAAUtAMjYBEs3EFIP6VvgDE3wYCEH8bCED8bSAAKVhpK7kUkAJdoYAApKAZmoWKRtIJQPwrfQGIvw0EIP42EID420AAUrDSVnIpIAW6QgEBSEEzNAISzcYVgPhX+gIQfxsIQPxtIADxt4EApGClreRSQAp0hQICkIJmaBYqGkknAPGv9AUg/jYQgPjbQADibwMBSMFKW8mlgBToCgUEIAXN0AhINBtXAOJf6QtA/G0gAPG3gQDE3wYCkIKVtpJLASnQFQoIQAqaoVmoaCSdAMS/0heA+NtAAOJvAwGIvw0EIAUrbSWXAlKgKxQQgBQ0QyMg0WxcAYh/pS8A8beBAMTfBgIQfxsIQApW2kouBaRAVyggAClohmahopF0AhD/Sl8A4m8DAYi/DQQg/jYQgBSstFucfNGyteGl+ctbnKuykwLDXwEBSEEbNwISzcYVgPhX+gIQfxsIQPxtIADxt4EAJISrB74JA/98p+Y2uHtbwdo9X/IR9z0dfnbr/fkiK5YUkAIlBQQgJSmaO2gWKhpJJwDxr/QFIP42EID420AA4m8DAUgIl95+LZz8b/+l5nZh2vjmKvUGUwlAGhRM0aXAdwoIQAq+Co2ARLNxBSD+lb4AxN8GAhB/GwhA/G0gAOkcgCx558Pwy98/FL7/0xHhez+6LbZ0zFrwbrh27VrJc8gCyIWLl8OEF14LP/n1vaU0j02cHU6dORfT5MnzwOHj4b5H/xF+eNPd8d43/XF0mD737dI9f3f3uDBvyaqwaeuu8PD4mYHf23d9E6/vP3g03PXwc+EHv7gjpuWY/NJw4NCxcO/fp4Yf3zwyxvvTvU+FD9f/K42iYynQdgUEIAUlbhYqGkknAPGv9AUg/jYQgPjbQADibwMBSOcA5NFnX4kAAkBMmrEo3PznR8K//8ctYf6S1SXPIQsgo8ZNj3Fw8CfPfCP85YEJ8fcXO7+OaerlOXBlMIIBwDN24uzwzLQFpfvaTSkDUMTejgEIwIJ0bJTZygLIDAxcicm/PXi0FIf8H3n6pZjXbSOfsOy1lwIdUUAAUlDmRkCi2bgCEP9KXwDibwMBiL8NBCD+NhCAdA5AzGk3N+HK4GBsMQAqLGQBBOf/1jvG2uW43733QDh56mw8rpfnjt37IlQ8//Lisjw2bN5Z+m3QsfrDTYEyWbh/7Asx7bETp+1UWLbq43juvQ82xnP3jHk+/t725d5SnPMXLgW7XjqpAynQZgUEIAUFbhYqGkknAPGv9AUg/jYQgPjbQADibwMBSOcABPdg87Y9YeqsJeGvoydHsMD5p0uUhSyAWCvJKwtXhH0Hjli0sn2tPM+euxABge5Ry1evDydOnSlLyw/K8OSUuUPO0ypC1683lq8tbTPnLYvxrQsXcWgRUZAC3goIQApaoBGQaDauAMS/0heA+NtAAOJvAwGIvw0EIJ0DkKemzo/OO60ajJMYM+Hl2AJSC0A+/XxHjAMksDEWAyCwcSN58qSLF/e0PICajVvKW0CyAHL58kApPpCR3QCQi5cuxzgPjnuxoOej5FKguAICkIIaNgsVjaQTgPhX+gIQfxsIQPxtIADxt4EApDMAcvT4qeis3zlqYnTczVVgrEQtACEeA9EZkzFp+uulsRpbtu8JefMkj1Onz8UWEMZoACIACZBB4HcWQDhPnHpwQVqARkEKeCsgAClogUZAotm4AhD/Sl8A4m8DAYi/DQQg/jYQgHQGQNZv2h4d/YVL15S8BFoxcN6rAQjXDx89WYrPgY3pmDbnrZAnzzNnzwfGZKRhzqKVsSx03SJUA5BfjRgTgcdm3LI8GCdi5yg76U+fPW+X4x44UpACnVRAAFJQ7WahopF0AhD/Sl8A4m8DAYi/DQQg/jYQgIRw5fNPwrkJD9XcLq96q1DtzmBxWhRYZHDpynXhrRXr4nS3OO/VAMS6QTEN7yefbYvT5D7w2LTr8PDF7jgTVb081238IkLE7NdXxPEntKQwFTD3pVWEUA1ASMu1n//2wbDynxtit61XF62M40IoP4GB68T5zV2Px1YaoIhWlj/c05l1U2Ih9I8UCCEIQAq+Bo2ARLNxBSD+lb4AxN8GAhB/GwhA/G0gAClYaTeQ/M0VH5WN52DAOYO80y5MzIhlK6HT0nD7356NDj5OPhtxF7z1fumu9fLcu/9QBBxLz57xJx9t2FrKg3Pjn59X+p0erFy7MQ4yT9MDMDYNMHFp1QGELA7HL81fnmajYynQdgUEIAUlbhYqGkknAPGv9AUg/jYQgPjbQADibwMBSMFKu8Hkg4NXw849+8Lxk0Nno6qWFWt5sCCgtVhk4+XJk3EkX+87VDb+JJtPrd90uSJ9tjtXmubIsZNxkULKoyAFOq2AAKSg4o2ARLNxBSD+lb4AxN8GAhB/GwhA/G0gAClYaSu5FJACXaGAAKSgGZqFikbSCUD8K30BiL8NBCD+NhCA+NtAAFKw0lZyKSAFukKBngIQmhS37vgqDu6qtsAPqjJ4jFU+6fNo09ZVUptZHzZs3jFk1opKcaudawQkmo0rAPGv9AUg/jYQgPjbQADibwMBSLXaWOelgBToJQV6BkDuHDWpNGDKBk4xi0O2XyYzRth127/3wcYym9C3MjtQjLm9z1+4WBYvz49moaKRdAIQ/0pfAOJvAwGIvw0EIP42EIDkqZkVRwpIgW5XoGcAZOQjU8L0V98OrDK6+Yvd4fmXF0fQYJEgC4eOnIjnmDGCOP/a/lXgOiDCzBIWxkyYFc8xM8WevQfC4uUfxBkh7h/7gkXJvW8EJJqNKwDxr/QFIP42EID420AA4m8DAUju6lkRpYAU6GIFegZAKmnIXNff/+mI0qVJMxZFsEgXAqLbFgDyxOS5Md658xfjb+a9TgOrihLPWlQAk1vvGBs+37orjRaembYgPPTkjNK5ZqGikXQCEP9KXwDibwMBiL8NBCAXwsGtO8Kxv/6/Nbej4/4WGvk/vpG4ApBS9asDKSAFeliBngUQVgsFGNL5uO96+LkyIDG7/PjmkaVFdnZ9tT+mS+flJt6yVR/H87ScEFhxlPzXrPvcsol7WlcAHwuNVBzNxhWA+DteAhB/GwhA/G0gALkQDn62OZz8b/+l5nb8f9wiALFKUnspIAWkQAUFehZAps99OwLCvCWrSo91y+2PBmAjG1iExxYKYnVSwALgSAOgwflVH3wWTwtA/J2dZoGtHekEIP7vgwDE3wYCEAFInv9f07pVx1JACkiBSgr0JIB89OnWCAu/GjEmsPKoBUCDVUqzgVYSAxMDjRVrNpRFs8Hr77y/Pp7PCyBXr14L7d6uXbsWrl0Lbb9Pu5+jl/OXDdr/ntd9P/g7CPo7qKtTO/9P4j+iPrfBwO4dNVs/aB0589f/3r7/r3vABmWVq35IASkgBSoo0HMAwpgMWioAipOnzpY90h/uGR9+eNPdZef4QesHsEIwsHhrxbqyeCvXboz5rtv4RVm8el2wDp64GNq9nTo3EC5cHmz7fdr9HL2c/4mzA+HSgGzgacPjpy+HgStX9XfQgf9zqtmZVqiBwWt9bYNDn2+pCyAn/sctbdPoyMlLYfBqd9ugrHLt0I+Tg5c6dCfdRgpIgVYo0FMAwgxYwAetHKzhkQ0PPDYtXh+4cqNVhK+F3/vRbXE2LOKzfgh5vDhnaVny2a+viOd37N4Xzxuo1AOQPM3RReNoDIh/1xN1wfK3gbpg+dtAXbDUBStPfVZWuXbgxy1frQz/+9a5QRDSerFZtmDv/sOBCXy8wlNT54fVH26qeXsmHKKcaa+YbAJmSmXCoV1ffxsvdcOz0buCch87cTpb3I78ZtKmqMlX+ztyv/QmPQMgjM0AHOhmdeLUmfQZSscLl66JcVhc0AILF5Ju1oJ34qnBwatxoDozXKVhxH1PR1CxhQsNQNKxIrwotKRoELq/I5SnEmxlHAGIv80FIP42EIAIQPL8v5rWre0+Bj7+7dPn4tbLEPLtwaNxfbJNmZk3261fmv89Y54P2Ql61n6yJfpQ2fNpunYf48PhJNcKQArxmMG0WmBpBuIYzHT62abNeSuMnTi7rHisP0eZ0N4jsGA398+ul9eJsvQEgBw8cjwKhEi0XDDwPN3shbMpd+meRcsFLxfdr0iXtphMfHFhPMeUuvyxT3np+poijyUvhgEIaXmx+eNjBix+C0D8HaE8lWAr4whA/G0uAPG3gQBEAJLn/9VOOC/cI4WPXoeQ7bu+if7Fu2s+7ZR8Q+5Db5FHn32l7PyBw8fDq4tWlloNyi526Ad+Vz0AoXs+5Tx99nzVUmUBpNPPxrp02XHKtNjMfeO9wDhkjyAAqaM6zVO8gNU2FhK0AFCwNojF5Q/q443b7HLc00XLumtZPBY6vHjpcimeAQgtIxYH8ABoBCD+jlCeSrCVcQQg/jYXgPjbQAAiAMnz/2qpIm3jQSX4aDWE8PHx4fEz46yZTGaDP8GenhVpOH/hUhg3+dXoXBKHHhYbNu+MUfA3cDwZo8pHUgtnz12IHzXxReh+g1+Br4GDSnq2/QePWvS6exx00nJ/fCBmBeUjbBroxfHam+/Ha8ThnhNeeC067XyB5/6ct/vzVZxFnPm9ccv15yE/nObnZi6K/hD3+81dj8dFotN7/e7ucWHWgnfjB178JuKhQzNdjSjX6PEzw/jn58Xxv5QR3Wg9sMAEQpQz7SpGV6vbRj5R0oQykZe1gFR6NuLwgRtfEtvzGzgkYA+We/jBL+6IOnEMxKSBHjh8rKaMfAynnHwknzFvWSwH9zd90ZDAuwE8pYHeN/bO0fNn7uIbM74SL8+7iRb4sIyNtvfizlGTylqJBCCp6i06pjmT8R78wVUL9P/buWdf4D+PbDAAoSWFbllpC0oaN89/xkXjaAyIv+MlAPG3gQDE3wYCkAvhwIET4dCnn9XeNm/TOiBpRdni41rw0UoIwTHEYWTj+N6/T43HOPcWcMYNHv46enJ06HE+SfP1vkMxmnUh57qF+x79R4yzZfueCCb2sZO8cVrZ0kWVLV21PU4+zjaOKRtOMmXAx7FALw/O4QADHvZ8tB7g3HPNnGbuz8fbSg6qwQr7519eHMGHtCxzYIHfbDi+PDdl47c53RYvzz7Ni3tSfs7hTFugmz3nTp2+DnkHDh0r3Z+Fp3l2no04BiCVno3rZj87pnWC/HgWNvIaNW56zAvnfmDgSiwGs6uShvtgA9OJnjuLlq2NNiG92ZeWDwJpHp80xx4l2HACuvxPnbWkZCd661gw25G22rsJ7Jn+k6a/HtCB+Lwbly4PxKwqaWD3aPe+J7pgtVuESvmnAFLpup0rChd50gtA/B0vAYi/DQQg/jYQgPjboN9XQh/17SelMR8GG9X2/7H9+hdmq68b3ePY4YymTjwOIU6cdfV5Y/na+NucWu4BOGSdyjETZsVzS975sLTw8SsLV5SKVLQLljnBliFf9ymD3YOv8Py+/W/Pln2YpWXDvuLjrGa7YGUdVLQgn7RLFC0R6MQXewvEAabS2UppKeFrfqOBvGiJoNXIwl8emBDLYZCWBRADhLS1KtsFK/ts5M29eBbsmQ5ov3/sC/Fa2oJjC1jTUkRcaxlJP2oDLtZ6VKkLlt3TAAQ7cv+b/ji6dH8+pNvzGmDleTdJl07KxL0MbrZ9uTdKWUmDeKED/whAqojMVwleRFpAaoU8AFE0jgDEv9IXgPjbQADibwMBiL8N+h1AmOnqf9s6ty6E/E+fvRA+u5C/C1Oleh4nj9aNNNjA5fWbtsfT5pji2AEjtuHMk94CX5xtTCrXcEbTHhqFAeTKYFizbnMEA/IGBvBhGOtKMMfzzRUfWZGG7PMAiOVjM0lZJk9MnhvvZyDEvVNIIR6tApzPOsWWR7V9pbwAK87b0glZAKFlAiBIQ14AyZabPIACuriZfdnPnLcsloGFsQ34AJ9qIQ+A0GrGc6WLbJMfQMR5e+/yvJuko9sf3e4eenJGfB+tFcjGnAhAqlnL+TxEm/4HUak4ReEiT3oBiH+lLwDxt4EAxN8GAhB/G/Q7gFAP14OQVsAH96nk5NlyAOb40k0GxxAHNbvx1ToN9mGT+NnZPIsACC0QtC6QL443XZOsu5UBiE2+Q/f0aiEPgEyasSjeJ23ZID/GOHB/a03hOOvI24Q/NttotXJkz1fKi+5enF+26pMYPQUQfDeu0dUpDc0CCOUlP7asjfkNgNgC2bRwVQt5AIT3ivuwNl0a7N2xNezyvJu79x6IEEZ+tDzR7Y93g98CkFTdHj3OAxBF4whA/Ct9AYi/DQQg/jYQgPjbQABy3VmoBiGtgg/uksfJo0tT9kt7NXcGhxjnj40xGGkwAGEwdaOBNOQ5/dW3yz6acs4AxFoMPtqwtWr2AAhdxdKQ/UJu+QBiaTAwsW5S3LudAEJrD/ewFoEUQCgX1xh3koZmAYQ80ObBcS+m2ZUd06WJezLWoloAQGiByAbSWRcsyycd70F8AxPrlZPn3bRuf+nUzhxzPwFI1go9+LsoXORJLwDxr/QFIP42EID420AA4m8DAcgNRyELIa2ED+6Sx8kzxzt18qyEx0/eWLOMLjs4fvOXrA7Mupk6gcS3MRp0cWo0TJ75RsyPbkAWmNWTexiA4HDym8HmaWAyHptNiq/5DJxOQxZALB/umQa+sJPeAvdqJ4AwVoV72ARBWQCxmbdYjNoCsEIaG6+TfTbiVSo352np4vnSmcw4T2sL55hxirSVxrjYuBG0TTWycpHOAOTM2fMxn3SiA+JZ9zVmhSXkeTfJI3s/a6npCQDhJWagDc02DIqB9CFBmvnoY8jsBiv/uaE0C4AJ2i/7PABRNI4AxL/SF4D420AA4m8DAYi/DQQg5d6FQUir4SOvk0dXKnwiHD3gAhChGw6zWtHvnrDrq/3RqaS1hICTab6UDaIGBHBEGXOCk4jfle2mFRNX+MdaT/jCjnPNlK34aORnAMIizDjHnGN6XGa+YmYmvshzTGB2KZ5l+er1cTwJX+OzTjr5mHNPGbluMDAnmUqW+7QSQNAFX5OyMvMW+Zu+lD0LIDwjcXD6GSROiwLPxrlmAMRaIKwcDCxn6lzGhVi3KBsHQ8sD15n9jMHzzIJFsC5o/GaGMZsmmTIZgBCPxQo5x4QH2JbZsviNfS3kAZDZr18fJ0NrGy0nqQZdDyAQOX9UPHi9jT8mRvv3WygKF3nSC0D8K30BiL8NBCD+NhCA+NtAADLUywBCPis44HxoriGu55D9os06D/hD6fpiO3bvizM+pX4SPhEDvhk/gJOKL2UzGHEvHFTi4/RbAFxskDDXsgO9LV6lvTm/VgbWJeH42WkLS9FZ1Nmm+7V4fNm3cRuMM+CruV0DqAxAcKYtsB6GDXK3uLQE0RpggfPZ1hYDBxuobnHr7ckLPe1e7FmDw7p7kd6Aw1ooGPRv0yYTH/gAWDjOAkj6bFzPltvKx7gMAzvisfF+oBEBiAS67Bp74lu3N4CSj/Z23aZl5jf2ssAsWnT3snjsgdf0/WGtkXrvJlrY7FnkgQaWbxZAUg2sHO3eV50Fi5YPMzgEicF4SXkgREYIXkLmPbYH5I8sbe5qd+G7If88AFE0jgDEv9IXgPjbQADibwMBiL8NBCDdUPNXLgN+E7MYpY5i5Zi1z+JrpQvq1Y594yqO6/W1zW4s0Hfj6o0jykl3rWr3oMtQntYX4ny5Z3+ErBu5t+8I4AD2GtGXMjIYO4WjoiXED8bO6XS7aZ7cC/847YKXXkf3Q0dO1PWXzZ60mBUJdFOjQaGVGhQpj6WtCiCbv9gd6Ss7J7QlTPfMFGV9GtM+iGmc4XpcFC7ypBeA+Ff6AhB/GwhA/G0gAPG3gQBkuHoTQ5+Lbja0iNTa0haUoTl075nh/Gzdq3p3lawqgNhcz9ZHrV6x6QNHE0926rB66Xr9eh6AKBpHAOJf6QtA/G0gAPG3gQDE3wYCkF73GvKXn6/sfPGvtfFluxfDcH62XrSHR5mrAogttsMgmzzBZmGgD2E/haJwkSe9AMS/0heA+NtAAOJvAwGIvw0EIP3kYehZpcDwVaAqgNB3zQatMINAtUFDDLCy1g/iM/Cnn0IegCgaRwDiX+kLQPxtIADxt4EAxN8GApB+8jD0rFJg+CpQFUB4ZOuGBViwMQ0vU4ox6JwFXvht19gzIL3fQlG4yJNeAOJf6QtA/G0gAPG3gQDE3wYCkH7zMvS8UmB4KlATQHhkpupi7uEUNLLHDEBnarp+DHkAomgcAYh/pS8A8beBAMTfBgIQfxsIQPrR09AzS4Hhp0BdALFHpmsVc0gz3zELsrBnGjG6YPVzKAoXedILQPwrfQGIvw0EIP42EID420AA0s8eh55dCgwfBXIDyPB55NY+SR6AKBpHAOJf6QtA/G0gAPG3gQDE3wYCkNbW4cpNCkgBHwUEIAV1LwoXedILQPwrfQGIvw0EIP42EID420AAUrDSVnIpIAW6QoGWAciadZ8HFpY5XXDFxq5QpYFC5AGIonEEIP6VvgDE3wYCEH8bCED8bSAAKa+gT+69Fo5sv7GVX9WvogpcuMiq6YerrppeNP9OpmfRbJ6Fld5rhU8/3xGenDI3sGI8gZXPSee5kri3HRjnjSaUo1WhZQAy94334kB1lnzvp1AULvKkF4D4V/oCEH8bCED8bSAA8beBAKTcw1j12JUw/zcDpa38au/8+vbg0XD7354Nm7buciv0PWOeDwveer/s/rYmXPZ8WaQeynrsxgAAIABJREFU+XH+wsXop/KctcLMectiPMCD8NTU+fF3JxZ9rPYeeNth1oJ3rmty+romtfTLe60qgFy9ei2cPXch9zZ97tuxcAKQ1ldQApDWa5oH/NI4AhB/GwhA/G0gAPG3gQCk3L0ZLgCyfdc30Yd6d82n5Q/YwV/f+9Ft4dFnXym744HDxwMLUu/6+tuy8734gxYMPpYzu2utkAWQz7fuihp0oodPtffA2w4dBZDNX+yOfwzZKXfr/RaAtL6CEoC0XtMULvIcC0D8bSAA8beBAMTfBgKQctexnQDCl++Hx88My1Z9HG7+8yMBB5391h1flRXi/IVLYdzkV8NPfn1vjHPrHWPDhs07Y5yBK4NxKQO6qNsXdS7wgfdP9z4VHnhsWuwS9PPfPhh9LvIgPdv+g0fL7lPrB91j7P7f/+mIcMvtjwa+mqeBLkivvfl+vEYc7jnhhddi13laBfDvOG/3ZxHqvfsPxd8bt1x/HvLDkX9u5qLws1vvj8/7m7seD3RbSgNrxs1a8G6Y8tLiUjyWdKjX/SnNg2MWwaY8b674qOzS3MWr4np0dhIbEY9yoCu2+vHNI8OSdz60KHGPHQCqNKxcuzGua2dpfvn7h6IWZq933l8f8z53/mJMRksI96K1aunKdeGvoyfHIQjMCovG5M86eeSHxmjOeQvV7PDVvoMxPnbIvgeV7HDi1Jn4/vzwprvDD35xR7jr4ecCoJKGPHZgVluehzwoM/vHJs4uG1LRUQBhyl1eKgMOXrRaGy8tcQUgra+gBCCt1zQPdKRxBCD+NhCA+NtAAOJvg34HEMZ8rH78SmlbNOJG9yu6YqXXvlp7NfXFGj7GWTUfiON7/z41/sa5t4AzbvCAI4pDb/7Q1/sOxWirPvgspuO6hfse/Uc8t2X7nggmI+57upQ3UMJ2+OhJi153j3/GAtFAExtOJGXfuWdfKS1OJedwNimnPR9f+Mc/Py9ew2m3+3+8cVtccoE0wIgFgxX2z7+8ODrLxPnks20WJebFORxanpuy8RtwaSQwDoN0U2ctKUv2xOS58bydtF44xAUgHhz3YskO6bgFrj8+aY4liwtocw4nHogkHWXmnAFI1vnevG1PvG52Jj4b9yFv0v5qxJgIX/ZuLFq2tnTPanb44JMtodp7ACSQr9mBpTEoM+eAz2emLYhloBynkm5SXGfjfDU70OrG+/LQkzPCpBmLYjzSpO9rVoPSwxQ4qNoFizzphoWIFKRe0BiQ9lVMApD2aZtCRq1jAYi/DQQg/jYQgPjboN8BhAHn6ZiPWsdbFw3Wc11qXsdBx8lMnXgcYXwi647zxvK18ffqDzeV8gIciJM6umMmzIrn+CLP13quv7JwRSlNta43pQh1DmgpSANfzNN78NWe34wzSb/G07JhX81xUrNdsLKOL1qQD06vBcZWoBOtQxaIA6idPHXWTsWP2sBBI6FRAEFbCzwb5VizbrOdir/NLvi4OPGU3Vo3iJjtgpV1vg1AeN4du28AnmkOHKWBewCIhHp2qPYeZO1AK8v1Z/u8dCuL8+y0haVzeexAK136TpDYANnOZzUo3aDAQU0AId9/vPJmfMh69xCAtK9iEoC0T9ta0JFeE4D420AA4m8DAYi/DQQgnQUQvmCnwQYDr9+0PZ6+f+wL0UdauHRNAEZsw5kHYCzwxRonFIeQa3RHMueOONUcT0tfb48TiaMNGJA3zjH34ss4gfLxO9uVKc03D4BYPtkxIdYiYSDEvVJI4T42mJuy5g2NAsiZZCZWG3Q+eeYbpdtRLgMQevrwmxafNOQFEGZ/TYPBKABg7wF7swXAY/pVs0O198DgwlpAaH3CXuk7RFl4X9MWurx2YJavaXPeiq189H6y1h1rPXIBEJrm7AVOhc4eE4+Xi76Q/RRSJ7VdxwIQ/0pfAOJvAwGIvw0EIP42EID4AghjDHDq1m38Iro61ksEhy27/eWBCWXuEN2tSMtG//00VHM80zjVjnG0rcs8X9vvHDUp9uHnPua/TXxxYbwvsyxVC3kAhC465Ju2bJDfjO9mjrLWFOJkAYTxIJxnrETeUARA6B7H/SizBX4bgDC1LL8XL//ALsd9swBi2mTfA/vNc9ezQ7X3IAsgvHfWqpIWnvEvvAMWeL56dqALFvHY6J5HVyyDJlcAsYfQvrIC7YKONF8BiH+lLwDxt4EAxN8GAhB/G/Q7gGRr4nYOQqcFI9sCkgUQujTRfz5P4Eu7OXqMwUiDOZ4MeG40kIZ8p7/6dtkXcc4ZgNDdi98fbdhaNXsAhK5iacg6vpZPdtC5Od8Mridwr3qOb3qfascGIClEENdaXCydjQFJW0DqAYg924tzllo2cd8sgJg2tSDP4lSzQ7X3wMpqLSDALRqjTxoAk7SbWx47MO4HaEnHHNEaQloBSKpulx2noNCuYwGIf6UvAPG3gQDE3wYCEH8bCEDKnQBvADHHu9L6HcdP3mjlsO4585esDiMfmRKdu3Q6WBsbQBedRgNdjHAWGYNgwRx3AxDuRRwGm6cBB5MWFAJf6unak4as42v5pN2aiI/TS3oLeRxfi1trT7cl8sp2kzKYs7TNAAitUOQNRKbB9Kw3CD3bBYtWMfLDec8GexdMv2p2qPYeZO2AXbmXdQXkfvY8dAu0QJxaIEirDHEY85EGa6kRgKSqdNlxu6AjzVcA4l/pC0D8bSAA8beBAMTfBgKQcifAG0Bw+mg5wPkGLgARBpozmxFdWQi7vtpf5ujylZ5WE9LZV2ccPRxBWlw++nRrHKie7aZV/uQ3ftlXc8Z+MBieKWr5ok1+BiCDg1cjJHCO6XHpNs/MTHz95phA9xvKtHz1+jieZNuXe4fMgkU+dP0hHgO+cYwZuE6+c5Lpbfldy/G9Ufr6R9YdiOlsuSfdjMifzUIzAEJaa0mge9iKNRvitMuWd6MAwngMxl+QHgeeQfBM8UurEjoT6tmh2nuQBZBDR07E+2ALZh/jvcN+3Jv3wQK/69mB9wZ7MpabFhamnjYNugJAaMqi/yJ95Wg6HDtxdpzBgWYkBO3XkIJCu44FIP6VvgDE3wYCEH8bCED8bSAAKfc2mGqX2a5sK79a7BeObtqdhdxs3ABT1FpgJiRzPM1xAzAYaMwXZtZ0AFDS6VFthiacRguAC46q5ZEd6G3xKu2tS5KlZUpZjtMZkRh0bdO8Wjy67Ni4DXy89DkAKnN8mUrYAuuTGBRYPrQE4Sda4Hz2Kz9T9nLeBqpb3Hp7ymDONekpo00hbGltDIp1AeN8tS5YaGMBTYA+ew7AzaDkBoC8G6/bbxvLk20BIU+g0VpnLE+c+3RmrHp2qPQeVLIDLS68V+l9sgtZcq2eHVjXhvfc8gFqDPJuAEi5BqZfkX3dWbDInD8UCmSF4w/Lmts4xx9MOgVdkQL1Wtp2QUearwDEv9IXgPjbQADibwMBiL8NBCDd6yXQ7Ym1P1LQaKa0AEE6LWzePJgEiGlyrUtVtXSUk+5a1e7BYoF5Wl+I8+We/Q0NKq9WpjznDxw6lqtcefLKxgGqao3dyMav9xv42XfgSGzhohtZpVDPDnneA1pdKDf2LNoYQB6t1KDSM6fn6gIIg3OADAievmtGQ5YJU3ex+iK01I8hBYV2HQtA/Ct9AYi/DQQg/jYQgPjbQADSP54GX/n5wFtrS1tQekmZ4fxsvWQHz7LWBBBr8nnk6ZfKmtayBWZUvwCkfRWTAKR92uaFRgGIvw0EIP42EID420AAkvVAhu9vWlPo3lVrY9ByL4bh/Gy9aA+PMtcEEPrFQd40EzHAh35klTb64xmA0C8y2wfN48E6dc+8DmyReAIQ/0pfAOJvAwGIvw0EIP42EIB0qnbXfaSAFGinAlUBhME2dL2yZe1tRgUbB5Ld2xRpk6a/XjYVWzsL3w15FwGLvGkFIP6VvgDE3wYCEH8bCED8bSAA6YaaX2WQAlKgqAJVAcRmaLDmPVo4GHjO4KZ0Y9o1YITmNALAwu9LDax0WfQhPNPnhYgi8QQg/pW+AMTfBgIQfxsIQPxtIADxrPF1bykgBVqlQFUAYTEcQMKmVWMasaemzh9yX65zjXmgCUxLRjqbrmxIgmF2oghY5E0rAPGv9AUg/jYQgPjbQADibwMByDBzIvQ4UqBPFagKIEyrC0gcOXYyMM0XkJGdDxvNGB/CNZuJgXmjSVd0OrBesUdeiCgSTwDiX+kLQPxtIADxt4EAxN8GApBe8Q5UTikgBWopUBVAmH8YkFizbnNMbwuzMOMVc0QTaOV4buaiGG/0+JnxHCs+skBNv4QiYJE3rQDEv9IXgPjbQADibwMBiL8NBCD94l3oOaXA8FagKoDw2LRs3Pv3qVEBFoDhN1DCxmKEdsx5FjBhjRAGq4+Z8PLwVi15urwQUSSeAMS/0heA+NtAAOJvAwGIvw0EIEkFrEMpIAV6VoGaAPLam+9HyFi5dmN8QFo8GAfCtLuABi0dLGnPsvIElpoHSnb36LzUzVixCFjkTSsA8a/0BSD+NhCA+NtAAOJvAwFIMzW10kgBKdBtCtQEEMZ+/O7ucbHlY9mqT6qWnYHoM+cti/AxddaSqvGG44W8EFEkngDEv9IXgPjbQADibwMBiL8NBCCJJ3H2SPLju8ODW4eeG0ZnNmzeEZ6cMjf2OBlGj6VH6UMFagIIehw9fipCCC0bI+57OixfvT4wNe/ZcxcCK6UveefDwIKFXH94/MwwcGWwr2QsAhZ50wpA/Ct9AYi/DQQg/jYQgPjbQADynYux6/0QZv4/IbC3YOfWTrEzw24/a8E70d86dfrcsHs2PVB/KVAXQJCDlhCm5f3+T0fEF9/Gftj+J7++N3zy2bb+Uu67p80LEUXiCUD8K30BiL8NBCD+NhCA+NtAABKuQwfwYRvgYfBh54YphAhA+tLVHJYPnQtA0ic/fPRkXOtjwVvvhw2bd7qs93Hi1Jlw7vzFtFilY8apVNpYPDEbaN2hOZNnajYUAYu8aQUg/pW+AMTfBgIQfxsIQPxtIAAJIaydfAM+DDiy+8X/XwiXi7USMOb1oSdnBPydm/44OnZHp1u6LbxsfgM+yQOPTYtjY5mg566HnwvMJGqBXiMsVfDWinXhnjHPx0l8fnzzyECXdT7wWmA9NcbSpmHT1l0xrd0zCyD4QvROYVwuEwLxofjOUZNiTxXLx+5PXktXrgt/HT05/OGe8eFynywYbTpo310KNAwg3sU/c/Z8/CN7ZtqCikWxVpnsnj82C8zWdfvfni1rzblt5BNxhXeLk3efFyKKxBOA+Ff6AhB/GwhA/G0gAPG3gQDku9q5FoS0AD64C36D+RI4+dbdHCf/0nfOO3ucf+IxNgPfBBBgs25Sm7ftKeUDeAAAlu+cRStL7gYT+/zmrsdLvzmwNdn+tf2reD4LICyLwL3Ic9L018MjT78U8waErIx2f+vFYuXDF1KQAl4K5AYQumCdPHXWq5xh19ffxi8H9gdUC0D4QsGXhnRLu4ixVgl//HzV4MvA4uUfxD/g+8e+0PDzFQGLvGkFIP6VvgDE3wYCEH8bCED8bSAASarpZaOHtoS89p+FWz7sDgAIPsfe/YfjKVorbO2zZas+judeXbQy+hNr1n1uyeL4WHyMZ6ctjOcMAPBbbJFmemAAAsCLhWYAhDJlx97ir3H/bV/uLbs/+e/Yvc9up70UcFUgN4CY409z5PZd33S80JA6a418uWd//MOqBSCPTZxdtXw0V/KHyVeCNPDlgvPHT56Jp63J8vOtu9Jo8esGGljICxFF4glA/Ct9AYi/DQQg/jYQgPjbQADyXe2bHfORdsHiWgsCAEKrRxpw6vEVaG0g0KUKkEi7UnGedCxZQDAASSGF87bAs7VUNAMg5EO3c5ZNwDehzLSyUMYP1/+r5v3jRf0jBZwUyA0gz7+8OP6R8VKz/WrEmPDO++sDU/B2MgwMXIn3rwUgd46aGD7euC0AD6fPni8r3q6vrgMMrR9p4GsGz7X5i93xdLX/MP5071Nl/yEVAYu8aQUg/pW+AMTfBgIQfxsIQPxtIACpMAg9hQ87bgGEVAIQYAFfwT5i4gv97Nb7U3ciHuMrWOtGNX+CFhLyOnLs+jjUZgCEdddsYehf/v6hcN+j/4hjQMhXADLELDrRRQrkBhDKzIAlujXxR8LLzUbLyAuz3yq1HLT72fIAiJXN9pNmLCoVi65YnLfmU7vAlwnOr/rgs3iq2n8YWQChMmj3dvrcQLhwebDt92n3c/Ry/ifPDoRLA7KBpw2BwIHBq/o76MD/OdXsfPT0pXBl8Jps4GmDU5fC4NXutoHVq23bZ8d/MOaj0rmCBagEICw/gK/AR1mCtWJcvFQ+ngIwAQgI1fwJxpWQl3WhwrciXRrqjQGxLuUMMLfAMfkKQEwR7btRgYYAJH0A/ghp7uMlt41ZILbuuD5QKo3byuN6AELfRyCDlgwgwwaNsV4JwUBjxZoNZcXiD5XnoFWHUO0/jCyAUBG0e7t67Vpgoox230f5V7elbFBdm069N1ev6u+gU1pXu0+0QdD/RdX06dR56qhO3auZ+5RVru36YcCRDjivdK7A/SsByPRX346+gn2spDcGvsP6TdtLd2JWLM7ZuNJK/oT5MmnrCfejNSMNjFElr2qD0OnmxYfgNHz06daYRgCSqqLjblOgaQDhQejeROsCfxzpBsXj/PMH1upgf7TVumBl78fAeco28pHrCxPZfwS05KRh5dqNMd66jV/E0xYv22czCyB5u1EViacuWP7dHtQFy98G6oLlbwN1wfK3Aa1TQEGROqXdadO6ta3Hm14bOuC80rkmCwEQML5j5rxlsTXBZqACGmww+aEjJ6LvwDk+ftq0ufgdNl7W/AlaPFb+c0N474ONgZk3icNvCxNfvN4li+l/icO0vOZbVQOQ2a+viHEmvPBa/MA65aUb3eUFIKas9t2oQFMAwh/V6PEzS38Y/IFA4bMWvBubD+0PhrEYrQ6NAgj35+sAc3AT9h04Esv94pylZUWzP2KbIcL+wxCA+Fe47a4s8+QvAPF/DwQg/jYQgPjbQABSVnW39QcAgj9jk/BwzDS5B4/cWOODAvDhMo0DtLy75tNS2cyfsOl6yYc485asKsXhgGl7GdRuPhStIeZrWe8S/CyuM/A8pjlzrtQNzPJ9cNyLMY4ByJbt16cBzvozZTfXDynQYQVyA0il8R+87AzE2rmnfFo3/tiYk5q+ka0OtQCENUKyM1Ewc5aVk7Lw1SIFEisfXyb4D8EW5rH/MNKxIuRN/8x0Vow8zmvROGoB8a/0BSD+NhCA+NtAAOJvAwGI1drt31sXLPwGFiw+e+5C1ZviH3x78GicrdNaRyyy+RMAAIsis0gh3RmrBeBi/8GjQ/yZavE5z7S+zN7Z6YmBapVJ16RALQVyA0hK7kzxRosBDn+tYFPL1YqT9xpfBj7asDWsWbc5AgWAw2/7KkA+NI/SDMrCPsyCRTcrK7c1hRLPmjnpxkVzKU2WQEo6fa/9h8F5mkOZNYvuV/wWgPhXwkWhrtH0AhB/mwtA/G0gAPG3gQAkr9dQPJ4BSNGczJ9QC0RRJZV+OCmQG0BoNWD1cJz+bCtDJwSxQeIAQLrZPNuUATix6egsDr+ZRSINzDjBgHmLw54xIuksFvYfhs1SQRzAA8ARgPhXwo0CRNH4AhB/mwtA/G0gAPG3gQAkrc3be5wd89ns3dQFqlnllG44K5AbQLZs2x2bIKuJQfMkq5V7wElaJu5PUyQtHpSpVnlY3JDuY+cvXEqziMcGIHyxoFsWeVYKRR3bPOnVBcu/0heA+NtAAOJvAwGIvw0EIJVq4vaco5tUtjtVs3eia1Qtf6TZfJVOCvSqArkBhBYQuj1VC3RTopXgm28PV4vSU+dTAKlV8DwAUTSOAMS/0heA+NtAAOJvAwGIvw0EILVqZF2TAlKgVxRoGYCw/gYAks6F3SsiVCpn3ibTonCRJ70AxL/SF4D420AA4m8DAYi/DQQglWpsnZMCUqDXFGgJgNBMaVPH2VzVvSZEpfLmaTLNAxBF4whA/Ct9AYi/DQQg/jYQgPjbQABSqbbWOSkgBXpNgZoA8uSUuXF9DwZ607rBNLUcpxuLDtr81+wZV9FPoShc5EkvAPGv9AUg/jYQgPjbQADibwMBSD95GHpWKTB8FagJILZSJ/BRb2NmKKa07beQByCKxhGA+Ff6AhB/GwhA/G0gAPG3gQCk37wMPa8UGJ4K1ASQc+cvhpOnzsaN1g8WFrTf6b6fF74pChd50gtA/Ct9AYi/DQQg/jYQgPjbQAAyPJ0xPZUU6DcFagJIKsaur/aHA4eOpad0HELIAxBF4whA/Ct9AYi/DQQg/jYQgPjbQAAy1PX4ZuBaWHr6ytALOiMFpEDXKpAbQLr2CZwLVhQu8qQXgPhX+gIQfxsIQPxtIADxt4EAZGilP/XYQPjPfReHXtAZKSAFulaBqgCy+Yvd4cc3jwzPTFsQC3/nqEnhN3c9Xnc7ffZ81z5sOwqWByCKxhGA+Ff6AhB/GwhA/G0gAPG3gQBkaE3+f355PvyvW88OvaAzUkAKdK0CVQHk443b4sDzBx6bFgtvM13VG4xebcXwrlWgYMGKwkWe9AIQ/0pfAOJvAwGIvw0EIP42EICUV9p0v/qft5yNW6u7YR04fDzc9+g/wg9vujvO9nnTH0eH6XPfLhVgyTsfhl/+/qF4jXGyP7v1/jBrwbtlK57Xy+N3d48L85asipP4PDx+ZuD39l3fxHvgh/Hhl7z5IMzMpNmZRjds3hH+dO9TsQzEwWfbs/dAqYw6kALdqkBVADl/4VL4YufXgT8ewolTZwJwUW+7du1atz5rW8qVByCKxhGA+Ff6AhB/GwhA/G0gAPG3Qb8DyAfnBsP/tedC+L+/2/7rl+dLAPJ/bD9XOs/1V080Py5k4MpgdPpx/sdOnB17g7DsAB9hLTz67CsRQB6bODtMmrEo2PX5S1bHKHnyIL/0Ay/HH67/V1j5zw3xXsAPeQMZxAWILKxYcz0O4PHU1Pml9dhenLPUomgvBbpWgaoA0rUl7rKCFYWLPOkFIP6VvgDE3wYCEH8bCED8bdDvAIILAIR8b/u5EnhYC4jt/5etZwsPSt+xe190+J9/eXGZ17Fh887S74GBcsBhRtAf/OKOOGMokfLkYQCy+sNNwWYUtXxY3iD9qDt6/MxYJj4IWxyAhQ/GFpgsaOOWG2W089pLgW5ToGEAOXz0ZFiz7vPw2pvvh0XL1sYX/fyF/h38lQcgisYRgPhX+gIQfxsIQPxtIADxt4EA5LobdepqCP/964tDIITxIHTLKhrOnrsQnX1aF5avXh97gVTKc/O2PWHqrCXhr6Mnh1vvGBvT0FWLkCcPAISuVWn45tvDMZ87R00MbyxfW9oeHPdiPM899+4/FI9HjZueJtWxFOgZBXIDyMVLl2PfQv5YshtNlAuXrumZh25lQYvCRZ70AhD/Sl8A4m8DAYi/DQQg/jYQgNyoweliZa0etv/bgRutATdiNndEVyr8G/N56GKVti7Q7YlrxKGL1JgJL8cWEAMQ7lovD9JnAcTG4JIvLRzZjTJ89OnWeG/GoShIgV5UIDeA3Pv3qaU/wvvHvhCJf8pLi8PIR6aUztMq0m8hD0AUjSMA8a/0BSD+NhCA+NtAAOJvAwHIDS/DWkDockXLBxDCOJBWhlOnz8UWkEeefqkEG5cvD8TxsMADrRR8oLVw28gnQgognK+WB9cqAQiD0DlfCy62fbk3xpk0/XW7tfZSoKcUyAUgzLrAHwM0TtNgNrBIoX0lyPaJzMYdbr+LwkWe9AIQ/0pfAOJvAwGIvw0EIP42EIBc9yLoggVwACEcs9H6wbktF68WdjXOnD1fNraCDOcsWhl9IbpArd+0PR6nvT8Yr0EriQFIvTzIsxKA0K2d88BMNpAng9vPnb8eh1m4suHYidPZU/otBbpOgVwAYn0N6eNYLYx/fl78g2HmrH4KeQCiaBwBiH+lLwDxt4EAxN8GAhB/GwhArnsYdL9iAcJsYCrecYdvtEhkr+f9vW7jF7Hr0+zXVwSAg5mpcPYBA1o0+NjKh1em3l26cl14a8W6OIUu1w1A6uVBWSoBCOeZ+Yprdz38XPhow9bY5YoB8dzTZid9YvLcGGfMhFmxa9iqDz6LZdAsWHmtrHieCuQCEGZb4A+B+airhVe/+zKwaeuualGG5fmicJEnvQDEv9IXgPjbQADibwMBiL8NBCD1XQlaQ4oGPrwCEvg+tjHOAxiw8OaKj+KYD7s+4r6nw09+fW9sBSFOnjxIywfcbKCVY9qct0q9S9J70ApCoHcK40fsGnum7U3LmM1Xv6VAtyiQC0AoLIvb8HJ/e/BoxbIzTzbXj588U/H6cD2ZByCKxhGA+Ff6AhB/GwhA/G0gAPG3gQCks94ETv7X+w6VjfNISzA4eDXs3LOvpu9TL480v+wx3bqOHDsZ9h04Eltdstf5zUfi/QeP1ixDpXQ6JwU8FcgNIBNfXBgB45bbHw3TX317yEazIDM1pNfeXfOp57N15N5F4SJPegGIf6UvAPG3gQDE3wYCEH8bCEA6UrXrJlJACrRZgdwAAlykzXx5jmmuHO4hD0AUjSMA8a/0BSD+NhCA+NtAAOJvAwHIcPcq9HxSoD8UyA0gK9dujAOtGGyVd/v08x3DXsWicJEnvQDEv9IXgPjbQADibwMBiL8NBCDD3q3QA0qBvlAgN4D0hRpNPGQegCgaRwDiX+kLQPxtIADxt4EAxN8GApAmKmolkQJSoOsUEIAUNElRuMiTXgDiX+kLQPxtIADxt4EAxN8GApCClbaSSwEp0BUKVAWQzV/sDj++eWR4ZtqCWNA7R02K0/AyFW+t7fR308N1xdN1oBB5AKJoHAGIf6UvAPG3gQDE3wYCEH8bCEA6ULHrFlJACrRdgaoA8vHGbXHQOdPvEvIOQj96/FRrFHfVAAAgAElEQVTbC91NNygKF3nSC0D8K30BiL8NBCD+NhCA+NtAANJNHoDKIgWkQLMKVAWQ8xcuBVY1txU3T5w6E4CLehtzVvdTyAMQReMIQPwrfQGIvw0EIP42EID420AA0k8ehp5VCgxfBaoCyPB95NY+WVG4yJNeAOJf6QtA/G0gAPG3gQDE3wYCkNbW4cpNCkgBHwUEIAV1zwMQReMIQPwrfQGIvw0EIP42EID420AAUrDSVnIpIAW6QoGqAPLI0y+Fm/44uuHt1JlzXfFgnSpEUbjIk14A4l/pC0D8bSAA8beBAMTfBgKQTtXuuo8pcPjoyfDklLlh11f74ym65vN7994DFqXr9kygtOqDz7quXCrQDQWqAsivRoxpeOVzVkfXIPTWV1ACkNZrmgf80jgCEH8bCED8bSAA8beBAOSGA1M6WvFZCHPXlH7qoLUKMB4Y/+69DzbGjDdv2xN/r1n3eWtvlORG3rf/7dlw9tyF5Gz+Q8r7+KQ5+RMoZscVqAogFy5ejobH+Gw//+2D4Qe/uCOcOXu+7DzXPlz/r/gyPjV1fscfwPuGqZParmMBiH+lLwDxt4EAxN8GAhB/GwhAklr//KUQnnszhN8/c30bPTuEY6eTCDpshQIeADL79RXRrzx2ojl7CkBaYfn25lEVQNLb0q0KYz705Iz0dNnxn+59KsYBUPoptAs60nwFIP6VvgDE3wYCEH8bCED8bSAA+c7D2L4/hNun3IAPgxDOffBFS9wQuhrd9+g/wg9vujsuRUC39Olz3y7lveSdD8Mvf/9QvPa9H90Wfnbr/WHWgndDOhtovTx+d/e4MG/JqrBp667w8PiZgd/bd30T78FyCKy7Rt6sy0a3Jz4Op2HD5h0B/4ulEojD0gl7Gugade78xTDivqfjM3If8mHdtzSPagDyysIVYeQjU+LHae49bc5bZc9OOes9Q6Xnn7d4VSwPfufNf34k3HrH2FjG9LnrHWcBBO1+8ut7o5Y84y23PxrWfrKllM2YCS+HO0dNHFL+N1d8FO9/8MjxGBfbvrpoZRyegF58nH/tzfdL6dCN8mLPpSvXhb+Onhz+cM/4cPnyQJxVttb7VCpMnxzkAhAExZiPPvtKVVl48YizftP2qnGG44UUFNp1LADxr/QFIP42EID420AA4m8DAUgIgZYPA45qewClQBi4MhgdepzMsRNnx0WZcYbxcyzgEwEgj02cHSbNWBSdZa7PX7I6RsmTB/HTddY4plfJyn9uiPcCfsjbPvLiwFpYseZ6HJx/eqDcM+b5mObFOUstSt09LQw8I47ypOmvB8b/UiZ6vFy6PBDTVwMQ4hn04Nzze+4b75XumecZKj3/y6+9E3XlGuUCqmr5n6UbJgekTbtgAYe3jXwi6oRWPB9xdu7ZF1MBU/zO+rA8F5BhgTyJxzCFKS8tjtf4vWjZ2hjFuqeZTdGW7fTZC3XfJ7tHv+xzAQgvIQJjiIGBKxW14eUgTjv7BFa8sfPJdkFHmq8AxL/SF4D420AA4m8DAYi/DQQg31X6dLeqBh+0ghQMO3bviz7N8y8vLstpw+adpd9Zf+jK4GB0bP/ywIQYJ08e+E04q6s/3BRIT7B8cHzT1pTR42fGMrEum8UhLeu2WThw6FjYuOVGGe18tT35A0ppWLh0TbzPti/3xtPVAISB3oODV2McWmYoC0BCsPLVegbiVXp+zre6C1bWVnv3H4r3BjwIp05f7+kDxFmgFYPyoQfB0jwxea5FiXsgEcAhGIAAq9jfQp53weL2yz4XgCAGTVMYgj1GsMBLR/MT19ia7a9n+fXaPgWFdh0LQPwrfQGIvw0EIP42EID420AA8p2XwMDzagAy/Z3CrgTjW/FpcKiXr14fcPorBRzOqbOWxC/1dL0hDV21CHnyID7dg9LwzbeHYz74W28sX1vaHhz3YjzPPc0ZHjVuepq0qWO62ePH0c2e7kI8M+WiJYZQDUCyH5wZNE46uhvleQbyrvT8nG85gFwZDGvWbY5ao6u1ZgFRFu79+9RYHpvNFb1pvbBub9iC8j47bWHJJpyzvK5evVYCkKw2ed4FK0e/7HMDyMlTZ0t98jAARrEmJn6zGUn2i3g8Z7ugI81XAOJf6QtA/G0gAPG3gQDE3waNAMjyMZfDO99tny272JH6irqrI4HB5tUA5LPdLSkCXanwdczHwdFMWxfoymP+EF2kGEdA1x4DEApRLw/SZwGEcROWL35WdqMMH326NcZhHEqRwFS61h2J7mR08WIMCPdvFECem7kopmPa3jzPQLkrPT/nWwkg5y9cjGNpuBetFTwf3eb4nQII42k4N2fRyjjhEsfEs0BXOM5l7WG/AS9rAckCCHnUexfsPv2yzw0gCEJXLMZ68MeFEcwQDGD69PMd/aJZ2XOmoNCuYwGIf6UvAPG3gQDE3wYCEH8bNAIg838zEGxbN+fS8AIQauJK3bBa0P0qreTpmkMLiI2NAEhwNFlyAB+Ir+kXL90YGM44gxRAyKtaHlwjjyyAMAid87Xggu5RxGHcRpEwZsKsmA/djSxY16NGAcSAjO5OeZ6B+1V6fs4bgDS7tAP52hiQd95fH+8z/dW3y7q0EScFELqj0fpDdyomBuB6utaJjRP59uBRk2rIvhaAELnWuzAks2F+oiEASbWo1G8wvd4vx+2CjjRfAYh/pS8A8beBAMTfBgIQfxsIQBLv4psjITDYPN0414LAjJ7p2Aqy5Ms4TilOJoOVObbxAVzHL6KVxACkXh6kIY8sgPDFnvPATDaQJ2M2mL2KOLRaZEMjXeGZDYov+Gmw1pVGAITuR4wTZiPkeQbiVXp+zi9463rX/q/33ejyHzPO+Q/5GoBMnvlGvE86fABoJE4KIGT90vzl8TygSZe6NKzb+EW8xof4bDh+8noXvWoAkuddyOY53H83DSDDXZi8z5eCQruOBSD+lb4AxN8GAhB/GwhA/G0gAMlbOxeLh7OJY86XeJxKnHGcfZxWvmLzlR8nla/lTLf61op1cQpdrhuA1MuDEhI/CyCct+4+dz38XPhow9bY5YoB8dyTqX0JDIYmPa0YdMti5W+mtW1kFixraZjwwmtxEiFmduIe5FsPQJihiq5WdF2yGbjQwUKeZ6j2/PSq4RrjMj75bFtgOtxGAmkNQKw1htYqBvvPTab5zQIIY31Iy0bLSRoATICNaxNfXBg1X7l2Y9TfBt9XA5A870J6r344bghAmAcZ2qefH4NwKm3ZLwbDXcR2QUearwDEv9IXgPjbQADibwMBiL8NagEIYz6sy1W9fVrHtPp4ONT7fC1Pu5vjdDLOAxiwgFNs4ye4Tnd0WgBoBSHkyYN045+fZ1mW9rRy8KXdYIB4dg++phMYHA282DX2jHFIy1jKsMoBA66Ztcvy4H422D0LIAAOYcv26yuhcy9Lx57B+LSEWMjzDKSr9Pw4+5y352ffSCDfcZNfLSUxWLPyco1j/NhssFah7OxgxANQmBbY8mFP2WxmLNMmOwYkz7uQLcdw/50bQOgDmQpe7bjZ/nq9KnSr/+OulJ8AxL/SF4D420AA4m8DAYi/DQQgnfUWcPLpBpSO80hLwDS0rCVhXXDSa3ZcLw+LV2mPI37k2Mmw78CRqssgMOXt/oNHa5ahUt7pOXw31nwjr0YC6WhhqKYPeeV5hmr3pDyMuWDcTdHAB3JsRfewauHQkRPR16X1plagXNiEAfcpdNVKw7Ui70K9vHvtei4A4Q/MKJRmNgZG0YwF4WW37FzL7RAEAqX/Y7VAGRigxdRxtV5a/nBoOuQFajZUAoZWnxOA+Ff6AhB/GwhA/G0gAPG3gQCk2dq6f9Ix3SxdguptLBTYKwHAq/c8XGcgfJFgA+lt5fMieSltbQVyAYjN50yzlHeg6REYyvbbs3LRZJhtnXnvg412Oe4hUJuv2uIy2KsWFZdlkPxoNWxUyk8A4l/pC0D8bSAA8beBAMTfBrUAhKl2me3KtrQbFt2z7Hy7Z8RKqkgdOiiAn8TCd/U2W+/CoYgN35IWh3rPw3VaMJoN+Ib4l/iHCu1XIBeA8JLiqDPgyCvs+vrbOMiJQWGUpRKAWNMZ/TQ3f7E7/Gv7V6UFFNPZD2zaOWZZoMlx8fIP4kt3/9gXGn68SsDQ6nMCEP9KXwDibwMBiL8NBCDttcG6Q0fqTpVbC0CydU8KIO2GjvTeDVekSiAFukQBQKeRLlVdUuyeLEYuAOHJmOkB5x/jeATIFIj4cs/+qgBiMy6kXaoMnmyAkE1dx5zeabCBXNaPEzBhCrbPk7mxiQ/4sFqohfQ/3XYdC0DaW+nnsZsAxN8GAhB/GwhA2meDp77+PPzbp88F9rX+TxKAWO2rvRSQAr2sQG4AsfmPZy14x/V5Gd9RrQWE6eqy81lTWPoF/uGe8bHcu766DjC0fqRh2aqPY760nBCqTaVG68rPf/tgKWmtiqJV1wQg7av089pIAOJvAwGIvw0EIO2xgcEHAFIPQgQgpepXB1JACvSwArkBxLo+2ZiJavtGFsBpRrdaAMIYFWAjG5i7mxYcAvNJU3aAIw0Mpue8TTMnAGlPRZvX4e+2eAIQ//dBAOJvAwFI622QhY96ENIIgHj9P5rWrTqWAlJAClRSIDeAMB82cz7X206eOlvpPi07VwtAAA1bhTO9IXNyG5gYaKxYUz77gw1et4Vn8gIIjmm7t3MXr4RLA1fbfp92P0cv53/2wpUwcEU28LTh6fMD4crVa/o76MD/OdXsfOrcQBi8GmSDFtlg4v4tscXDoCO753rWFifPDQSWWsie76bfaf2rYykgBaRAJQVyA0ilxB7nagEI3awApGyg9eNXI8bE0wYW6WqdXGA1S1pA6GpGsHgASxqyXbAuXh4M7d5wfAevXmv7fdr9HL2c/2XZwP39uzxwNQ4O7OX3qNfLfmlgMFy9pv+LWmHHqYe21oQPgxHipfe7dHkwrquQnuu247TO1LEUkAJSoJICwwpAbHXKdPVKZjNgWrU7R02Mz8/CMYDGi3OWlukx+/UV8TzTuBHyAkgnmrg1BqT13R4atRtfF6nkG02n+K2znbpgtU7LZt9LdcFqjQ2Y7coAI88+nR1LXbDKqm79kAJSoEcVaAhA6F41eeYb4c5RkwJdsj76dGt8bMZVjJnwcli6cl3bZajVArJw6ZoIESwuaGHrjq/iORs8z6KKjGdhhqs0jLjv6QgqtnChAUg6VoTVPGlJ0SD01lTCzTpBHukEIP42F4D420AA0jobVBv7kQWS7KxYApC05taxFJACvapAbgDZvuub6KDTemCbOeesKM65H/zijtg03A4xTp0+Fz7asDWwwif3Yk0SfgMYFmzKXcZ70HVq7Sdb4uBz4lNGCxNfXBjzYErdTVt3hSkvLY6/H5s426KUWkBIy8qYzJpF9yt+C0BaVwl7wEQz9xSA+NtcAOJvAwFIa21QD0Ky8MH/XQKQUjWtAykgBXpYgdwAwgBvnO9npy2Mzj3HBiA8v3V/ootTO4INEue+6ZZdnR2gSGfsovvVxxu3lRWJLlpWXstr5CNTwsVLl0vxrAWElhGLA3gwnkQA0tpKuBkg6HQaAYi/zQUg/jYQgLTeBtUgpBJ8CEBKVbQOpIAU6HEFcgHI+QsXoxPObFIEa/FIAWTOopUxTtbZ99Ln24NHAzBEt6lqgcUNd+7ZF85fuDQkigEILSl0y0pbUNLInXCENQak9ZV+o3YTgPjbQADibwMBSHtskIWQavAhAElrXx1LASnQywrkApCv9x2KcDF11pL4rJUAZNGytTEO40GGQ0gBpNbzNOrINhNfANKeSr8RWwhA/G0gAPG3gQCkfTYwCKkFHwKQWrWxrkkBKdBLCuQCEFoA6IZk3Z0qAch9j/4jxjl45HgvPX/Vsm7Zvic+T3Ya3myCRpzYZuMKQNpX6ee1iQDE3wYCEH8bCEDaa4N0tqtq/zdpDEi2FtZvKSAFelGBXADCgzFrFBDy3gcbw+GjJ+OxdcFi9iuuMfibaW+HS7gyeH2+9VrPU62SaOV5AUh7K/08thKA+NtAAOJvAwGIvw0EILVqZF2TAlKgVxTIDSDWDQvQYLYr9kzFy4BsjtlsEb9eefhWlDOP81o0jgDEv9IXgPjbQADibwMBiL8NBCCtqLmVhxSQAt4K5AYQCrrrq/3hN3c9XgIOAw9WHx8uYz8aNUhRuMiTXgDiX+kLQPxtIADxt4EAxN8GApBGa2nFlwJSoBsVyA0ge/YeCNu+3BtYyI81OTZ/sTtCB92xas001Y0P3coy5QGIonEEIP6VvgDE3wYCEH8bCED8bSAAaWUNrrykgBTwUiA3gFi3K1YiV7ihQFG4yJNeAOJf6QtA/G0gAPG3gQDE3wYCkBv1r46kgBToXQVyAwgDzOly1c+tHZXMnAcgisYRgPhX+gIQfxsIQPxtIADxt4EApFJNrHNSQAr0mgK5AeShJ2dEAKErlsINBYrCRZ70AhD/Sl8A4m8DAYi/DQQg/jYQgNyof3UkBaRA7yqQG0BYWZwWkPvHvtC7T9uGkucBiKJxBCD+lb4AxN8GAhB/GwhA/G0gAGlDRa4spYAU6LgCuQFk5CNTwvd+dFuEkN/dPS784Z7xFbfTZ893/CE8b1gULvKkF4D4V/oCEH8bCED8bSAA8beBAMSzxte9pYAUaJUCuQHk+z8dEeHDpt6ttmeV9H4KeQCiaBwBiH+lLwDxt4EAxN8GAhB/GwhA+snD0LNKgeGrQG4AASwOHTlRdxtOK6HnMXtRuMiTXgDiX+kLQPxtIADxt4EAxN8GApA8NbPiSAEp0O0K5AaQbn8Qr/LlAYiicQQg/pW+AMTfBgIQfxsIQPxtIADxqu11XykgBVqpQEMAwhS86zdtD68uWhmembYg7Ni9L5aFmbGWr14ftu/6ppVl64m8isJFnvQCEP9KXwDibwMBiL8NBCD+NhCA9IRroEJKASlQR4HcAHLi1JnwqxFjysaBLFv1ccwe8GBMyE1/HF3ndsPvch6AKBpHAOJf6QtA/G0gAPG3gQDE3wYCkOHnR+iJpEA/KpAbQP46enKEjJv//Ehs/QA4DEAQbsR9T8frGoTe+gpKANJ6TRuFQgGIvw0EIP42EID420AA0o+ump5ZCgw/BXIByMDAlQgXP7zp7rgSOpCRBZBpc96K5zZt3TX8VKrxRI06ss3EF4D4V/oCEH8bCED8bSAA8beBAKRGhaxLUkAK9IwCuQBk7/5DES4mTX89PlglAHlzxUcxztpPtvTMw7eioM0ARaNpBCD+lb4AxN8GAhB/GwhA/G0gAGlFza08pIAU8FYgF4CcOXs+wgXdsAiVAOSpqfNjHGCln0KjMNFMfAGIf6UvAPG3gQDE3wYCEH8bCED6ycPQs0qB4atALgDh8X9888gIGIePnhwCINZCwkrpA1cGh69aFZ6sGaBoNI0AxL/SF4D420AA4m8DAYi/DQQgFSpinZICUqDnFMgNIHStstXPbUD6qHHTw5NT5pbOz1uyqucEKFrg/7+9s3+Sorr3/99x/4Dv/SG/favyY+qbyi+ppL43lVSZukVdU7GsmGsSyVe9Grw+BIxfvGrwASP4FHkMomgggIqiIMpFRNENj/LggiCIPIisu+wuu+zuufU+5AynZ3tme6Z3+tMz8+qq2enpOadn+vWe3nNe0316GpWJZsojIPaNPgJinwECYp8BAmKfAQKSt9WmPgQgUAYCmQVEb3bdxm1ORzmCiMT3Ty5b68bGxsuwTYW+h2aEotE6CIh9o4+A2GeAgNhngIDYZ4CAFNrE82IQgECLCDQkIHoPff0X3dvv9bglL77unvnLen8p3hOnzrbo7ZV/tY3KRDPlERD7Rh8Bsc8AAbHPAAGxzwABKX+/gHcIAQhMTaBhAZl6ld1VohmhaLQOAmLf6CMg9hkgIPYZICD2GSAg3dXHYGsh0KkEmhaQ8fEJ17P3sFuzYavTL6F369SoTDRTHgGxb/QREPsMEBD7DBAQ+wwQkG7tbbDdEOgsAnUF5N5HlvoxH/c//pfEVh87cdp958czE2NBVFY/WNhtUzNC0WgdBMS+0UdA7DNAQOwzQEDsM0BAuq2XwfZCoDMJ1BQQyUQYcC7hiKfb/vCklw9JyN0PPue+99Nb/OMlq16Pi3XFfKMy0Ux5BMS+0UdA7DNAQOwzQEDsM0BAuqJrwUZCoOMJ1BSQA58e91Jx65wFCQhHj5+qHPn46utv/HNf9/V7WfG/A9JlR0GaEYpG6yAg9o0+AmKfAQJinwECYp8BApLokvAAAhBoUwI1BWTrjj1eNJ5d8Upi0zTmQ5ffveuBPyeWz7xrvl/OL6FPfwOFgEw/00YlEAGxzwABsc8AAbHPAAFJdD14AAEItCmBmgISROONLR8mNm3u/OVeNF5cuzmxXJfllZhs37kvsbzTHzTakW2mPAJi3+gjIPYZICD2GSAg9hkgIJ3eq2D7INAdBGoKyOZtPV4oqo+AzPj1fX65roAVTyonAXn/4/3x4o6fb0YoGq2DgNg3+giIfQYIiH0GCIh9BghIx3cr2EAIdAWBmgJy+OgJLxTX3fxABUTvZyf9MonGwMWhynLN3DpnoX9OZbppalQmmimPgNg3+giIfQYIiH0GCIh9BghIN/Uw2FYIdC6BmgKiTQ5HOx5csNL/3sc1N8z2kqHxHvEkGQlXzKoWk7hcJ843IxSN1kFA7Bt9BMQ+AwTEPgMExD4DBKQTexJsEwS6j0BdAdl78GhFLHTUI9x0Jax4WrB4jX/uxlkPx4u7Yr5RmWimPAJi3+gjIPYZICD2GSAg9hkgIF3RtWAjIdDxBOoKiLZevwHy+HN/db+953E376kX3YlTZxNQRkZG3bU3zXU6OrLxnZ2J57rhQTNC0WgdBMS+0UdA7DNAQOwzQEDsM0BAuqFnwTZCoPMJTCkgnY8g3xY2KhPNlEdA7Bt9BMQ+AwTEPgMExD4DBCRfm01tCECgHAQQkJw5NCMUjdZBQOwbfQTEPgMExD4DBMQ+AwQkZ6NNdQhAoBQEEJCcMTQqE82UR0DsG30ExD4DBMQ+AwTEPgMEJGejTXUIQKAUBBCQnDE0IxSN1kFA7Bt9BMQ+AwTEPgMExD4DBCRno011CECgFAQQkJwxNCoTzZRHQOwbfQTEPgMExD4DBMQ+AwQkZ6NNdQhAoBQEEJCcMTQjFI3WQUDsG30ExD4DBMQ+AwTEPgMEJGejTXUIQKAUBBCQnDE0KhPNlEdA7Bt9BMQ+AwTEPgMExD4DBCRno011CECgFAQ6TkD6+i+6tNvg0PAk4OfO97mP9xxyZ85dmPRc1gXNCEWjdRAQ+0YfAbHPAAGxzwABsc8AAcnaOlMOAhAoM4GOE5Dwa+3V97+649FKDkPDI+7m3z9R+WV3ldWvuKdJSqVSjZlGZaKZ8giIfaOPgNhngIDYZ4CA2GeAgNRojFkMAQi0FYGOFJBf3j7PvbZpR+L24d8PVIK5//EVXj5Wv/auO3r8lFu/8T337R/c6O5+8LlKmawzzQhFo3UQEPtGHwGxzwABsc+gUQH5aN+Qe3jhJXfb7BF/0/y+3mHX6P9Ayl/NHgHJ2jpTDgIQKDOBjhSQhxasrMn84uCwl4+585cnyjzy9Cq//PyFfr9cYnL9LQ+63ft7E+X+tGi1u/eRpZVlRTSMCMjVxrcI3mmvgYDYZ4CA2GfQiIBs2nbJ/eL/jU663TRr1G37CAlJ+z+TZRkCUml+mYEABNqYQEcKyK1zFrgPeg54efhmYDART+9nJytHP+In3tjygV++55MjfvGeA0f94607dsfF3G/ufMxdc8PsyrIsDUbeMgiIfccLAbHPAAGxzyCrgPSeHHISjTQB0TIdEVGZvP8bu7E+AlJpfpmBAATamEBHCkj1+I+FS9dWItKpWHpewhFPEg0t3/Le3/1iBITOQdy5QUDsPw8IiH0GWQXk1U3DNeUjSAlHQZrLEwGJW27mIQCBdiXQcQKyZsNWJ8nQkQxJho5WSCxeeXO7zyiIxqatHycy275zny/35rs7/fKsAtI/dNm1+jY8MuZGxyZa/jqt3o52Xv/QyJi7TAamn8HBS2NubJz9wHI/ujh82Y1PuCk/By+tr330IwjIK2+2/n+nJatWvbYymMiQQateP8t6E40rDyAAAQikEOg4Aanexgt9A14sZs192j8VxEKD1ONp87YeX25HzyeJclOdgtU/OOpaffMCcnm85a/T6u1o5/UPXZKAkIFlhoPDl68ISAH7nOV2lvm11fn1EjhFBpu2Ti0gPXta/7+zzCybfW8DQ5LAiVK3B3HbyjwEIACBNAIdLyDa6O/8eKYfUK75E6fOetFY/MKGBI+Vf9vklx86csIvD6IylYDEp+m0ap4xIM2dqjCdeXAKln0GnIJln0HWU7B0paubfldbQm77PWNAmv3/xClYiaabBxCAQJsS6CgB6R8YdBM6Nh1Nx0+e9mIRrno1NjaeEJJQdOZd8/2leEdGRv2iICDxWBGt+2cz72cQ+lf2HaFmG+9m6yEg9pkjIPYZZBUQ7Wf1xoEw/qP5LBGQ0GpzDwEItDOBjhKQFavfdD+5/m73wtrN/ipYOs3q+zNu9wJysPfzSk4LFq/xy3RJ3V37e93Ty9f7x/Hle4OAaPzIY8++7PSbIboClh5zFazmG89mBcC6HgJinzkCYp9BIwKifVai8f8fGfED0nVEZN7CS06/DWK9P7fz6yMglaacGQhAoI0JdJSAbN2xx33vp7d4SZAo6KbH72zflYho9PKYu+ehRYlyGiMyfGmkUi4IiI6MhHVJPCQ4CEj3dSAQEPvMERD7DBoVkHbu6Jf1vSMglWaaGQhAoI0JdJSAKAedJnXufJ/TEY8z5y5MOiUrzmpoeMQdPnrCDQ5dihf7+SAgGgOi07K0zrSpiEaKMSD2HS8ExD4DBMQ+AwTEPgMEJK0lZhkEINBuBDpOQKYrgFhA6hcogKYAACAASURBVK0TAbFvkIvIAAGxzxkBsc8AAbHPAAGp1yLzHAQg0C4EEJAaSe09mP5L6NXFi+j8cgTEvtFHQOwzQEDsM0BA7DNAQKpbYR5DAALtSAABqZPa5bGxuqdwqSoCYt8gF5EBAmKfMwJinwECYp8BAlKn0eYpCECgbQggIDmjKqLzyxEQ+0YfAbHPAAGxzwABsc8AAcnZaFMdAhAoBQEEJGcMCIh9g1xEBgiIfc4IiH0GCIh9BghIzkab6hCAQCkIICA5Yyii88sREPtGHwGxzwABsc8AAbHPAAHJ2WhTHQIQKAUBBCRnDAiIfYNcRAYIiH3OCIh9BgiIfQYISM5Gm+oQgEApCCAgOWMoovPLERD7Rh8Bsc8AAbHPAAGxzwABydloUx0CECgFAQQkZwwIiH2DXEQGCIh9zgiIfQYIiH0GCEjORpvqEIBAKQggIDljKKLzyxEQ+0YfAbHPAAGxzwABsc8AAcnZaFMdAhAoBQEEJGcMCIh9g1xEBgiIfc4IiH0GCIh9BghIzkab6hCAQCkIICA5Yyii88sREPtGHwGxz6DdBOTpzwcL+aHSIv4HhddAQOz3AwQkZ6NNdQhAoBQEEJCcMYSGuZX3CIh9o4+A2GfQbgLyvz8ZcDvP2HObzv9NCIh9nghIzkab6hCAQCkIICA5Y5jOxr3WuhAQ+0YfAbHPoJ0E5O0vB90/7R1w84531lEQBMR+P0BAcjbaVIcABEpBAAHJGUMtaZjO5QiIfaOPgNhn0E4C8h9HL3oB+e7Bix11GhYCYr8fICA5G22qQwACpSCAgOSMYTpFo9a6EBD7Rh8Bsc+grAJy4NyQk3D8y6Grt3/eN+AFREdB4uUzDl90OjpSa18v+3IExH4/QEByNtpUhwAESkEAAckZQxEdBgTEvtFHQOwzKKuA6H+AJOSG3itHPSQdabf/c7C95UPbiYDY7wcISM5Gm+oQgEApCCAgOWNAQOwb5CIyQEDscy6zgITP4IqTg+5/RUc/gojoCEko0873CIj9foCA5Gy0qQ4BCJSCAAKSM4YiOhMcAbFv9BEQ+wzaQUD0/0BHOoJ4hPtOuRoWAmK/HyAgORttqkMAAqUggIDkjAEBsW+Qi8gAAbHPuR0ERKIRpCMWkU65GhYCYr8fICA5G22qQwACpSCAgOSMoYjOL0dA7Bt9BMQ+g3YQEImGBGT2sSsDzdedunJK1v89zClYRfyv7IbXQEByNtpUhwAESkEAAckZQxENHgJi3/lFQOwzaAcBSbvKlQaoa7nui/h/0crX4AiIfYYISM5Gm+oQgEApCCAgOWNoZWMf1o2A2Df6CIh9Bu0gIPUko95zYV8v+z0CYr8fICA5G22qQwACpSCAgOSMoYgOAwJi3+gjIPYZtIOAFPH/wPI1EBD7/QABydloUx0CECgFAQQkZwxFdAYQEPtGHwGxzwABsc8AAbHPAAHJ2WhTHQIQKAUBBCRnDAiIfYNcRAYIiH3OCIh9BgiIfQYISM5Gm+oQgEApCCAgOWMoovPLERD7Rh8Bsc8AAbHPAAGxzwABydloUx0CECgFAQQkZwwIiH2DXEQGCIh9zgiIfQYIiH0GCEjORpvqEIBAKQggIDljKKLzyxEQ+0YfAbHPAAGxzwABsc8AAcnZaFMdAhAoBQEEJGcMCIh9g1xEBgiIfc4IiH0GCIh9BghIzkab6hCAQCkIICA5Yyii88sREPtGHwGxzwABsc8AAbHPAAHJ2WhTHQIQKAUBBCRnDAiIfYNcRAYIiH3OCIh9BgiIfQYISM5Gm+oQgEApCCAgOWMoovPLERD7Rh8Bsc8AAbHPAAGxzwABydloUx0CECgFAQQkZwwIiH2DXEQGCIh9zgiIfQYIiH0GCEjORpvqEIBAKQggIDljKKLzyxEQ+0YfAbHPAAGxzwABsc8AAcnZaFMdAhAoBQEEJGcMCIh9g1xEBgiIfc4IiH0GCIh9BghIzkab6hCAQCkIICA5Yyii88sREPtGHwGxzwABsc8AAbHPAAHJ2WhTHQIQKAUBBCRnDAiIfYNcRAYIiH3OCIh9BgiIfQYISM5Gm+oQgEApCCAgOWMoovPLERD7Rh8Bsc8AAbHPAAGxzwABydloUx0CECgFAQQkZwwIiH2DXEQGCIh9zgiIfQYIiH0GCEjORpvqEIBAKQggIDljKKLzyxEQ+0YfAbHPAAGxzwABsc8AAcnZaFMdAhAoBQEEJGcMCIh9g1xEBgiIfc4IiH0GCIh9BghIzkab6hCAQCkIICA5Yyii8zu4b7Mb/vKIi1/rwq633NkjBxPL4ueZn96OAgIyvTyb+XwiIPYZICD2GSAgORttqkMAAqUggIBMEcPExIQ7fvK069l72PUPDE4q3UxHqpE6Eg237N+cW3lDRTjCsonnry5rZJ2UbbwTgYA0zmy6P2cIiH0GCIh9BgjIpGaYBRCAQBsSQEDqhHbyy3Puh9fOct/67nWV258WrXaSkjBNdycrXl8QDS8gy/7NSTj6t794RUgkJf9YxpGQ1ncKEJDWM44/+2nzCIh9BgiIfQYISGh9uYcABNqZAAJSIz1Jxo9+fqf7/ozb3dYdu13vsS/cwqVrvYis27itUiutozRdyyQWE8//IiEcQUbC/ejf7nCnvjzHqVhftbZjgIC0lm+WfQYBsc8AAbHPAAGpNL/MQAACbUwAAakR3gc9B7xsvLHlg0SJa26Y7XQLU5aOU54yXkJWpksI8lFcZwABKY51rf0FAbHPAAGxzwABCa0v9xCAQDsTQEBqpLdq/RYvIEePn0qUmDt/uV8eTsOq1VmazuXDH65KPQpy7lAPRz5afOQj5IiA2He8EBD7DBAQ+wwQkESTzAMIQKBNCSAgNYJ7ctmV063OnLuQKPHoMy95Aenrv+iXf/XNiGvlbWDv5lT58OM/Vt7gLhz/tKWv38pta6d19w9ddiOXx2Hd4s97vc9E38VRd3lsggwMM9BvEo2NOzIwzODrgRE3PlHuDBKNJg8gAAEIpBBAQFKgaFEQjfMX+hMlnli0xgvIufN9ieUtedD7bk35CGNAdHUsd/5YS16elUIAAhCAAAQgAAEIQGC6CSAgNYguWfW6F41TZ84nSjy0YKVfPjIy6pdfGh13rbqNnDnqXDT+Y2Ldf7rRE3snLbt0sb9l76FV29Zu6x0dG3fj4xNwbuHnfarPxOjlcTc+QQZTcWrl8zoKqIsAtvI1WHf9NqUdMkg0mjyAAAQgkEIAAUmBokXrN77nRWP3/t5EiVvnLHDf/sGNlWVhjECr7sMg9PF1/1m52lW4OhaD0Is7H5sxIMWxrrUvMQbEPgPGgNhnwBiQSvPLDAQg0MYEEJAa4R07cdoLyDN/WV8pcWlk1MvHzLvmV5bV6ixN5/L+E5+6wf5vEgPOJSFcfre4zgACUhzrWvsOAmKfAQJinwECUml+mYEABNqYAAJSJ7xrb5rrJUS/+6FfQr/tD0/6x5u39VRq1eosTedyDfwcvDSWEJDpXD/rmrpTgYBMzajVnyMExD4DBMQ+AwSk0vwyAwEItDEBBKROeGe/uuBm/Po+Lx3h19A1NiSeWt3p0voREPtGHwGxzwABsc8AAbHPAAGJW2DmIQCBdiWAgGRI7kLfgP8l9MtjY5NKIyD2DXIRGSAg9jkjIPYZICD2GSAgk5phFkAAAm1IAAHJGVoRnV+OgNg3+giIfQYIiH0GCIh9BghIzkab6hCAQCkIICA5Y0BA7BvkIjJAQOxzRkDsM0BA7DNAQHI22lSHAARKQQAByRlDEZ1fjoDYN/oIiH0GCIh9BgiIfQYISM5Gm+oQgEApCCAgOWNAQOwb5CIyQEDsc0ZA7DNAQOwzQEByNtpUhwAESkEAAckZQxGdX46A2Df6CIh9BgiIfQYIiH0GCEjORpvqEIBAKQggIDljQEDsG+QiMkBA7HNGQOwzQEDsM0BAcjbaVIcABEpBAAHJGUMRnV+OgNg3+giIfQYIiH0GCIh9BghIzkab6hCAQCkIICA5Y0BA7BvkIjJAQOxzRkDsM0BA7DNAQHI22lSHAARKQQAByRlDEZ1fjoDYN/oIiH0GCIh9BgiIfQYISM5Gm+oQgEApCCAgOWNAQOwb5CIyQEDsc0ZA7DNAQOwzQEByNtpUhwAESkEAAckZQxGdX46A2Df6CIh9BgiIfQYIiH0GCEjORpvqEIBAKQggIDljQEDsG+QiMkBA7HNGQOwzQEDsM0BAcjbaVIcABEpBAAHJGUMRnV+OgNg3+giIfQYIiH0GCIh9BghIzkab6hCAQCkIICCliIE3AQEIQAACEIAABCAAge4ggIB0R85sJQQgAAEIQAACEIAABEpBAAEpRQy8CQhAAAIQgAAEIAABCHQHAQSkO3JmKyEAAQhAAAIQgAAEIFAKAghIwTFcHBx2u/b3umMnTrvx8YmGXj1L3aHhEbfnwFH38Z5DbnT0ckPr78TCExMT7vjJ065n72HXPzDY0CaK34FPj7tPDh9zIyOjNeueOHXWvf/Rfnfq9FepZQaHhl1f/8XUW2qFLliYlW0aiix1x8bGfe4f7T7kzpy7kLaarlqWZz/IWlecxVv/28S/emI/qCaS/jjL5zu95pWlaicu9A3UK8JzEIAABMwJICAFRvDsilfct757XeX2vZ/e4hvrLG8hS9233+tx3/7BjZX167Vu+8OTXdsBO/nlOffDa2clePxp0WqnDtVU0/ad+xL1xFJ840lC89t7Hk+U+8n1d7vVr70bF3O/vH1eokz8GUgU7JIHWdjWQpGlbu9nJ901N8xOMNd+0KiA1noP7bY8z36Qpa7k+nf3PZXgLf69x75IoGI/SOBIfZDl851a8R8L9aXWr+541M349X31ivEcBCAAAXMCCEhBEbz57k7fQD/27Mvu06MnnRqaH/38Tvf9Gbe7y2Njdd9FlrrHT57x61fjc/T4KacjIW9s+cAvu3HWw3XX34lPSjIC3607dvvO0MKlaz2PdRu31d3k02e/9uV+c+djbs8nR9y+g5+5W+cs8Mt0NCVMs+Y+7ZeJs3iLu+pIMHSUK0zqeH3nxzPda5t2JG6q121TVrZpXLLUPX+h37NW9hs273A6OrVmw1afiWQxi3ymvXa7LsuzH2Stq8+3vvh4fs0m/4XKtg/3+v9r+oJFchIm9oNAIv0+y+c7vabzR7vnzFvi9AWI/v8gILVIsRwCECgLAQSkoCR+NvN+/218/HLqGKuxeGf7rnjxpPksdbe893e/LnW24kkNkTpj3TZ90HPA86ju5OubWd3qTUFU4lN31JFSVg8/tapSVY29xCI+lU6dL5Xb/N8fV8qp46WyTM5lZZvGKkvdVze97/nv3HUwsYpQV0dHumnKsx9kqavTDvV5f2rZugTWUHfV+i2V5ewHFRSpM+EzOtX/nbTKkkV9OaKb/r8hIGmUWAYBCJSJAAJSUBr6hvCO+59JvJoaGjXeS158PbG8+kGWut8MDPp1qaw63eoUHzpywi9Tw9Ztkzo+YqujEvE0d/5yv7zeN+E6XUdiUT3pdC4dYQrTgsVr/Lr0zbqy1DqVsV43Pt1HHS99G6xOscbmxB2MsK5uuc/KNo1HlrorVr/l+WscVDxpn1AuEvVumvLsB1nqanyU/x+2Kvk/LPxvm/fUixXc7AcVFKkzWT7fqRWrFuoLKwSkCgoPIQCB0hFAQAqIRIMK1Uir8xtPAxeH/PK4kY6f13wjdUNnQK+lDq9k5J6HFk15ilf1a3bC4yeXXTndqrqz/+gzL3nm8akh1dt73c0PTDpapTL/+u/3Jo5k6NS5ux74s1+fmOubRzE/2Pt5YpXqeOn5+Hbz759wg0OXEuW64UFWtmksstTd0fOJ51y9T4UjhG9s+TBt1R27LM9+kKWuTj3U51odXv2vClM4Yhj/z2M/CHTS77N8vtNrJpciIEkePIIABMpJAAEpIJcgGg8uWJl4tdB43//4isTy+EHWuuoMP7Rgpe8A61vgX9z2R98x0BgTiUm3TUE0NCYgnp5YdOWoxbnzffHixLxEI+20tWtvmpsQE4mGRE8DcDW2R/Khzphee/Ty1XE9OsVOp2bpW/mtO/ZUxpPMnrc48brd8CAr2zQWWeqKuzpgykFjnyQi8YUCdGpQN0159oOsdfXZF2+dZijekvKwL8RHX9kP6n/ysny+66/hyrMISBZKlIEABKwJICAFJKBTc9RA3/vI0sSr6VKJWh430okCzvnTerLUVeOucvHYA3V21RHQqUPdNi1Z9brncerM+cSmS9LEqd5ldXWalcStelIHS417mMJ4knDJUcmiTqPQ+ut9067T49TZSDvNK6y7U++zsk3b/qx1+7656P648IXKRQgkIJJH5RJfRCDtNTptWZ79IGtdff71pYc+0xJyfZMfropVPSYt5tvN+0HMIcxn/XyH8rXuEZBaZFgOAQiUiQACUlAaapirr0alb9DVKXph7ea67yJL3TC2QWNB4umBJ573r/H5F2fixR0/v37je367d0dXo9JG62pWkrJ6k05bUy7xUQx1llRP9TWFc9zjQela/nVfv69794PP1XsJd+d/PevLTXUFtLoracMns7CttVnN1lV22oeUXz3xrPW67bw8z36Qp27Iav+hz+ri69b9IA1KYFbv/05aveplCEg1ER5DAAJlJICAFJRKGJysH+MK0/KXN/pOaHyK1N6DOk1nt4vLZamrjrA6zaobT+GysPVOOYrLd8q8fgxNPJ75y/rKJl0aGfWd0Jl3za8s02BxMYszCJdt1YDxMKkjpfWtWP2mXxTOca++opYu2atyEj9N6vAOXxoJq/H3OldeR1i68chUFraCJGFWLvGPO2atm4DtnD/9TZlUX6mpulwnPs6zH2StW80tSLiOhISJ/SCQqH2f9fOdtm/Ea0VAYhrMQwACZSWAgBSUjH4pW50gnQ6ijq1+E0KPqy/PqsPwWq7fCglTlrr63QnV09iFt7Z+5Dtwui6/lukc7W6cwmk3+t0P/RJ6OD1q87arPyiocRlidOuchRVEQS4kCOoEa/yGclK5WORCVjpX/sjxU07jC66/5UFfLvwOiMRGp1o9vXy9H/+h0+LCYNxV696uvGa3zGRlG07/iRllratMNehcv7UTBlJrv9CpWd04NbsfiFWWutondLTkw78fcLpylj7vOtqk39AJE/tBIFH7PuvnO23f0Glw73+839/0Wdf/rvC43hX/ar8bnoEABCDQWgIISGv5Jtb+4trNvnOqjqxu+va8eoxCOGJR/XsFWerqUqNq/MP6da+jJ9WnZSXeVAc/OPvVBX91npiHGu940hEnPS85iScJRMxSHarqAcwa4C5xidevOvE4HH2Dr5zjMppf/MIGF8aOxK/bDfNZ2C596Q3PTB3aeMpSV6c0Bt7Kbc68JYkjivH6umE+z36QpW643HdgLsHWD6PGE/tBTKP2fJbPd9q+ES5oEjKI7+NTumq/Ms9AAAIQKJYAAlIsb39JXH1bXn11pixvQ+MFpqqrMjpEr28cu1U8qllqsH/vsS+auhzxF1+e87+mXe9bRH1zKd7iXmtMh7LQe9Ag6Fplqt93pz/OwrYWg3p1dcqbjiDql6WZrhLIsx/UqyuR1ula+mxPJdXsB1fzqDdX7/Ndrx7PQQACEGgXAghIuyTF+4QABCAAAQhAAAIQgEAHEEBAOiBENgECEIAABCAAAQhAAALtQgABaZekeJ8QgAAEIAABCEAAAhDoAAIISAeEyCZAAAIQgAAEIAABCECgXQggIO2SFO8TAhCAAAQgAAEIQAACHUAAAemAENkECEAAAhCAAAQgAAEItAsBBKRdkuJ9QgACEIAABCAAAQhAoAMIICAdECKbAAEIQAACEIAABCAAgXYhgIC0S1K8TwhAAAIQgAAEIAABCHQAAQSkA0JkEyAAAQhAAAIQgAAEINAuBBCQdkmK9wkBCEAAAhCAAAQgAIEOIICAdECIbAIEIAABCEAAAhCAAATahQAC0i5J8T4hAAEIQAACEIAABCDQAQQQkA4IkU2AAAQgAAEIQAACEIBAuxBAQNolKd4nBCAAAQhAAAIQgAAEOoAAAtIBIbIJECiSwMTEhAu3Wq8bntd9rSlLGdXNWq7W61waGXX7D33mDh894UZHL9cqVnP50PCIGxkZrfl8eGL08pg72Pu5O3TkhLs8NhYWcw8BCEAAAhCAQBUBBKQKCA8hAIH6BH708zvdt757nfv2D250g0OXJhUeH59woYzKjY2NTyqjBTN+fZ9fj8qcOXchtcyDC1ZWyqicbt+fcbu764E/+85+aiXn/GuuWP2Wu+aG2ZPqa9mq9Vuc3udU0+DQsK//23ser1lUgrRw6dpJr/Pn51+tWYcnIAABCEAAAt1MAAHp5vTZdgg0QSCWi+Uvb5y0hne270p0xtOOBhw9fipRZulLb0xajxYEAfnXf7/X3fz7J9wvbvuj+86PZ1bqfrT70KR6x0+edtfeNNeX+d19T7mXXtni9nxyxPXsPexefuUdd+uchf45rev02a8n1deCvQePutc27aisp56ABPn43k9vcU8sWuMef+6vlfe4+IUNqetnIQQgAAEIQKCbCSAg3Zw+2w6BJggEAdEREMnA8KWRxFpC5z8csUgTEHXU9fw9Dy3y91pn2hQE5M13d1ae1voeeXqVr/fDa2dVlmumr/+if096X+9/tD/xXPzg7fd6/BGcn82835/iFT+n+Vhy9D5rCYi2XRx0O37yTGU1R/4hWFqPTs1iggAEIAABCEDgKgEE5CoL5iAAgQwEgoDoFCN1zl9Yu7lS6+M9h/wyHQVQp1zPVwuITskKHfy+by5WTtfad/CzynrCTJqA6DmtU+vWTdIRJp2apWVBBjQeY/a8xf50L0nEo8+85GbNfdp98eU5t3XHbl923cZtoXrl/tiJ0+7Toyfdpq0f+zK1BGTzth7/vI60VE+/ufMx/9z2nfuqn+IxBCAAAQhAoKsJICBdHT8bD4HGCQQB6R8Y9JIhmQiDtH91x6O+033+Qn9NAdGRCUmCToXSpNOv9FiyUT3VEhCVC4LzdV+/r6bTprSe59ds8o937jroH2uZxpvcOOvhyuNPDh/zZSQJtY6+qEDvZyd9nVoColPQtP4Vq9/064v/BEHTKWBMEIAABCAAAQhcJYCAXGXBHAQgkIFAEBAVDeMf/vrqu+7Ap8d9Z/zhp1b5tQRBqD4Ccud/PevLaayIJg1AVyde5YPI+CeiMSDxKVh67pU3t/s6GpAeJomH1qOrXmlguE7P0mOJSZj0PrUsCMiaDVv94+rXDeWnEhAdUdH60o6iaKC7nnty2dqwOu4hAAEIQAACEHDOISB8DCAAgYYIxAJyoW/Ad7I1AFuDxNXhPvvVlStapQnIwMUhX0bPxWMjfnn7PL9cpzTFUzgCMnf+cidZWPTCa+7WOQt8Wb2WTqMKk06/+sn1d/uHQWo00DyegnAEAdHAdK1HopE2TSUgc+Yt8fXf2PLhpOqSEq37oZQjO5MKswACEIAABCDQRQQQkC4Km02FwHQQiAVE6wtHAdTZfuCJ5ysvkSYgQQB06pN+myPcwjqqT3UKAqJ1xzddSlfyEE/X3fxAZbC4rnql8hrsHk/h9YOA6IpZKrdrf29crDI/lYCEwfA6IlM9hSMgOkrEBAEIQAACEIDAVQIIyFUWzEEAAhkIVAvIufN9FTn48uz5yhrSBERXnYpFIm1e40fCFATk2RWv+Evp6upS1VfdCmV1lCRcFUuDzLVuiU48VQtIGDSvbUibphKQMH4lHogf1vPcytf8e5CIMEEAAhCAAAQgcJUAAnKVBXMQgEAGAtUCoio6deqtrR8lalcLiK5MJSnQuA2Nxai+hdOwwiByrSwISPUYkMQL/eOBfuND69ePI2rcSbjS1rYP9/oxIZKSMBA9HAF5atk6Xy5tfVo2lYDofek1dfpX9aSjOXouPk2sugyPIQABCEAAAt1IAAHpxtTZZgjkIJAmIGmrqxYQdfbVIdeRgbTpg54D/nmdXhWmRgQkyEI45WnD5h1+fXrN8F7CvcZuqJzGrixZ9Xp4uUn3YZ3Vp4aFgpIdrVO3U2euHv0Jp3Zpea0B7mEd3EMAAhCAAAS6jQAC0m2Js70QyEmgGQEZH5/wnX3JgI5EpE3x74Mc7P3cF2lEQFQhjMnQFbk06RQrreMPjy5zr25630kM7nt0WeWm07YkEdWTfsFc40fCIHMdtdFj3cJ7C3X+tGi1Fx2VWbjkb27B4jWVbdWpY0wQgAAEIAABCCQJICBJHjyCAASmINCogEgsNMhb8qExIPWmMBhdP2SoKQiIfhAwyzQ0PFK5/K7GZUh8mpnC6Vt6z9W31zbtSKxSl/yVmFSXQz4SmHgAAQhAAAIQqBBAQCoomIEABDqBwDcDg+7eR5Z6IdCg9HseWuT0g4EaI6LTwHQ61eDQ8LRvqsadHD56wo8bqf7tk2l/MVYIAQhAAAIQaGMCCEgbh8dbhwAEahPQ6Vc6JUu/BRLGfujIhh63QkBqvxOegQAEIAABCEAgJoCAxDSYhwAEIAABCEAAAhCAAARaSgABaSleVg4BCEAAAhCAAAQgAAEIxAQQkJgG8xCAAAQgAAEIQAACEIBASwkgIC3Fy8ohAAEIQAACEIAABCAAgZgAAhLTYB4CEIAABCAAAQhAAAIQaCkBBKSleFk5BCAAAQhAAAIQgAAEIBATQEBiGsxDAAIQgAAEIAABCEAAAi0lgIC0FC8rhwAEIAABCEAAAhCAAARiAghITIN5CEAAAhCAAAQgAAEIQKClBBCQluJl5RCAAAQgAAEIQAACEIBATAABiWkwDwEIQAACEIAABCAAAQi0lAAC0lK8rBwCEIAABCAAAQhAAAIQiAkgIDEN5iEAAQhAAAIQgAAEIACBlhJAQFqKl5VDAAIQgAAEIAABCEAAAjEBBCSmwTwEIAABCEAAAhCAAAQg0FICbA2HHAAAAXBJREFUCEhL8bJyCEAAAhCAAAQgAAEIQCAmgIDENJiHAAQgAAEIQAACEIAABFpKAAFpKV5WDgEIQAACEIAABCAAAQjEBBCQmAbzEIAABCAAAQhAAAIQgEBLCSAgLcXLyiEAAQhAAAIQgAAEIACBmAACEtNgHgIQgAAEIAABCEAAAhBoKQEEpKV4WTkEIAABCEAAAhCAAAQgEBNAQGIazEMAAhCAAAQgAAEIQAACLSWAgLQULyuHAAQgAAEIQAACEIAABGICCEhMg3kIQAACEIAABCAAAQhAoKUEEJCW4mXlEIAABCAAAQhAAAIQgEBMAAGJaTAPAQhAAAIQgAAEIAABCLSUAALSUrysHAIQgAAEIAABCEAAAhCICSAgMQ3mIQABCEAAAhCAAAQgAIGWEkBAWoqXlUMAAhCAAAQgAAEIQAACMQEEJKbBPAQgAAEIQAACEIAABCDQUgIISEvxsnIIQAACEIAABCAAAQhAICbwP2ZmWhuL99OkAAAAAElFTkSuQmCC" + } + }, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![image.png](attachment:image.png)" + ] + }, { "cell_type": "markdown", "metadata": {},