From 54b316ca1cca5809ec97495e79175eb586c13fd8 Mon Sep 17 00:00:00 2001
From: trueroyce <trueecyor@gmail.com>
Date: Thu, 9 Jan 2025 02:36:47 +0000
Subject: [PATCH] Added news_fake_example.ipynb and updated retnet/somefile.py
 for compatibility

---
 news_fake_example.ipynb   | 612 ++++++++++++++++++++++++++++++++++++++
 retnet/modeling_retnet.py |   2 +-
 2 files changed, 613 insertions(+), 1 deletion(-)
 create mode 100644 news_fake_example.ipynb

diff --git a/news_fake_example.ipynb b/news_fake_example.ipynb
new file mode 100644
index 0000000..d89f353
--- /dev/null
+++ b/news_fake_example.ipynb
@@ -0,0 +1,612 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "bb6eb078",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !pip install transformers==4.38.2\n",
+    " "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c555460c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from retnet.modeling_retnet import RetNetForSequenceClassification\n",
+    "from retnet.configuration_retnet import load_config_from_json\n",
+    "from transformers import AutoTokenizer, TextStreamer\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "from retnet.configuration_retnet import RetNetConfig\n",
+    "\n",
+    "import wandb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "74d4285a",
+   "metadata": {},
+   "source": [
+    "## Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "10092094",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "49301758b2e04e24ab743e6282f75d08",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/786 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1b425463743e47bb902cf6b4e192a83b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "(…)-00000-of-00001-31aecafc15fe32e0.parquet:   0%|          | 0.00/57.0M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cd89f9c0733641c5a076a500ef7af034",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "(…)-00000-of-00001-1d49200ad03190fd.parquet:   0%|          | 0.00/14.3M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "594fa51c24894cb9928559aab0a9a126",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating train split:   0%|          | 0/35918 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "048032ed67074563ab1ece21e1d7282a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Generating test split:   0%|          | 0/8980 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset = load_dataset(\"mohammadjavadpirhadi/fake-news-detection-dataset-english\")\n",
+    "\n",
+    "# data_files = {\n",
+    "#     \"train\": \"data/train-00000-of-00001-31aecafc15fe32e0.parquet\",   \n",
+    "#     \"test\": \"data/test-00000-of-00001-1d49200ad03190fd.parquet\"    \n",
+    "# }\n",
+    "# dataset = load_dataset(\"parquet\", data_files=data_files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "7cc9dee4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['title', 'text', 'subject', 'date', 'label'],\n",
+       "        num_rows: 35918\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['title', 'text', 'subject', 'date', 'label'],\n",
+       "        num_rows: 8980\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "5f5ad60a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "4272\n"
+     ]
+    }
+   ],
+   "source": [
+    "#long sequence\n",
+    "print(len(dataset['train'][1]['text']))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b17fee29",
+   "metadata": {},
+   "source": [
+    "## Token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "bb7acdf3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:795: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b0c324851b994236862cf1a6d3298b92",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "tokenizer = AutoTokenizer.from_pretrained('gpt2')\n",
+    "tokenizer.model_max_length = 16384\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "tokenizer.unk_token = tokenizer.eos_token\n",
+    "tokenizer.bos_token = tokenizer.eos_token"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "8373d419",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def tokenize_function(examples):\n",
+    "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True, max_length=4096)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "509a8afb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bbc5e46ba3494b86adb72f56b662b03f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/35918 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5d431f45537b49a58765e33d62b73eea",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/8980 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "tokenized_datasets = dataset.map(tokenize_function, batched=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1286e618",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c56f91f5",
+   "metadata": {},
+   "source": [
+    "## Train config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "7a631d34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# config = load_config_from_json('configs/retnet-base/config.json')\n",
+    "# print(config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "f99d104d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = RetNetConfig(\n",
+    "    activation_dropout=0.0,\n",
+    "    activation_fn=\"swish\",\n",
+    "    decoder_embed_dim=256,  \n",
+    "    decoder_ffn_embed_dim=512,  \n",
+    "    decoder_layers=6,  \n",
+    "    decoder_normalize_before=True, \n",
+    "    decoder_retention_heads=8,  \n",
+    "    decoder_value_embed_dim=512,  \n",
+    "    deepnorm=False,  \n",
+    "    drop_path_rate=0.0,  \n",
+    "    dropout=0.1,  \n",
+    "    initializer_range=0.02,  \n",
+    "    layernorm_embedding=True,  \n",
+    "    layernorm_eps=1e-06,  \n",
+    "    no_scale_embedding=False, \n",
+    "    recurrent_chunk_size=512,  \n",
+    "    subln=True,  \n",
+    "    use_ffn_rms_norm=False,  \n",
+    "    use_glu=True,  \n",
+    "    use_lm_decay=False,  \n",
+    "    z_loss_coeff=0.0  \n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "ce726fbf",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "RetNetForSequenceClassification(\n",
+      "  (model): RetNetModel(\n",
+      "    (dropout_module): Dropout(p=0.1, inplace=False)\n",
+      "    (embed_tokens): Embedding(50257, 256, padding_idx=0)\n",
+      "    (layernorm_embedding): RMSNorm()\n",
+      "    (layers): ModuleList(\n",
+      "      (0-5): 6 x RetNetDecoderLayer(\n",
+      "        (dropout_module): Dropout(p=0.1, inplace=False)\n",
+      "        (retention): MultiScaleRetention(\n",
+      "          (q_proj): Linear(in_features=256, out_features=256, bias=False)\n",
+      "          (k_proj): Linear(in_features=256, out_features=256, bias=False)\n",
+      "          (v_proj): Linear(in_features=256, out_features=512, bias=False)\n",
+      "          (g_proj): Linear(in_features=256, out_features=512, bias=False)\n",
+      "          (out_proj): Linear(in_features=512, out_features=256, bias=False)\n",
+      "          (group_norm): RMSNorm()\n",
+      "        )\n",
+      "        (retention_layer_norm): RMSNorm()\n",
+      "        (ffn): GLU(\n",
+      "          (activation_dropout_module): Dropout(p=0.0, inplace=False)\n",
+      "          (dropout_module): Dropout(p=0.1, inplace=False)\n",
+      "          (fc1): Linear(in_features=256, out_features=512, bias=False)\n",
+      "          (fc2): Linear(in_features=512, out_features=256, bias=False)\n",
+      "          (gate): Linear(in_features=256, out_features=512, bias=False)\n",
+      "        )\n",
+      "        (final_layer_norm): RMSNorm()\n",
+      "      )\n",
+      "    )\n",
+      "    (layer_norm): RMSNorm()\n",
+      "    (retnet_rel_pos): RetNetRelPos()\n",
+      "  )\n",
+      "  (score): Linear(in_features=256, out_features=2, bias=False)\n",
+      ")\n"
+     ]
+    }
+   ],
+   "source": [
+    "model = RetNetForSequenceClassification(config)\n",
+    "print(model)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "654233e7",
+   "metadata": {},
+   "source": [
+    "## Train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "68825db3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import Trainer, TrainingArguments\n",
+    "\n",
+    "train_args = TrainingArguments(\n",
+    "    output_dir=\"retnet-new-fake-test\",\n",
+    "    per_device_train_batch_size=8,\n",
+    "    per_device_eval_batch_size=8,\n",
+    "    evaluation_strategy=\"steps\",\n",
+    "    eval_steps=500,\n",
+    "    logging_steps=37,\n",
+    "    gradient_accumulation_steps=8,\n",
+    "    num_train_epochs=5,\n",
+    "    weight_decay=0.01,\n",
+    "    warmup_steps=10,\n",
+    "    fp16=True,\n",
+    "    lr_scheduler_type=\"cosine\",\n",
+    "    learning_rate=6e-4,\n",
+    "    optim=\"adamw_torch\"\n",
+    "#    report_to=\"none\"  #close wandb\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "44ba58fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=train_args,\n",
+    "    train_dataset=tokenized_datasets['train'],\n",
+    "    eval_dataset=tokenized_datasets['test'],\n",
+    "    tokenizer=tokenizer\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "494258d5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Using device: cuda\n"
+     ]
+    }
+   ],
+   "source": [
+    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "print(\"Using device:\", device)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d15143e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trainer.train()\n",
+    "trainer.save_model()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bb5e9f2a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fb5ef9b0",
+   "metadata": {},
+   "source": [
+    "## test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "b57f33eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load model\n",
+    "\n",
+    "model_path = \"retnet-new-fake/checkpoint-1000\"  \n",
+    "model = RetNetForSequenceClassification.from_pretrained(model_path)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "a80e2622",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "model = model.to(device)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "eef4560d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#use sample data to test\n",
+    "sample = tokenized_datasets['test'][4]\n",
+    "\n",
+    "input_ids = torch.tensor([sample['input_ids']]).to(device)  \n",
+    "attention_mask = torch.tensor([sample['attention_mask']]).to(device)  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "f2aa42a3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# forward_impl=\"parallel\" or \"recurrent\" or \"chunkwise\"\n",
+    "\n",
+    "model.eval()  \n",
+    "with torch.no_grad():\n",
+    "    outputs = model(input_ids=input_ids, attention_mask=attention_mask, forward_impl=\"recurrent\")\n",
+    "    logits = outputs.logits\n",
+    "    predictions = torch.argmax(logits, dim=-1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "57740e57",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([1], device='cuda:0')"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "predictions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "1a1895dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sample['label']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73c5c5c9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5253ce14",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/retnet/modeling_retnet.py b/retnet/modeling_retnet.py
index 2a2a580..d599cd6 100644
--- a/retnet/modeling_retnet.py
+++ b/retnet/modeling_retnet.py
@@ -9,7 +9,7 @@
 from timm.models.layers import drop_path
 from torch import nn
 from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
-from transformers import top_k_top_p_filtering
+from transformers.generation.utils import top_k_top_p_filtering
 from transformers.modeling_outputs import ModelOutput, SequenceClassifierOutputWithPast
 from transformers.modeling_utils import PreTrainedModel
 from transformers.utils import logging