{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bxRbYcm0xml2",
    "outputId": "faa839d8-b1a2-444d-f60b-e2d0e47d6a05",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.272050Z",
     "start_time": "2025-05-12T17:20:33.238764Z"
    }
   },
   "source": [
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "# Download required NLTK data\n",
    "nltk.download('punkt_tab')"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     C:\\Users\\Asus\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 112
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "0JRlocqh2M2X",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.438559Z",
     "start_time": "2025-05-12T17:20:33.392866Z"
    }
   },
   "source": [
    "# Function to remove repeated words or sentences\n",
    "def remove_repeated(text):\n",
    "    # Remove repeated words (same word repeated consecutively)\n",
    "    text = re.sub(r'\\b(\\w+)( \\1)+\\b', r'\\1', text)\n",
    "    return text"
   ],
   "outputs": [],
   "execution_count": 113
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "-CnvJDYM2NXn",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.512683Z",
     "start_time": "2025-05-12T17:20:33.480576Z"
    }
   },
   "source": [
    "# Function to remove HTML tags\n",
    "def remove_html_tags(text):\n",
    "    # Regex to remove HTML tags\n",
    "    return re.sub(r'<.*?>', '', text)"
   ],
   "outputs": [],
   "execution_count": 114
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "B7tKJ-TQ2NdG",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.579255Z",
     "start_time": "2025-05-12T17:20:33.542969Z"
    }
   },
   "source": [
    "\n",
    "# Function to remove extra spaces\n",
    "def remove_extra_spaces(text):\n",
    "    # Replace multiple spaces with a single space\n",
    "    return re.sub(r'\\s+', ' ', text).strip()"
   ],
   "outputs": [],
   "execution_count": 115
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "AXGDy3RQ2NhV",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.641530Z",
     "start_time": "2025-05-12T17:20:33.614727Z"
    }
   },
   "source": [
    "# Function to remove special characters (except spaces and common punctuation)\n",
    "def remove_special_characters(text):\n",
    "      # Define the pattern to match special characters (including Hindi characters)\n",
    "    pattern = r'[^\\u0900-\\u097F\\s]'  # Range for Hindi Unicode characters\n",
    "\n",
    "    # Use regex to substitute special characters with an empty string\n",
    "    clean_text = re.sub(pattern, '', text)\n",
    "\n",
    "   # clean_text = re.sub(r'[^\\u0900-\\u097F\\u0966-\\u096F\\s]', '', text)\n",
    "    return clean_text\n"
   ],
   "outputs": [],
   "execution_count": 116
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "yqCQBBXY2NkJ",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.689658Z",
     "start_time": "2025-05-12T17:20:33.672666Z"
    }
   },
   "source": [
    "# Function to replace Devanagari numerals with digits\n",
    "def replace_devanagari_numerals(text):\n",
    "    devanagari_to_digits = {'०': '0', '१': '1', '२': '2', '३': '3', '४': '4',\n",
    "                            '५': '5', '६': '6', '७': '7', '८': '8', '९': '9'}\n",
    "    # Replace each Devanagari numeral with corresponding digit\n",
    "    for devanagari, digit in devanagari_to_digits.items():\n",
    "        text = text.replace(devanagari, digit)\n",
    "    return text"
   ],
   "outputs": [],
   "execution_count": 117
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "JD0sUF2R2NnD",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.740034Z",
     "start_time": "2025-05-12T17:20:33.709019Z"
    }
   },
   "source": [
    "\n",
    "# Function to filter out sentences with more than 80 words\n",
    "def filter_long_sentences(text):\n",
    "    # Tokenize the sentence into words\n",
    "    words = word_tokenize(text)\n",
    "    # If the sentence has more than 80 words, return None (exclude)\n",
    "    if len(words) > 80:\n",
    "        return None\n",
    "    return text"
   ],
   "outputs": [],
   "execution_count": 118
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.783812Z",
     "start_time": "2025-05-12T17:20:33.767595Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Function # Remove space before and after the Halant (्), but preserve newlines\n",
    "def halant(text):\n",
    "\n",
    "    clean_text = re.sub(r'(?<=\\S)\\s+्\\s+', '्', text)\n",
    "    return clean_text"
   ],
   "outputs": [],
   "execution_count": 119
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "NSohqCGrx7Fv",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.825627Z",
     "start_time": "2025-05-12T17:20:33.806247Z"
    }
   },
   "source": [
    "# Pre-processing function that applies all steps\n",
    "def preprocess_text(hindi_text, dogri_text):\n",
    "    # Remove HTML tags from both texts\n",
    "    hindi_text = remove_html_tags(hindi_text)\n",
    "    dogri_text = remove_html_tags(dogri_text)\n",
    "\n",
    "    # Remove repeated words or sentences\n",
    "    hindi_text = remove_repeated(hindi_text)\n",
    "    dogri_text = remove_repeated(dogri_text)\n",
    "\n",
    "    # Remove extra spaces\n",
    "    hindi_text = remove_extra_spaces(hindi_text)\n",
    "    dogri_text = remove_extra_spaces(dogri_text)\n",
    "\n",
    "    # Remove special characters\n",
    "    hindi_text = remove_special_characters(hindi_text)\n",
    "    dogri_text = remove_special_characters(dogri_text)\n",
    "\n",
    "    # Replace Devanagari numerals with digits\n",
    "    hindi_text = replace_devanagari_numerals(hindi_text)\n",
    "    dogri_text = replace_devanagari_numerals(dogri_text)\n",
    "\n",
    "    # Filter long sentences\n",
    "    hindi_text = filter_long_sentences(hindi_text)\n",
    "    dogri_text = filter_long_sentences(dogri_text)\n",
    "\n",
    "    hindi_text = halant(hindi_text)\n",
    "    dogri_text  = halant(dogri_text)\n",
    "\n",
    "    # If any text is excluded due to long sentence, return None\n",
    "    if hindi_text is None or dogri_text is None:\n",
    "        return None\n",
    "    return hindi_text, dogri_text"
   ],
   "outputs": [],
   "execution_count": 120
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "XXFcvs2D4Zni",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "15cbda47-7a7d-4709-e48e-99b6f04e25dc",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.879292Z",
     "start_time": "2025-05-12T17:20:33.853753Z"
    }
   },
   "source": [
    "hindi_text = \"नमस्ते! क्२या   हाल <html> है? 😊\"\n",
    "dogri_text = \"नमस्ते! क्या २ ह   ाल है? 😊\"\n",
    "cleaned_text = preprocess_text(hindi_text,dogri_text)\n",
    "print(cleaned_text)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('नमस्ते क्2या हाल है ', 'नमस्ते क्या 2 ह ाल है ')\n"
     ]
    }
   ],
   "execution_count": 121
  },
  {
   "cell_type": "code",
   "source": [
    "def fix_smt_spacing(text):\n",
    "\n",
    "    #Removes unwanted spaces around Devanagari characters, especially around the Halant (्),\n",
    "    # Remove space before and after the Halant (्), but preserve newlines\n",
    "    text = re.sub(r'(?<=\\S)\\s+्\\s+', '्', text)\n",
    "    return text"
   ],
   "metadata": {
    "id": "ElYqLE9a-j9q",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:33.989715Z",
     "start_time": "2025-05-12T17:20:33.948020Z"
    }
   },
   "outputs": [],
   "execution_count": 122
  },
  {
   "cell_type": "code",
   "source": [
    "def process_hindi_text_file(input_file, output_file):\n",
    "\n",
    "\n",
    "    #Reads a Hindi text file, processes it to fix spacing issues, and writes the cleaned text to another file.\n",
    "\n",
    "    try:\n",
    "        # Open the input file in read mode with UTF-8 encoding\n",
    "        with open(input_file, 'r', encoding='utf-8') as infile:\n",
    "            content = infile.read()\n",
    "\n",
    "        # Apply the spacing fix function to the content\n",
    "        cleaned_content = fix_smt_spacing(content)\n",
    "\n",
    "        # Open the output file in write mode with UTF-8 encoding\n",
    "        with open(output_file, 'w', encoding='utf-8') as outfile:\n",
    "            outfile.write(cleaned_content)\n",
    "\n",
    "        print(f\"File processed successfully. Cleaned text saved to '{output_file}'.\")\n",
    "\n",
    "    except FileNotFoundError:\n",
    "        print(f\"Error: The file '{input_file}' was not found.\")\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {e}\")\n"
   ],
   "metadata": {
    "id": "2NKRKRhaZ1uF",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:34.037393Z",
     "start_time": "2025-05-12T17:20:34.023351Z"
    }
   },
   "outputs": [],
   "execution_count": 123
  },
  {
   "cell_type": "code",
   "source": [
    "# Example usage\n",
    "#input_file_path = '/content/drive/MyDrive/code/TestData/testPredCombine1.txt'\n",
    "#output_file_path = '/content/drive/MyDrive/code/TestData/testPredCombine1_Aftr.txt'\n",
    "#process_hindi_text_file(input_file_path, output_file_path)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4gz-6739aEAL",
    "outputId": "b69f3476-3de6-41c9-ccf2-d0a0cb77c4f2",
    "ExecuteTime": {
     "end_time": "2025-05-12T17:20:34.065167Z",
     "start_time": "2025-05-12T17:20:34.060181Z"
    }
   },
   "outputs": [],
   "execution_count": 124
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
