{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!apt-get install tesseract-ocr\n",
        "!pip install pdfplumber pdf2image pytesseract\n",
        "!apt-get install -y poppler-utils\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mXFZpTwLOjTZ",
        "outputId": "7bea32f6-f53f-4946-c7ba-31b3b03059b7"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "tesseract-ocr is already the newest version (4.1.1-2.1build1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 35 not upgraded.\n",
            "Requirement already satisfied: pdfplumber in /usr/local/lib/python3.12/dist-packages (0.11.7)\n",
            "Requirement already satisfied: pdf2image in /usr/local/lib/python3.12/dist-packages (1.17.0)\n",
            "Requirement already satisfied: pytesseract in /usr/local/lib/python3.12/dist-packages (0.3.13)\n",
            "Requirement already satisfied: pdfminer.six==20250506 in /usr/local/lib/python3.12/dist-packages (from pdfplumber) (20250506)\n",
            "Requirement already satisfied: Pillow>=9.1 in /usr/local/lib/python3.12/dist-packages (from pdfplumber) (11.3.0)\n",
            "Requirement already satisfied: pypdfium2>=4.18.0 in /usr/local/lib/python3.12/dist-packages (from pdfplumber) (4.30.0)\n",
            "Requirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from pdfminer.six==20250506->pdfplumber) (3.4.3)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.12/dist-packages (from pdfminer.six==20250506->pdfplumber) (43.0.3)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.12/dist-packages (from pytesseract) (25.0)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.12/dist-packages (from cryptography>=36.0.0->pdfminer.six==20250506->pdfplumber) (1.17.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.12/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six==20250506->pdfplumber) (2.22)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "poppler-utils is already the newest version (22.02.0-2ubuntu0.10).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 35 not upgraded.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import cv2\n",
        "import pytesseract\n",
        "import pandas as pd\n",
        "from pdf2image import convert_from_path\n",
        "from tqdm import tqdm\n",
        "import re\n"
      ],
      "metadata": {
        "id": "OlXc31-OPKrF"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pdf_path = \"/content/Science_E_prim6_TR1.pdf\"\n",
        "output_csv = \"output_text_prime6_en.csv\"\n",
        "output_stats = \"output_stats_prime6_en.csv\"\n",
        "output_txt = \"output_text_prime6_en.txt\"\n",
        "images_dir = \"pdf_images_prime6_en\"\n",
        "batch_size = 50\n",
        "low_word_threshold = 10\n",
        "\n",
        "os.makedirs(images_dir, exist_ok=True)"
      ],
      "metadata": {
        "id": "u1W69-mMyhMG"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"📄 Converting PDF to images...\")\n",
        "pages = convert_from_path(pdf_path, dpi=300)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b682e60-e7e8-42de-f4c8-89bbb113cc45",
        "id": "C_LneoyryryP"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "📄 Converting PDF to images...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "image_paths = []\n",
        "for i, page in enumerate(pages, start=1):\n",
        "    img_path = os.path.join(images_dir, f\"page_{i}.png\")\n",
        "    page.save(img_path, \"PNG\")\n",
        "    image_paths.append(img_path)\n"
      ],
      "metadata": {
        "id": "wgX68ISuyvIn"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_text(text):\n",
        "    text = str(text)\n",
        "    text = re.sub(r\"\\s+\", \" \", text)\n",
        "    text = re.sub(r\"[^\\x00-\\x7F\\u0600-\\u06FF]+\", \" \", text)\n",
        "    return text.strip()"
      ],
      "metadata": {
        "id": "IRBX4I2bhZng"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"🔎 Running OCR on pages... (Batch mode)\")\n",
        "results = []\n",
        "\n",
        "for batch_start in range(0, len(image_paths), batch_size):\n",
        "    batch_paths = image_paths[batch_start:batch_start+batch_size]\n",
        "    for i, img_path in enumerate(tqdm(batch_paths, desc=f\"Batch {batch_start//batch_size+1}\")):\n",
        "        page_number = batch_start + i + 1\n",
        "\n",
        "        img = cv2.imread(img_path)\n",
        "\n",
        "        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
        "\n",
        "        _, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n",
        "\n",
        "        text = pytesseract.image_to_string(thresh, lang=\"eng\")\n",
        "\n",
        "        cleaned_text = clean_text(text)\n",
        "\n",
        "        results.append({\"page\": page_number, \"text\": cleaned_text})\n"
      ],
      "metadata": {
        "id": "ZAwUNKFkyz7m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3625b751-cd0e-426a-cb12-f9495058d18c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🔎 Running OCR on pages... (Batch mode)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Batch 1: 100%|██████████| 50/50 [03:36<00:00,  4.34s/it]\n",
            "Batch 2: 100%|██████████| 50/50 [03:43<00:00,  4.47s/it]\n",
            "Batch 3: 100%|██████████| 38/38 [02:54<00:00,  4.60s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"💾 Saving results...\")\n",
        "\n",
        "df = pd.DataFrame(results)\n",
        "df.to_csv(output_csv, index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "with open(output_txt, \"w\", encoding=\"utf-8\") as f:\n",
        "    for _, row in df.iterrows():\n",
        "        f.write(f\"\\n--- Page {row['page']} ---\\n{row['text']}\\n\")\n"
      ],
      "metadata": {
        "id": "wA9oIFuMy3gA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a3bbab0-f1a0-4b82-b8f8-c6d3ce765eac"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "💾 Saving results...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"word_count\"] = df[\"text\"].apply(lambda x: len(x.split()))\n",
        "stats = {\n",
        "    \"total_pages\": len(df),\n",
        "    \"total_words\": df[\"word_count\"].sum(),\n",
        "    \"avg_words_per_page\": df[\"word_count\"].mean()\n",
        "}\n",
        "stats_df = pd.DataFrame(list(stats.items()), columns=[\"metric\", \"value\"])\n",
        "stats_df.to_csv(output_stats, index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "print(\"✅ Done!\")\n",
        "print(f\"📂 Text file saved: {output_csv}\")\n",
        "print(f\"📂 Clean text file saved: {output_txt}\")\n",
        "print(f\"📊 Stats file saved: {output_stats}\")"
      ],
      "metadata": {
        "id": "CPYgXgLMy6Xh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ddd6ce3b-e481-4c9d-f33a-f485ebbd7d27"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✅ Done!\n",
            "📂 Text file saved: output_text_prime6_en.csv\n",
            "📂 Clean text file saved: output_text_prime6_en.txt\n",
            "📊 Stats file saved: output_stats_prime6_en.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "def super_clean_text_en(text):\n",
        "    text = str(text)\n",
        "\n",
        "    text = re.sub(r\"[^a-zA-Z0-9\\s\\.\\,\\?\\!\\:\\;\\-\\(\\)\\/%]\", \" \", text)\n",
        "\n",
        "    text = re.sub(r\"\\b(\\w+)( \\1){2,}\\b\", r\"\\1\", text)\n",
        "\n",
        "    text = re.sub(r\"(.)\\1{2,}\", r\"\\1\\1\", text)\n",
        "\n",
        "    text = re.sub(r\"(\\d)\\1{2,}\", r\"\\1\\1\\1\", text)\n",
        "\n",
        "    words = text.split()\n",
        "    cleaned_words = []\n",
        "    for w in words:\n",
        "        if len(w) >= 15:\n",
        "            continue\n",
        "        cleaned_words.append(w)\n",
        "    text = \" \".join(cleaned_words)\n",
        "\n",
        "    text = re.sub(r\"\\s+\", \" \", text)\n",
        "\n",
        "    lines = text.split(\".\")\n",
        "    seen = set()\n",
        "    cleaned_lines = []\n",
        "    for line in lines:\n",
        "        l = line.strip()\n",
        "        if l and l not in seen:\n",
        "            cleaned_lines.append(l)\n",
        "            seen.add(l)\n",
        "    text = \". \".join(cleaned_lines)\n",
        "\n",
        "    return text.strip()\n",
        "\n",
        "df = pd.read_csv(\"/content/output_text_prime6_en.csv\")\n",
        "\n",
        "df[\"text\"] = df[\"text\"].apply(super_clean_text_en)\n",
        "\n",
        "df.to_csv(\"output_text_en_cleaned.csv\", index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "print(\"✅ النصوص اتنضفت وحُفظت في output_text_en_cleaned_prime5.csv\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "37b3fbf7-f5dc-4d71-b09a-c4c5326a4321",
        "id": "kko7q5b7VHP5"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✅ النصوص اتنضفت وحُفظت في output_text_en_cleaned_prime5.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# ---------------- structure ----------------\n",
        "structure = {\n",
        "    \"Unit 1: What is system\": {\n",
        "        \"Get Started\": (9, 12),\n",
        "        \"Concepts\": {\n",
        "            \"Concept 1.1 The cell as a system\": {\n",
        "                \"Lesson 1\": (13, 17),\n",
        "                \"Lesson 2\": (18, 20),\n",
        "                \"Lesson 3\": (21, 23),\n",
        "                \"Lesson 4\": (24, 27),\n",
        "                \"Lesson 5\": (28, 30),\n",
        "                \"Lesson 6\": (31, 34),\n",
        "            },\n",
        "            \"Concept 1.2 The body as a system\": {\n",
        "                \"Lesson 1\": (35, 38),\n",
        "                \"Lesson 2\": (39, 41),\n",
        "                \"Lesson 3\": (42, 45),\n",
        "                \"Lesson 4\": (46, 48),\n",
        "                \"Lesson 5\": (49, 51),\n",
        "                \"Lesson 6\": (52, 54),\n",
        "            },\n",
        "            \"Concept 1.3 Energy as a system\": {\n",
        "                \"Lesson 1\": (55, 57),\n",
        "                \"Lesson 2\": (58, 60),\n",
        "                \"Lesson 3\": (61, 65),\n",
        "                \"Lesson 4\": (66, 67),\n",
        "                \"Lesson 5\": (68, 71),\n",
        "                \"Lesson 6\": (72, 73),\n",
        "            }\n",
        "        },\n",
        "        \"Unit Project\": (74, 76),\n",
        "        \"Unit assessment\": (77, 80),\n",
        "    },\n",
        "    \"Unit 2: Getting energy\": {\n",
        "        \"Get Started\": (81, 84),\n",
        "        \"Concepts\": {\n",
        "            \"Concept 2.1 Thermal energy and states of matter\": {\n",
        "                \"Lesson 1\": (85, 88),\n",
        "                \"Lesson 2\": (89, 91),\n",
        "                \"Lesson 3\": (92, 93),\n",
        "                \"Lesson 4\": (94, 95),\n",
        "                \"Lesson 5\": (96, 98),\n",
        "                \"Lesson 6\": (99, 100),\n",
        "            },\n",
        "            \"Concept 2.2 Heat transfer\": {\n",
        "                \"Lesson 1\": (101, 104),\n",
        "                \"Lesson 2\": (105, 108),\n",
        "                \"Lesson 3\": (109, 110),\n",
        "                \"Lesson 4\": (111, 113),\n",
        "                \"Lesson 5\": (114, 116),\n",
        "                \"Lesson 6\": (117, 120),\n",
        "            },\n",
        "        },\n",
        "        \"Unit Project\": (121, 122),\n",
        "        \"Interdisciplinary project\": (123, 131),\n",
        "        \"Unit assessment\": (132, 135)   # توحيد الاسم\n",
        "    },\n",
        "}\n",
        "\n",
        "# ---------------- load OCR text ----------------\n",
        "df = pd.read_csv(\"/content/output_text_en_cleaned.csv\")\n",
        "df_raw = pd.read_csv(\"/content/output_text_prime6_en.csv\")\n",
        "df[\"page\"] = df_raw[\"page\"]\n",
        "\n",
        "rows = []\n",
        "\n",
        "# ---------------- split text ----------------\n",
        "for unit, udata in structure.items():\n",
        "    # Get Started\n",
        "    if \"Get Started\" in udata:\n",
        "        start, end = udata[\"Get Started\"]\n",
        "        text = \" \".join(df[(df[\"page\"] >= start) & (df[\"page\"] <= end)][\"text\"])\n",
        "        rows.append({\n",
        "            \"Unit\": unit,\n",
        "            \"Concept\": \"Get Started\",\n",
        "            \"Lesson\": \"Get Started\",\n",
        "            \"From page\": start,\n",
        "            \"To page\": end,\n",
        "            \"Lesson text\": text\n",
        "        })\n",
        "\n",
        "    # Concepts & Lessons\n",
        "    if \"Concepts\" in udata:\n",
        "        for concept, lessons in udata[\"Concepts\"].items():\n",
        "            for lesson, (ls, le) in lessons.items():\n",
        "                lesson_text = \" \".join(df[(df[\"page\"] >= ls) & (df[\"page\"] <= le)][\"text\"])\n",
        "                rows.append({\n",
        "                    \"Unit\": unit,\n",
        "                    \"Concept\": concept,\n",
        "                    \"Lesson\": lesson,\n",
        "                    \"From page\": ls,\n",
        "                    \"To page\": le,\n",
        "                    \"Lesson text\": lesson_text\n",
        "                })\n",
        "\n",
        "    # Other sections\n",
        "    for section in [\"Unit Project\", \"Unit assessment\", \"Interdisciplinary project\"]:\n",
        "        if section in udata:\n",
        "            start, end = udata[section]\n",
        "            text = \" \".join(df[(df[\"page\"] >= start) & (df[\"page\"] <= end)][\"text\"])\n",
        "            rows.append({\n",
        "                \"Unit\": unit,\n",
        "                \"Concept\": section,\n",
        "                \"Lesson\": section,\n",
        "                \"From page\": start,\n",
        "                \"To page\": end,\n",
        "                \"Lesson text\": text\n",
        "            })\n",
        "\n",
        "# ---------------- save output ----------------\n",
        "df_out = pd.DataFrame(rows)\n",
        "df_out.to_csv(\"output_units_lessons_prime6_EN.csv\", index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "print(\"✅ Done! Lessons & units split and saved to output_units_lessons_prime6_EN.csv\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pICh8K7rnZyd",
        "outputId": "22525901-3cc3-4103-b889-e9c4410fd46d"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✅ Done! Lessons & units split and saved to output_units_lessons_prime6_EN.csv\n"
          ]
        }
      ]
    }
  ]
}