{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ATlQdSWeYEwu"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install tesseract-ocr\n",
        "!pip install pdfplumber pdf2image pytesseract\n",
        "!apt-get install -y poppler-utils\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mXFZpTwLOjTZ",
        "outputId": "181e3c60-149b-4582-e882-17f6fbf16113"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "tesseract-ocr is already the newest version (4.1.1-2.1build1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 35 not upgraded.\n",
            "Collecting pdfplumber\n",
            "  Downloading pdfplumber-0.11.7-py3-none-any.whl.metadata (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.8/42.8 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pdf2image\n",
            "  Downloading pdf2image-1.17.0-py3-none-any.whl.metadata (6.2 kB)\n",
            "Collecting pytesseract\n",
            "  Downloading pytesseract-0.3.13-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting pdfminer.six==20250506 (from pdfplumber)\n",
            "  Downloading pdfminer_six-20250506-py3-none-any.whl.metadata (4.2 kB)\n",
            "Requirement already satisfied: Pillow>=9.1 in /usr/local/lib/python3.12/dist-packages (from pdfplumber) (11.3.0)\n",
            "Collecting pypdfium2>=4.18.0 (from pdfplumber)\n",
            "  Downloading pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (48 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.5/48.5 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from pdfminer.six==20250506->pdfplumber) (3.4.3)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.12/dist-packages (from pdfminer.six==20250506->pdfplumber) (43.0.3)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.12/dist-packages (from pytesseract) (25.0)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.12/dist-packages (from cryptography>=36.0.0->pdfminer.six==20250506->pdfplumber) (1.17.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.12/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six==20250506->pdfplumber) (2.22)\n",
            "Downloading pdfplumber-0.11.7-py3-none-any.whl (60 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pdfminer_six-20250506-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m79.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pdf2image-1.17.0-py3-none-any.whl (11 kB)\n",
            "Downloading pytesseract-0.3.13-py3-none-any.whl (14 kB)\n",
            "Downloading pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.8 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m101.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pytesseract, pypdfium2, pdf2image, pdfminer.six, pdfplumber\n",
            "Successfully installed pdf2image-1.17.0 pdfminer.six-20250506 pdfplumber-0.11.7 pypdfium2-4.30.0 pytesseract-0.3.13\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  poppler-utils\n",
            "0 upgraded, 1 newly installed, 0 to remove and 35 not upgraded.\n",
            "Need to get 186 kB of archives.\n",
            "After this operation, 697 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 poppler-utils amd64 22.02.0-2ubuntu0.10 [186 kB]\n",
            "Fetched 186 kB in 1s (196 kB/s)\n",
            "Selecting previously unselected package poppler-utils.\n",
            "(Reading database ... 126374 files and directories currently installed.)\n",
            "Preparing to unpack .../poppler-utils_22.02.0-2ubuntu0.10_amd64.deb ...\n",
            "Unpacking poppler-utils (22.02.0-2ubuntu0.10) ...\n",
            "Setting up poppler-utils (22.02.0-2ubuntu0.10) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import cv2\n",
        "import pytesseract\n",
        "import pandas as pd\n",
        "from pdf2image import convert_from_path\n",
        "from tqdm import tqdm\n",
        "import re\n"
      ],
      "metadata": {
        "id": "OlXc31-OPKrF"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pdf_path = \"/content/Science_EN_prim5_TR1 (1).pdf\"\n",
        "output_csv = \"output_text.csv\"\n",
        "output_stats = \"output_stats.csv\"\n",
        "output_txt = \"output_text.txt\"\n",
        "images_dir = \"pdf_images\"\n",
        "batch_size = 50\n",
        "low_word_threshold = 10\n",
        "os.makedirs(images_dir, exist_ok=True)"
      ],
      "metadata": {
        "id": "u1W69-mMyhMG"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"📄 Converting PDF to images...\")\n",
        "pages = convert_from_path(pdf_path, dpi=300)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8e9b878-0c10-4a6b-ea99-92fc7caa1d99",
        "id": "C_LneoyryryP"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "📄 Converting PDF to images...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "image_paths = []\n",
        "for i, page in enumerate(pages, start=1):\n",
        "    img_path = os.path.join(images_dir, f\"page_{i}.png\")\n",
        "    page.save(img_path, \"PNG\")\n",
        "    image_paths.append(img_path)\n"
      ],
      "metadata": {
        "id": "wgX68ISuyvIn"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_text(text):\n",
        "    text = str(text)\n",
        "    text = re.sub(r\"\\s+\", \" \", text)\n",
        "    text = re.sub(r\"[^\\x00-\\x7F\\u0600-\\u06FF]+\", \" \", text)\n",
        "    return text.strip()"
      ],
      "metadata": {
        "id": "IRBX4I2bhZng"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"🔎 Running OCR on pages... (Batch mode)\")\n",
        "results = []\n",
        "\n",
        "for batch_start in range(0, len(image_paths), batch_size):\n",
        "    batch_paths = image_paths[batch_start:batch_start+batch_size]\n",
        "    for i, img_path in enumerate(tqdm(batch_paths, desc=f\"Batch {batch_start//batch_size+1}\")):\n",
        "        page_number = batch_start + i + 1\n",
        "\n",
        "        img = cv2.imread(img_path)\n",
        "\n",
        "        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
        "\n",
        "        _, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n",
        "\n",
        "        text = pytesseract.image_to_string(thresh, lang=\"eng\")\n",
        "\n",
        "        cleaned_text = clean_text(text)\n",
        "\n",
        "        results.append({\"page\": page_number, \"text\": cleaned_text})\n"
      ],
      "metadata": {
        "id": "ZAwUNKFkyz7m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80adc825-c300-400f-9a8b-c52dc1f5c21b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🔎 Running OCR on pages... (Batch mode)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Batch 1: 100%|██████████| 50/50 [02:54<00:00,  3.50s/it]\n",
            "Batch 2: 100%|██████████| 50/50 [02:51<00:00,  3.44s/it]\n",
            "Batch 3: 100%|██████████| 29/29 [01:52<00:00,  3.89s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"💾 Saving results...\")\n",
        "\n",
        "df = pd.DataFrame(results)\n",
        "df.to_csv(output_csv, index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "with open(output_txt, \"w\", encoding=\"utf-8\") as f:\n",
        "    for _, row in df.iterrows():\n",
        "        f.write(f\"\\n--- Page {row['page']} ---\\n{row['text']}\\n\")\n"
      ],
      "metadata": {
        "id": "wA9oIFuMy3gA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "397832ee-5722-43aa-d66d-b85e5f1768c2"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "💾 Saving results...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"word_count\"] = df[\"text\"].apply(lambda x: len(x.split()))\n",
        "stats = {\n",
        "    \"total_pages\": len(df),\n",
        "    \"total_words\": df[\"word_count\"].sum(),\n",
        "    \"avg_words_per_page\": df[\"word_count\"].mean()\n",
        "}\n",
        "stats_df = pd.DataFrame(list(stats.items()), columns=[\"metric\", \"value\"])\n",
        "stats_df.to_csv(output_stats, index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "print(\"✅ Done!\")\n",
        "print(f\"📂 Text file saved: {output_csv}\")\n",
        "print(f\"📂 Clean text file saved: {output_txt}\")\n",
        "print(f\"📊 Stats file saved: {output_stats}\")"
      ],
      "metadata": {
        "id": "CPYgXgLMy6Xh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "785517e6-145c-4ead-a012-353340643741"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✅ Done!\n",
            "📂 Text file saved: output_text.csv\n",
            "📂 Clean text file saved: output_text.txt\n",
            "📊 Stats file saved: output_stats.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "def super_clean_text_en(text):\n",
        "    text = str(text)\n",
        "\n",
        "    text = re.sub(r\"[^a-zA-Z0-9\\s\\.\\,\\?\\!\\:\\;\\-\\(\\)\\/%]\", \" \", text)\n",
        "\n",
        "    text = re.sub(r\"\\b(\\w+)( \\1){2,}\\b\", r\"\\1\", text)\n",
        "\n",
        "    text = re.sub(r\"(.)\\1{2,}\", r\"\\1\\1\", text)\n",
        "\n",
        "    text = re.sub(r\"(\\d)\\1{2,}\", r\"\\1\\1\\1\", text)\n",
        "\n",
        "    words = text.split()\n",
        "    cleaned_words = []\n",
        "    for w in words:\n",
        "        if len(w) >= 15:\n",
        "            continue\n",
        "        cleaned_words.append(w)\n",
        "    text = \" \".join(cleaned_words)\n",
        "\n",
        "    text = re.sub(r\"\\s+\", \" \", text)\n",
        "\n",
        "    lines = text.split(\".\")\n",
        "    seen = set()\n",
        "    cleaned_lines = []\n",
        "    for line in lines:\n",
        "        l = line.strip()\n",
        "        if l and l not in seen:\n",
        "            cleaned_lines.append(l)\n",
        "            seen.add(l)\n",
        "    text = \". \".join(cleaned_lines)\n",
        "\n",
        "    return text.strip()\n",
        "\n",
        "df = pd.read_csv(\"/content/output_text_prime5_en.csv\")\n",
        "\n",
        "df[\"text\"] = df[\"text\"].apply(super_clean_text_en)\n",
        "\n",
        "\n",
        "df.to_csv(\"output_text_en_cleaned.csv\", index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "print(\"✅ النصوص اتنضفت وحُفظت في output_text_en_cleaned_prime5.csv\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9f2f030a-f8ab-448f-fa95-b4b18a017fa1",
        "id": "kko7q5b7VHP5"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✅ النصوص اتنضفت وحُفظت في output_text_en_cleaned_prime5.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# ---------------- structure ----------------\n",
        "structure = {\n",
        "    \"Unit 1: Interactions of Organisms\": {\n",
        "        \"Get Started\": (10, 13),\n",
        "        \"Concepts\": {\n",
        "            \"Concept 1.1 Plant Needs\": {\n",
        "                \"Lesson 1\": (14, 17),\n",
        "                \"Lesson 2\": (18, 21),\n",
        "                \"Lesson 3\": (22, 25),\n",
        "                \"Lesson 4\": (26, 29),\n",
        "                \"Lesson 5\": (30, 33),\n",
        "            },\n",
        "            \"Concept 1.2 Energy Flow in Ecosystems\": {\n",
        "                \"Lesson 1\": (34, 37),\n",
        "                \"Lesson 2\": (38, 40),\n",
        "                \"Lesson 3\": (41, 43),\n",
        "                \"Lesson 4\": (44, 45),\n",
        "            },\n",
        "            \"Concept 1.3 Changes in Food Webs\": {\n",
        "                \"Lesson 1\": (46, 50),\n",
        "                \"Lesson 2\": (51, 53),\n",
        "                \"Lesson 3\": (54, 55),\n",
        "                \"Lesson 4\": (56, 58),\n",
        "            }\n",
        "        },\n",
        "        \"Unit Project\": (59, 59),\n",
        "        \"Interdisciplinary project\": (60, 67),\n",
        "        \"Assess your learning\": (68, 69)\n",
        "    },\n",
        "    \"Unit 2: Particles in Motion\": {\n",
        "        \"Get Started\": (70, 73),\n",
        "        \"Concepts\": {\n",
        "            \"Concept 2.1 Matter in the World around Us\": {\n",
        "                \"Lesson 1\": (74, 76),\n",
        "                \"Lesson 2\": (77, 80),\n",
        "                \"Lesson 3\": (81, 83),\n",
        "                \"Lesson 4\": (84, 86),\n",
        "                \"Lesson 5\": (87, 89),\n",
        "            },\n",
        "            \"Concept 2.2 Describing and Measuring Matter\": {\n",
        "                \"Lesson 1\": (90, 93),\n",
        "                \"Lesson 2\": (94, 95),\n",
        "                \"Lesson 3\": (96, 100),\n",
        "                \"Lesson 4\": (101, 103),\n",
        "            },\n",
        "            \"Concept 2.3 Comparing Changes in Matter\": {\n",
        "                \"Lesson 1\": (104, 107),\n",
        "                \"Lesson 2\": (108, 111),\n",
        "                \"Lesson 3\": (112, 116),\n",
        "                \"Lesson 4\": (117, 120),\n",
        "                \"Lesson 5\": (121, 123),\n",
        "            }\n",
        "        },\n",
        "        \"Unit Project\": (124, 125),\n",
        "        \"Assess your learning\": (126, 127)\n",
        "    },\n",
        "}\n",
        "\n",
        "# ---------------- load OCR text ----------------\n",
        "df = pd.read_csv(\"/content/output_text_en_cleaned.csv\")\n",
        "df_raw = pd.read_csv(\"/content/output_text_prime5_en.csv\")\n",
        "df[\"page\"] = df_raw[\"page\"]\n",
        "\n",
        "rows = []\n",
        "\n",
        "# ---------------- split text ----------------\n",
        "for unit, udata in structure.items():\n",
        "    # Get Started\n",
        "    if \"Get Started\" in udata:\n",
        "        start, end = udata[\"Get Started\"]\n",
        "        text = \" \".join(df[(df[\"page\"] >= start) & (df[\"page\"] <= end)][\"text\"])\n",
        "        rows.append({\n",
        "            \"Unit\": unit,\n",
        "            \"Concept\": \"Get Started\",\n",
        "            \"Lesson\": \"Get Started\",\n",
        "            \"From page\": start,\n",
        "            \"To page\": end,\n",
        "            \"Lesson text\": text\n",
        "        })\n",
        "\n",
        "    # Concepts & Lessons\n",
        "    if \"Concepts\" in udata:\n",
        "        for concept, lessons in udata[\"Concepts\"].items():\n",
        "            for lesson, (ls, le) in lessons.items():\n",
        "                lesson_text = \" \".join(df[(df[\"page\"] >= ls) & (df[\"page\"] <= le)][\"text\"])\n",
        "                rows.append({\n",
        "                    \"Unit\": unit,\n",
        "                    \"Concept\": concept,\n",
        "                    \"Lesson\": lesson,\n",
        "                    \"From page\": ls,\n",
        "                    \"To page\": le,\n",
        "                    \"Lesson text\": lesson_text\n",
        "                })\n",
        "\n",
        "    # Other sections\n",
        "    for section in [\"Unit Project\", \"Assess your learning\", \"Interdisciplinary project\"]:\n",
        "        if section in udata:\n",
        "            start, end = udata[section]\n",
        "            text = \" \".join(df[(df[\"page\"] >= start) & (df[\"page\"] <= end)][\"text\"])\n",
        "            rows.append({\n",
        "                \"Unit\": unit,\n",
        "                \"Concept\": section,\n",
        "                \"Lesson\": section,\n",
        "                \"From page\": start,\n",
        "                \"To page\": end,\n",
        "                \"Lesson text\": text\n",
        "            })\n",
        "\n",
        "# ---------------- save output ----------------\n",
        "df_out = pd.DataFrame(rows)\n",
        "df_out.to_csv(\"output_units_lessons_prime5_EN.csv\", index=False, encoding=\"utf-8-sig\")\n",
        "\n",
        "print(\"✅ Done! Lessons & units split and saved to output_units_lessons_prime5_EN.csv\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pICh8K7rnZyd",
        "outputId": "cc0d291d-e292-494c-fc15-310f5bb047e7"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✅ Done! Lessons & units split and saved to output_units_lessons_prime5_EN.csv\n"
          ]
        }
      ]
    }
  ]
}