changes for deploment

self_hosted_env/data-handler/Dockerfile

@@ -8,4 +8,8 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 #just keep the container running without doing anything
-CMD ["sh", "-c", "while :; do sleep 10; done"]
+#CMD ["sh", "-c", "while :; do sleep 10; done"]
+
+# Run apply_test_schema.py, then insert_csv_embeddings.py
+CMD ["sh", "-c", "python apply_test_schema.py && python insert_csv_embeddings.py"]
+

self_hosted_env/data-handler/apply_test_schema.py

@@ -48,22 +48,35 @@ BEGIN
 END $$;
 """
 
-conn = psycopg2.connect(
+def setup_database(drop_existing_tables: bool = False):
+    """
+    Sets up the database schema and tables.
+    Args:
+        drop_existing_tables: If True, drops all existing tables before creating them.
+    """
+    try:
+        conn = psycopg2.connect(
             host=os.getenv("POSTGRES_HOST", "localhost"),
             port=os.getenv("POSTGRES_PORT", "5432"),
             user=os.getenv("POSTGRES_USER"),
             password=os.getenv("POSTGRES_PASSWORD"),
             dbname=os.getenv("POSTGRES_DB")
-)
-conn.autocommit = True
+        )
+        conn.autocommit = True
         
-with conn.cursor() as cur:
-    # Drop all existing tables (uncomment if needed)
+        with conn.cursor() as cur:
+            if drop_existing_tables:
+                print("Dropping all existing tables...")
                 cur.execute(drop_all_tables_sql)
+                print("All tables dropped.")
+            
+            print("Setting up schema and inserting data...")
             cur.execute(schema_sql)
             
+            print("Database setup complete. Verifying data...")
+            
             # Verifications: Select from students and chat_history tables
-    print("Students table rows:")
+            print("\nStudents table rows:")
             cur.execute("SELECT * FROM students ORDER BY id;")
             students = cur.fetchall()
             for row in students:
@@ -75,4 +88,19 @@ with conn.cursor() as cur:
             for row in chat_history:
                 print(row)
                 
-conn.close()
+    except psycopg2.OperationalError as e:
+        print(f"Database connection failed: {e}")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+    finally:
+        if 'conn' in locals() and conn:
+            conn.close()
+            print("Database connection closed.")
+
+
+if __name__ == "__main__":
+    # To run with a clean slate, pass True
+    # setup_database(drop_existing_tables=True)
+    
+    # To run without dropping tables (default)
+    setup_database()
\ No newline at end of file

self_hosted_env/data-handler/insert_csv_embeddings.py

@@ -21,9 +21,8 @@ def get_db_connection():
     register_vector(conn)
     return conn
 
-def create_schema_and_table():
+def create_schema_and_table(conn, drop_existing_table: bool):
     create_extension = "CREATE EXTENSION IF NOT EXISTS vector;"
-    drop_table = "DROP TABLE IF EXISTS educational_chunks;"
     create_table = """
     CREATE TABLE IF NOT EXISTS educational_chunks (
       id SERIAL PRIMARY KEY,
@@ -48,12 +47,15 @@ def create_schema_and_table():
         "CREATE INDEX IF NOT EXISTS idx_grade_is_arabic ON educational_chunks (grade, is_arabic);"
     ]
 
-    conn = get_db_connection()
     cur = conn.cursor()
     cur.execute(create_extension)
     print("CREATE EXTENSION vector operation fine.")
+    
+    if drop_existing_table:
+        drop_table = "DROP TABLE IF EXISTS educational_chunks;"
         cur.execute(drop_table)
         print("DROP TABLE educational_chunks operation fine.")
+        
     cur.execute(create_table)
     print("CREATE TABLE educational_chunks operation fine.")
     for idx_query in create_indexes:
@@ -61,7 +63,7 @@ def create_schema_and_table():
         print(f"CREATE INDEX operation fine for: {idx_query}")
     conn.commit()
     cur.close()
-    conn.close()
+
 
 def insert_chunks_from_csv(csv_file: str):
     df = pd.read_csv(csv_file)
@@ -125,8 +127,16 @@ def insert_chunks_from_csv(csv_file: str):
     conn.close()
     print("All data inserted successfully.")
 
-if __name__ == "__main__":
-    create_schema_and_table()
+def setup_embeddings_database(drop_existing_tables: bool = False):
+    """
+    Sets up the educational chunks table and populates it with embeddings from CSV files.
+    Args:
+        drop_existing_tables: If True, drops the existing table before creating it.
+    """
+    try:
+        conn = get_db_connection()
+        create_schema_and_table(conn, drop_existing_tables)
+        
         csv_files = ["prime4_ar_embeddings.csv", "Prime5_en_chunked_with_embeddings.csv", "prime6_ar_embeddings.csv", "Prime6_en_chunked_with_embeddings.csv"]
         for file in csv_files:
             if os.path.exists(file):
@@ -134,3 +144,19 @@ if __name__ == "__main__":
                 insert_chunks_from_csv(file)
             else:
                 print(f"File not found: {file}")
+
+    except psycopg2.OperationalError as e:
+        print(f"Database connection failed: {e}")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+    finally:
+        if 'conn' in locals() and conn:
+            conn.close()
+            print("Database connection closed.")
+
+if __name__ == "__main__":
+    # To run with a clean slate, pass True
+    # setup_embeddings_database(drop_existing_tables=True)
+    
+    # To run without dropping the table (default)
+    setup_embeddings_database()
\ No newline at end of file

self_hosted_env/docker-compose.yml

@@ -35,10 +35,8 @@ services:
       timeout: 20s
       retries: 3
 
-  voice-agent:
-    image: salmamohammedhamedmustafa/voice-agent:latest
-    ports:
-      - "8000:8000"
+  data-handler:
+    image: salmamohammedhamedmustafa/data-handler:latest
     environment:
       MINIO_ENDPOINT: "http://minio:9000"
       MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
@@ -49,13 +47,19 @@ services:
       POSTGRES_USER: "${POSTGRES_USER}"
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
       POSTGRES_DB: "${POSTGRES_DB}"
-    volumes:
-      - ./uploads:/app/uploads
     depends_on:
       - minio
+      - postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      interval: 10s
+      retries: 3
+      timeout: 5s
 
-  data-handler:
-    image: salmamohammedhamedmustafa/data-handler:latest
+  voice-agent:
+    image: salmamohammedhamedmustafa/voice-agent:latest
+    ports:
+      - "8000:8000"
     environment:
       MINIO_ENDPOINT: "http://minio:9000"
       MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
@@ -66,9 +70,11 @@ services:
       POSTGRES_USER: "${POSTGRES_USER}"
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
       POSTGRES_DB: "${POSTGRES_DB}"
+    volumes:
+      - ./uploads:/app/uploads
     depends_on:
       - minio
-      - postgres
+      - data-handler
 
 volumes:
   pgdata:

self_hosted_env/voice_agent/Dockerfile

@@ -8,7 +8,7 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 #just keep the container running without doing anything
-CMD ["sh", "-c", "while :; do sleep 10; done"]
+#CMD ["sh", "-c", "while :; do sleep 10; done"]
 
 #run the app automatically when the container starts
-#CMD ["python", "main.py"]
+CMD ["python", "main.py"]

self_hosted_env/voice_agent/static/audio-recorder.html

@@ -3,7 +3,7 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>الدردشة</title>
+    <title>SSLabs AI Testing</title>
     <style>
         body {
             font-family: 'Arial', sans-serif;
@@ -161,7 +161,7 @@
 </head>
 <body>
     <div class="container">
-        <h1>الدردشة</h1>
+        <h1>SSLabs AI Testing</h1>
 
         <div class="controls">
             <div class="student-id-controls">