#!/usr/bin/env python3 """ Generate FULL enriched dataset for Gemma-4 training Includes ALL 28 question types per shloka Target: ~2GB, ~300k+ conversations """ import json import re import random from typing import Dict, List, Any, Optional from dataclasses import dataclass from pathlib import Path # Full section mappings SECTIONS = { "SU": "Sutrasthana (सूत्रस्थान)", "NI": "Nidanasthana (निदानस्थान)", "VI": "Vimanasthana (विमानस्थान)", "SHA": "Sharirasthana (शारीरस्थान)", "IND": "Indriyasthana (इन्द्रियस्थान)", "CI": "Cikitsasthana (चिकित्सास्थान)", "CHI": "Cikitsasthana (चिकित्सास्थान)", "KA": "Kalpasthana (कल्पस्थान)", "KAL": "Kalpasthana (कल्पस्थान)", "SI": "Siddhisthana (सिद्धिस्थान)", "SID": "Siddhisthana (सिद्धिस्थान)", "IN": "Indriyasthana (इन्द्रियस्थान)" } SECTION_DESCRIPTIONS = { "SU": "General Principles and Fundamentals", "NI": "Pathology and Disease Causation", "VI": "Quantitative and Measurement Science", "SHA": "Human Body and Anatomy", "IND": "Sensory Organs and Clinical Signs", "CI": "Treatment and Therapeutics", "CHI": "Treatment and Therapeutics", "KA": "Pharmaceutical Preparations", "KAL": "Pharmaceutical Preparations", "SI": "Attainment of Perfection and Results", "SID": "Attainment of Perfection and Results", "IN": "Sensory Organs and Clinical Signs" } SECTIONS_HI = { "SU": "सूत्रस्थान", "NI": "निदानस्थान", "VI": "विमानस्थान", "SHA": "शारीरस्थान", "IND": "इन्द्रियस्थान", "CI": "चिकित्सास्थान", "CHI": "चिकित्सास्थान", "KA": "कल्पस्थान", "KAL": "कल्पस्थान", "SI": "सिद्धिस्थान", "SID": "सिद्धिस्थान", "IN": "इन्द्रियस्थान" } SECTIONS_SA = { "SU": "सूत्रस्थानम्", "NI": "निदानस्थानम्", "VI": "विमानस्थानम्", "SHA": "शारीरस्थानम्", "IND": "इन्द्रियस्थानम्", "CI": "चिकित्सास्थानम्", "CHI": "चिकित्सास्थानम्", "KA": "कल्पस्थानम्", "KAL": "कल्पस्थानम्", "SI": "सिद्धिस्थानम्", "SID": "सिद्धिस्थानम्", "IN": "इन्द्रियस्थानम्" } @dataclass class ShlokaData: id: int shloka_id: str sanskrit: str transliteration: str translation_en: str translation_hi: str section_code: str section_name: str section_name_hi: str section_name_sa: str section_description: str chapter: int verse: int class ComprehensiveDatasetGenerator: def __init__(self, source_file: str): self.source_file = source_file self.shlokas: List[ShlokaData] = [] self.section_questions: Dict[str, List[Dict]] = self._load_section_questions() def _load_section_questions(self) -> Dict[str, List[Dict]]: """Load or generate section-specific question templates""" return { "SU": self._get_sutrasthana_questions(), "NI": self._get_nidanasthana_questions(), "VI": self._get_vimanasthana_questions(), "SHA": self._get_sharirasthana_questions(), "IND": self._get_indriyasthana_questions(), "CI": self._get_cikitsasthana_questions(), "CHI": self._get_cikitsasthana_questions(), "KA": self._get_kalpasthana_questions(), "KAL": self._get_kalpasthana_questions(), "SI": self._get_siddhisthana_questions(), "SID": self._get_siddhisthana_questions(), "IN": self._get_indriyasthana_questions() } def _get_sutrasthana_questions(self): return { "topics": ["longevity", "fundamental principles", "daily regimen", "preventive medicine", "seasonal routines"], "concepts": ["svasthavṛtta", "dīnācārya", "ṛtucaryā", "sādhitasādhanīya"], "applications": ["lifestyle medicine", "preventive health", "wellness practices"] } def _get_nidanasthana_questions(self): return { "topics": ["disease causation", "pathogenesis", "diagnostic criteria", "prognostic factors"], "concepts": ["samprāpti", "hetu", "liṅga", "āśaya", "doṣa-doṣya-sammūrcchanā"], "applications": ["clinical diagnosis", "differential diagnosis", "prognosis assessment"] } def _get_vimanasthana_questions(self): return { "topics": ["measurement", "quantitative assessment", "dietetic calculations", "therapeutic quantities"], "concepts": ["nimitta", "aupadhikī", "tulyatva", "pramāṇa"], "applications": ["dosage calculation", "diet planning", "treatment measurement"] } def _get_sharirasthana_questions(self): return { "topics": ["anatomy", "embryology", "physiology", "body constituents"], "concepts": ["garbha", "dhātu", "srotas", "marma", "doṣa-dhātu-mala"], "applications": ["surgical anatomy", "reproductive health", "constitutional assessment"] } def _get_indriyasthana_questions(self): return { "topics": ["sensory organs", "clinical signs", "prognostic indicators", "terminal signs"], "concepts": ["indriya", "artha", "jñāna", "ārṣa", "asādhya-liṅga"], "applications": ["prognosis", "clinical observation", "end-of-life care"] } def _get_cikitsasthana_questions(self): return { "topics": ["fever treatment", "gastrointestinal disorders", "urinary conditions", "mental health"], "concepts": ["cikitsā", "bheṣaja", "pathya", "apathy", "samsarjana"], "applications": ["therapeutics", "clinical management", "drug therapy"] } def _get_kalpasthana_questions(self): return { "topics": ["pharmaceuticals", "rasāyana", "vājīkaraṇa", "detoxification"], "concepts": ["aṣṭāpana", "anuvāsana", "rasāyana", "virecana"], "applications": ["rejuvenation therapy", "formulation", "purification procedures"] } def _get_siddhisthana_questions(self): return { "topics": ["therapeutic outcomes", "treatment completion", "clinical success", "complications"], "concepts": ["siddhi", "prasava", "vyāpatsiddhi", "bheda"], "applications": ["treatment evaluation", "outcome prediction", "clinical mastery"] } def load_source(self): """Parse source JSON and create sorted list""" with open(self.source_file, 'r', encoding='utf-8') as f: data = json.load(f) for item in data: match = re.match(r'CH_([A-Z]+)_([\d]+)/([\d]+)', item['shloka_id']) if not match: continue section_code, chapter, verse = match.groups() shloka = ShlokaData( id=item['id'], shloka_id=item['shloka_id'], sanskrit=item['sanskrit_shloka'], transliteration=item['transliteration'], translation_en=item.get('translations', {}).get('english', ''), translation_hi=item.get('translations', {}).get('hindi', ''), section_code=section_code, section_name=SECTIONS.get(section_code, section_code), section_name_hi=SECTIONS_HI.get(section_code, section_code), section_name_sa=SECTIONS_SA.get(section_code, section_code), section_description=SECTION_DESCRIPTIONS.get(section_code, ''), chapter=int(chapter), verse=int(verse) ) self.shlokas.append(shloka) # Sort by shloka_id def sort_key(s): match = re.match(r'CH_([A-Z]+)_(\d+)/(\d+)', s.shloka_id) if match: sec, chap, ver = match.groups() # Order sections: SU, NI, VI, SHA, IND, CI, KA, SI sec_order = {'SU': 0, 'NI': 1, 'VI': 2, 'SHA': 3, 'IND': 4, 'IN': 4, 'CI': 5, 'CHI': 5, 'KA': 6, 'KAL': 6, 'SI': 7, 'SID': 7} return (sec_order.get(sec, 99), int(chap), int(ver)) return (99, 99, 99) self.shlokas.sort(key=sort_key) print(f"Loaded {len(self.shlokas)} shlokas") def _extract_speaker(self, shloka: ShlokaData) -> str: """Extract speaker from Sanskrit text""" text = shloka.sanskrit.lower() if 'ātreya' in text or 'atreya' in shloka.transliteration.lower(): return "Ātreya" elif 'agniveśa' in text or 'agnivesa' in shloka.transliteration.lower(): return "Agniveśa" elif 'bhela' in text: return "Bhela" elif 'hareta' in text: return "Hareta" elif 'kāṅkṣeya' in text: return "Kāṅkṣeya" return "the Ayurvedic sage" def _extract_key_concepts(self, shloka: ShlokaData) -> Dict[str, str]: """Extract key concepts from shloka""" text = shloka.sanskrit.lower() trans = shloka.transliteration.lower() concepts = { 'speaker': self._extract_speaker(shloka), 'topic': 'Ayurvedic teaching', 'action': 'expounds', 'section_theme': shloka.section_description } # Topic detection if any(x in text for x in ['jvara', 'ज्वर']): concepts['topic'] = 'fever management' concepts['action'] = 'treats' elif any(x in text for x in ['dīrgha', 'दीर्घ', 'āyuṣya', 'आयुष्य']): concepts['topic'] = 'longevity' concepts['action'] = 'expounds' elif any(x in text for x in ['doṣa', 'दोष']): concepts['topic'] = 'humoral balance' concepts['action'] = 'analyzes' elif any(x in text for x in ['dhātu', 'धातु']): concepts['topic'] = 'tissue integrity' concepts['action'] = 'describes' elif any(x in text for x in ['cikitsā', 'चिकित्सा']): concepts['topic'] = 'therapeutics' concepts['action'] = 'prescribes' return concepts def _build_system_prompt(self, lang: str, section: str) -> str: """Build system prompt based on language""" if lang == "hi": return f"You are an expert Ayurvedic scholar fluent in Hindi. Answer based on Charaka Samhita {section} with proper citations and full Sanskrit section names." elif lang == "sa": return f"You are an expert Ayurvedic scholar fluent in Sanskrit. Answer based on Charaka Samhita {section} with classical Sanskrit citations." else: return f"You are an expert Ayurvedic scholar trained in Charaka Samhita. Provide accurate responses with precise citations including full Sanskrit section names from {section}." def generate_all_conversation_types(self, shloka: ShlokaData) -> List[Dict]: """Generate ALL 28 question types per shloka""" conversations = [] concepts = self._extract_key_concepts(shloka) sec_questions = self.section_questions.get(shloka.section_code, self.section_questions["SU"]) # === NATURAL QUESTIONS (No text references) === # 1-3. Conceptual Understanding (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_conceptual(shloka, concepts, lang)) # 4-6. Analytical Reasoning (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_analytical(shloka, concepts, lang)) # 7-9. Application/Clinical (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_application(shloka, concepts, lang, sec_questions)) # === REFERENCE QUESTIONS (With shloka_id) === # 10-12. Translation requests (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_translation_ref(shloka, lang)) # 13-15. ID lookup (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_id_lookup(shloka, lang)) # 16-18. Complete details (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_complete_details(shloka, lang)) # 19-21. Chapter context (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_chapter_context(shloka, lang)) # 22-24. Transliteration request (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_transliteration_ref(shloka, lang)) # 25. Multi-turn conversation conversations.append(self._make_multi_turn(shloka, concepts)) # === ADVANCED QUESTIONS === # 26. Clinical Case Study conversations.append(self._make_clinical_case(shloka, concepts, sec_questions)) # 27. Word Analysis/Padacchedha conversations.append(self._make_word_analysis(shloka)) # 28. Historical/Philosophical Context conversations.append(self._make_historical_context(shloka, concepts)) # === CROSS-REFERENCE QUESTIONS === # 29. Comparative Analysis conversations.append(self._make_comparative(shloka)) # 30. Cross-chapter integration conversations.append(self._make_cross_chapter(shloka, concepts)) # 31. Section relationship conversations.append(self._make_section_relationship(shloka, concepts)) # 32. Contradiction resolution (if applicable) conversations.append(self._make_contradiction_resolution(shloka)) # === QnA VARIATIONS === # 33-35. Meaning questions (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_meaning_qna(shloka, lang)) # 36-38. Specific verse explanation (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_specific_verse_qna(shloka, lang)) # 39-41. Concept-based QnA (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_concept_qna(shloka, concepts, lang)) # 42-44. Location-based QnA (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_location_qna(shloka, lang)) # 45-47. Context-based QnA (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_context_qna(shloka, lang)) # 48-50. Short/Quick QnA (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_short_qna(shloka, lang)) # 51-53. Section info QnA (EN/HI/SA) for lang in ["en", "hi", "sa"]: conversations.append(self._make_section_info_qna(shloka, lang)) # 54. Analytical QnA conversations.append(self._make_analytical_qna(shloka, concepts, "en")) # 55. Word-by-word breakdown conversations.append(self._make_word_by_word(shloka, "en")) # 56. Application QnA conversations.append(self._make_application_qna(shloka, concepts, "en")) return conversations def _make_conceptual(self, shloka: ShlokaData, concepts: Dict, lang: str) -> Dict: """Generate conceptual understanding question""" if lang == "en": question = f"What significance did {concepts['speaker']} attach to {concepts['topic']} when establishing the framework for {shloka.section_description}?" answer = self._build_answer_en(shloka, "conceptual") elif lang == "hi": question = f"{concepts['speaker']} ने {shloka.section_name_hi} में {concepts['topic']} के विषय में क्या महत्व स्थापित किया?" answer = self._build_answer_hi(shloka, "conceptual") else: # sa question = f"{concepts['speaker']} {shloka.section_name_sa} {concepts['topic']} का प्राधान्यं ददाति?" answer = self._build_answer_sa(shloka, "conceptual") return { "conversation_type": f"natural_conceptual_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_analytical(self, shloka: ShlokaData, concepts: Dict, lang: str) -> Dict: """Generate analytical reasoning question""" if lang == "en": question = f"How does {concepts['speaker']}'s approach to {concepts['topic']} in {shloka.section_name} relate to the broader Ayurvedic understanding of {concepts['section_theme']}?" answer = self._build_answer_en(shloka, "analytical") elif lang == "hi": question = f"{shloka.section_name_hi} में {concepts['topic']} के प्रति {concepts['speaker']} का दृष्टिकोण आयुर्वेद की समग्र समझ से कैसे जुड़ता है?" answer = self._build_answer_hi(shloka, "analytical") else: question = f"{shloka.section_name_sa} {concepts['topic']} {concepts['speaker']} दृष्टिकोणः कथं सम्बद्धः?" answer = self._build_answer_sa(shloka, "analytical") return { "conversation_type": f"natural_analytical_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_application(self, shloka: ShlokaData, concepts: Dict, lang: str, sec_q: Dict) -> Dict: """Generate practical application question""" applications = sec_q.get('applications', ['clinical practice']) app = random.choice(applications) if lang == "en": question = f"How would a practitioner apply {concepts['speaker']}'s teaching on {concepts['topic']} in {app}?" answer = self._build_answer_en(shloka, "application") elif lang == "hi": question = f"{app} में {concepts['topic']} के विषय में {concepts['speaker']} के उपदेश का प्रयोग कैसे किया जाए?" answer = self._build_answer_hi(shloka, "application") else: question = f"{app} {concepts['topic']} {concepts['speaker']} उपदेशः कथं प्रयोक्तव्यः?" answer = self._build_answer_sa(shloka, "application") return { "conversation_type": f"natural_application_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_translation_ref(self, shloka: ShlokaData, lang: str) -> Dict: """Generate translation reference question""" if lang == "en": question = f"Translate this Sanskrit text: {shloka.sanskrit}" answer = self._build_reference_answer(shloka, "translation") elif lang == "hi": question = f"इस संस्कृत श्लोक का अनुवाद करें: {shloka.sanskrit}" answer = self._build_reference_answer(shloka, "translation") else: question = f"एतं संस्कृतश्लोकं भाषान्तरितं कुर्वन्तु: {shloka.sanskrit}" answer = self._build_reference_answer(shloka, "translation") return { "conversation_type": f"reference_translation_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_id_lookup(self, shloka: ShlokaData, lang: str) -> Dict: """Generate ID lookup question""" if lang == "en": question = f"What does {shloka.shloka_id} say?" answer = self._build_reference_answer(shloka, "complete") elif lang == "hi": question = f"{shloka.shloka_id} में क्या कहा गया है?" answer = self._build_reference_answer(shloka, "complete") else: question = f"किं {shloka.shloka_id} वदति?" answer = self._build_reference_answer(shloka, "complete") return { "conversation_type": f"reference_id_lookup_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_complete_details(self, shloka: ShlokaData, lang: str) -> Dict: """Generate complete details question""" if lang == "en": question = f"Give me complete details for {shloka.shloka_id}" answer = self._build_reference_answer(shloka, "complete") elif lang == "hi": question = f"मुझे {shloka.shloka_id} की पूरी जानकारी दीजिए" answer = self._build_reference_answer(shloka, "complete") else: question = f"{shloka.shloka_id} सम्पूर्णविवरणं ददातु" answer = self._build_reference_answer(shloka, "complete") return { "conversation_type": f"reference_complete_details_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_chapter_context(self, shloka: ShlokaData, lang: str) -> Dict: """Generate chapter context question""" if lang == "en": question = f"Which chapter is {shloka.shloka_id} from?" answer = self._build_chapter_answer(shloka, "en") elif lang == "hi": question = f"{shloka.shloka_id} किस अध्याय से है?" answer = self._build_chapter_answer(shloka, "hi") else: question = f"किमध्यायः {shloka.shloka_id} कुतः?" answer = self._build_chapter_answer(shloka, "sa") return { "conversation_type": f"reference_chapter_context_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_transliteration_ref(self, shloka: ShlokaData, lang: str) -> Dict: """Generate transliteration request""" if lang == "en": question = f"Give me the transliteration for {shloka.shloka_id}" elif lang == "hi": question = f"मुझे {shloka.shloka_id} का लिप्यंतरण दीजिए" else: question = f"{shloka.shloka_id} लिप्यन्तरं ददातु" answer = self._build_transliteration_answer(shloka) return { "conversation_type": f"reference_transliteration_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_multi_turn(self, shloka: ShlokaData, concepts: Dict) -> Dict: """Generate multi-turn conversation""" return { "conversation_type": "reference_multi_turn", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": f"Explain this teaching from {shloka.shloka_id}"}, {"role": "model", "content": self._build_reference_answer(shloka, "brief")}, {"role": "user", "content": "What is the Sanskrit formulation?"}, {"role": "model", "content": f"The Sanskrit text is: {shloka.sanskrit}"}, {"role": "user", "content": "Where does this appear in the text?"}, {"role": "model", "content": self._build_chapter_answer(shloka, "en")} ] } def _make_clinical_case(self, shloka: ShlokaData, concepts: Dict, sec_q: Dict) -> Dict: """Generate clinical case study""" topics = sec_q.get('topics', ['clinical condition']) topic = random.choice(topics) question = f"A patient presents with symptoms related to {topic}. Using {concepts['speaker']}'s framework from {shloka.section_name}, how would you approach diagnosis and treatment?" answer = self._build_clinical_answer(shloka) return { "conversation_type": "advanced_clinical_case", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_word_analysis(self, shloka: ShlokaData) -> Dict: """Generate word-level analysis""" question = f"Analyze the key terms in {shloka.shloka_id} from {shloka.section_name} and explain their technical significance in Ayurvedic medicine." answer = self._build_word_analysis_answer(shloka) return { "conversation_type": "advanced_word_analysis", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_historical_context(self, shloka: ShlokaData, concepts: Dict) -> Dict: """Generate historical/philosophical context question""" question = f"How does {shloka.shloka_id} reflect the classical Ayurvedic approach to {concepts['section_theme']}, and what philosophical principles underlie this teaching?" answer = self._build_historical_answer(shloka) return { "conversation_type": "advanced_historical_context", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_comparative(self, shloka: ShlokaData) -> Dict: """Generate comparative analysis""" question = f"Compare the approach in {shloka.shloka_id} ({shloka.section_name}) with similar concepts in other Ayurvedic texts. What distinguishes Charaka's perspective?" answer = self._build_comparative_answer(shloka) return { "conversation_type": "advanced_comparative", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_cross_chapter(self, shloka: ShlokaData, concepts: Dict) -> Dict: """Generate cross-chapter integration""" question = f"How does {shloka.shloka_id} connect to the broader progression of teachings within {shloka.section_name}?" answer = self._build_cross_chapter_answer(shloka) return { "conversation_type": "cross_reference_cross_chapter", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_section_relationship(self, shloka: ShlokaData, concepts: Dict) -> Dict: """Generate section relationship question""" question = f"How does {concepts['speaker']}'s teaching in {shloka.shloka_id} ({shloka.section_name}) relate to the overall structure of the Charaka Samhita?" answer = self._build_section_relationship_answer(shloka) return { "conversation_type": "cross_reference_section_relationship", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_contradiction_resolution(self, shloka: ShlokaData) -> Dict: """Generate contradiction resolution""" question = f"Some scholars interpret {shloka.shloka_id} differently from traditional commentators. How do we reconcile these perspectives while maintaining fidelity to the text?" answer = self._build_contradiction_answer(shloka) return { "conversation_type": "cross_reference_contradiction_resolution", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_meaning_qna(self, shloka: ShlokaData, lang: str) -> Dict: """Generate meaning QnA""" if lang == "en": question = f"What is the meaning of the verse beginning with '{shloka.sanskrit[:30]}...' in {shloka.section_name}?" answer = self._build_qna_answer(shloka, lang) elif lang == "hi": question = f"{shloka.section_name_hi} में '{shloka.sanskrit[:20]}...' से शुरू होने वाले श्लोक का अर्थ क्या है?" answer = self._build_qna_answer(shloka, lang) else: question = f"{shloka.section_name_sa} '{shloka.sanskrit[:20]}...' आरभ्यमानः श्लोकः का अर्थः?" answer = self._build_qna_answer(shloka, lang) return { "conversation_type": f"qna_meaning_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_specific_verse_qna(self, shloka: ShlokaData, lang: str) -> Dict: """Generate specific verse QnA""" if lang == "en": question = f"Can you explain {shloka.shloka_id} in detail?" answer = self._build_qna_answer(shloka, lang) elif lang == "hi": question = f"क्या आप {shloka.shloka_id} को विस्तार से समझा सकते हैं?" answer = self._build_qna_answer(shloka, lang) else: question = f"किं भवान् {shloka.shloka_id} विस्तरेण व्याख्यातुं शक्नोति?" answer = self._build_qna_answer(shloka, lang) return { "conversation_type": f"qna_specific_verse_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_concept_qna(self, shloka: ShlokaData, concepts: Dict, lang: str) -> Dict: """Generate concept QnA""" if lang == "en": question = f"What does the Charaka Samhita teach about {concepts['topic']} in {shloka.shloka_id}?" answer = self._build_qna_answer(shloka, lang) elif lang == "hi": question = f"चरक संहिता में {shloka.shloka_id} के संबंध में {concepts['topic']} के बारे में क्या उपदेश है?" answer = self._build_qna_answer(shloka, lang) else: question = f"चरकसंहितायाः {shloka.shloka_id} {concepts['topic']} का उपदेशः?" answer = self._build_qna_answer(shloka, lang) return { "conversation_type": f"qna_concept_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_location_qna(self, shloka: ShlokaData, lang: str) -> Dict: """Generate location QnA""" if lang == "en": question = f"Where can I find the teaching about {shloka.section_description} in verse {shloka.verse}?" answer = self._build_location_answer(shloka, lang) elif lang == "hi": question = f"श्लोक {shloka.verse} में {shloka.section_description} के बारे में शिक्षा कहाँ मिलती है?" answer = self._build_location_answer(shloka, lang) else: question = f"श्लोकः {shloka.verse} {shloka.section_description} शिक्षा कुत्र लभ्यते?" answer = self._build_location_answer(shloka, lang) return { "conversation_type": f"qna_location_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_context_qna(self, shloka: ShlokaData, lang: str) -> Dict: """Generate context QnA""" if lang == "en": question = f"What is the broader context of {shloka.shloka_id} in {shloka.section_name}?" answer = self._build_context_answer(shloka, lang) elif lang == "hi": question = f"{shloka.section_name_hi} में {shloka.shloka_id} का व्यापक संदर्भ क्या है?" answer = self._build_context_answer(shloka, lang) else: question = f"{shloka.section_name_sa} {shloka.shloka_id} विस्तीर्णः संदर्भः?" answer = self._build_context_answer(shloka, lang) return { "conversation_type": f"qna_context_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_short_qna(self, shloka: ShlokaData, lang: str) -> Dict: """Generate short QnA""" if lang == "en": question = f"Briefly, what does {shloka.shloka_id} say?" answer = self._build_brief_answer(shloka, lang) elif lang == "hi": question = f"संक्षेप में, {shloka.shloka_id} में क्या कहा गया है?" answer = self._build_brief_answer(shloka, lang) else: question = f"संक्षेपेण, किं {shloka.shloka_id} वदति?" answer = self._build_brief_answer(shloka, lang) return { "conversation_type": f"qna_short_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_section_info_qna(self, shloka: ShlokaData, lang: str) -> Dict: """Generate section info QnA""" if lang == "en": question = f"What is {shloka.section_name} about?" answer = self._build_section_info_answer(shloka, lang) elif lang == "hi": question = f"{shloka.section_name_hi} किस बारे में है?" answer = self._build_section_info_answer(shloka, lang) else: question = f"किं {shloka.section_name_sa} विषये?" answer = self._build_section_info_answer(shloka, lang) return { "conversation_type": f"qna_section_info_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_analytical_qna(self, shloka: ShlokaData, concepts: Dict, lang: str) -> Dict: """Generate analytical QnA""" question = f"What teaching does {shloka.shloka_id} provide regarding {shloka.section_description}?" answer = self._build_analytical_answer(shloka) return { "conversation_type": f"qna_analytical_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_word_by_word(self, shloka: ShlokaData, lang: str) -> Dict: """Generate word-by-word analysis""" question = f"Can you break down {shloka.shloka_id} word by word?" answer = self._build_word_by_word_answer(shloka) return { "conversation_type": f"qna_word_by_word_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } def _make_application_qna(self, shloka: ShlokaData, concepts: Dict, lang: str) -> Dict: """Generate application QnA""" question = f"How can we apply the teaching of {shloka.shloka_id} in practice?" answer = self._build_application_answer(shloka) return { "conversation_type": f"qna_application_{lang}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": question}, {"role": "model", "content": answer} ] } # === ANSWER BUILDERS === def _build_answer_en(self, shloka: ShlokaData, style: str) -> str: """Build English answer""" base = f"""{shloka.translation_en} This teaching from **{shloka.section_name}** establishes foundational principles for {shloka.section_description}. Located in Chapter {shloka.chapter}, Verse {shloka.verse}, this verse contributes to the systematic understanding of Ayurvedic medicine. **Sanskrit:** {shloka.sanskrit} **Transliteration:** {shloka.transliteration} **Full Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" return base def _build_answer_hi(self, shloka: ShlokaData, style: str) -> str: """Build Hindi answer""" base = f"""{shloka.translation_hi} **{shloka.section_name_hi}** का यह उपदेश {shloka.section_description} के लिए मूलभूत सिद्धांत स्थापित करता है। अध्याय {shloka.chapter}, श्लोक {shloka.verse} में स्थित, यह श्लोक आयुर्वेद की व्यवस्थित समझ में योगदान देता है। **संस्कृत:** {shloka.sanskrit} **पूर्ण सन्दर्भ:** {shloka.section_name_hi} अध्याय {shloka.chapter}, श्लोक {shloka.verse}""" return base def _build_answer_sa(self, shloka: ShlokaData, style: str) -> str: """Build Sanskrit answer""" base = f"""{shloka.translation_en} **{shloka.section_name_sa}** एतत् उपदेशं {shloka.section_description} इति मूलभूतं महत्त्वं स्थापयति। अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}। **मूलम्:** {shloka.sanskrit} **सम्पूर्ण सन्दर्भः:** {shloka.section_name_sa} अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}""" return base def _build_reference_answer(self, shloka: ShlokaData, detail: str) -> str: """Build reference answer""" return f"""**Complete text of {shloka.shloka_id}:** **Sanskrit:** {shloka.sanskrit} **Transliteration (IAST):** {shloka.transliteration} **English Translation:** {shloka.translation_en} **Hindi Translation:** {shloka.translation_hi} **Source:** **{shloka.section_name}** ({shloka.section_name_hi}) **Section:** {shloka.section_description} **Chapter:** {shloka.chapter}, Verse {shloka.verse} **Full Citation:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_chapter_answer(self, shloka: ShlokaData, lang: str) -> str: """Build chapter context answer""" if lang == "hi": return f"""**{shloka.shloka_id}** **{shloka.section_name_hi}**, अध्याय {shloka.chapter}, श्लोक {shloka.verse} से है। **अनुभाग:** {shloka.section_name_hi} - {shloka.section_description} **श्लोक:** {shloka.sanskrit} **पूर्ण सन्दर्भ:** {shloka.section_name_hi} अध्याय {shloka.chapter}, श्लोक {shloka.verse}""" elif lang == "sa": return f"""**{shloka.shloka_id}** **{shloka.section_name_sa}**, अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}। **विभागः:** {shloka.section_name_sa} - {shloka.section_description} **श्लोकः:** {shloka.sanskrit} **सम्पूर्ण सन्दर्भः:** {shloka.section_name_sa} अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}""" else: return f"""**{shloka.shloka_id}** is from **{shloka.section_name}**, Chapter {shloka.chapter}, Verse {shloka.verse}. **Section:** {shloka.section_name} - {shloka.section_description} **Verse Text:** {shloka.sanskrit} **Full Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_transliteration_answer(self, shloka: ShlokaData) -> str: return f"""**Transliteration of {shloka.shloka_id}:** **IAST:** {shloka.transliteration} **Sanskrit:** {shloka.sanskrit} **Translation:** {shloka.translation_en} **Source:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_clinical_answer(self, shloka: ShlokaData) -> str: return f"""**Clinical Application:** {shloka.translation_en} **Diagnostic Framework:** This teaching from **{shloka.section_name}** provides the theoretical basis for clinical assessment: 1. **Assessment Phase:** Understanding condition within {shloka.section_description} 2. **Analysis Phase:** Applying principles from this verse 3. **Treatment Planning:** Developing interventions aligned with the teaching **Sanskrit Reference:** {shloka.sanskrit} **Source:** {shloka.section_name}, Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_word_analysis_answer(self, shloka: ShlokaData) -> str: return f"""**Technical Analysis of {shloka.shloka_id}:** **Sanskrit:** {shloka.sanskrit} **Transliteration:** {shloka.transliteration} **Key Terms:** The terminology reflects precise Ayurvedic technical language from **{shloka.section_name}** dealing with {shloka.section_description}. **Translation:** {shloka.translation_en} **Significance:** These terms establish the conceptual framework for {shloka.section_description}. **Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_historical_answer(self, shloka: ShlokaData) -> str: return f"""**Historical and Philosophical Context:** **The Teaching:** {shloka.translation_en} **Philosophical Basis:** This verse from **{shloka.section_name}** ({shloka.section_description}) reflects classical Ayurvedic wisdom integrating empirical observation with systematic understanding. **Context:** Located in Chapter {shloka.chapter} of {shloka.section_name}, this teaching represents authoritative exposition on {shloka.section_description}. **Sanskrit:** {shloka.sanskrit} **Source:** {shloka.section_name}, Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_comparative_answer(self, shloka: ShlokaData) -> str: return f"""**Comparative Analysis:** **Charaka's Approach in {shloka.shloka_id}:** {shloka.translation_en} **Distinctive Features:** This verse from **{shloka.section_name}** demonstrates Charaka's systematic methodology: 1. **Comprehensive Scope:** Theory and practice 2. **Systematic Organization:** Logical progression 3. **Clinical Orientation:** Practical applicability **Within Charaka Samhita:** Part of Chapter {shloka.chapter}, connecting to broader themes while maintaining emphasis on {shloka.section_description}. **Technical Source:** {shloka.sanskrit} **Location:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_cross_chapter_answer(self, shloka: ShlokaData) -> str: return f"""**Cross-Chapter Integration:** **{shloka.shloka_id}** in Context: {shloka.translation_en} **Connection to {shloka.section_name}:** This verse in Chapter {shloka.chapter} relates to the systematic exposition on {shloka.section_description}. **Sequential Understanding:** - Previous verses establish preliminary concepts - {shloka.shloka_id} introduces the specific teaching - Subsequent verses expand on applications **Sanskrit:** {shloka.sanskrit} **Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_section_relationship_answer(self, shloka: ShlokaData) -> str: return f"""**Section Relationship:** **{shloka.shloka_id}** in {shloka.section_name}: {shloka.translation_en} **Role in Charaka Samhita:** This verse contributes to **{shloka.section_name}** ({shloka.section_description}), one of the eight sections of Charaka Samhita. **Integration:** The teaching connects to broader Ayurvedic principles while maintaining {shloka.section_name}'s specific focus on {shloka.section_description}. **Sanskrit:** {shloka.sanskrit} **Source:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_contradiction_answer(self, shloka: ShlokaData) -> str: return f"""**Resolution of Perspectives:** **{shloka.shloka_id}:** {shloka.translation_en} **Traditional Interpretation:** This verse from **{shloka.section_name}** is understood within the classical commentarial tradition as establishing {shloka.section_description}. **Reconciliation:** Apparent differences in interpretation resolve when considering the verse's placement in Chapter {shloka.chapter} and its relationship to preceding and following teachings. **Sanskrit Text:** {shloka.sanskrit} **Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_qna_answer(self, shloka: ShlokaData, lang: str) -> str: """Build QnA style answer""" if lang == "hi": return f"""{shloka.translation_hi} **संस्कृत:** {shloka.sanskrit} **सन्दर्भ:** {shloka.section_name_hi} अध्याय {shloka.chapter}, श्लोक {shloka.verse}""" elif lang == "sa": return f"""{shloka.translation_en} **मूलम्:** {shloka.sanskrit} **सन्दर्भः:** {shloka.section_name_sa} अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}""" else: return f"""{shloka.translation_en} **Sanskrit:** {shloka.sanskrit} **Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_location_answer(self, shloka: ShlokaData, lang: str) -> str: if lang == "hi": return f"**{shloka.shloka_id}** **{shloka.section_name_hi}**, अध्याय {shloka.chapter}, श्लोक {shloka.verse} में मिलता है।\n\n**श्लोक:** {shloka.sanskrit}" elif lang == "sa": return f"**{shloka.shloka_id}** **{shloka.section_name_sa}**, अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}।\n\n**श्लोकः:** {shloka.sanskrit}" else: return f"**{shloka.shloka_id}** is found in **{shloka.section_name}**, Chapter {shloka.chapter}, Verse {shloka.verse}.\n\n**Verse:** {shloka.sanskrit}" def _build_context_answer(self, shloka: ShlokaData, lang: str) -> str: if lang == "hi": return f"""**{shloka.shloka_id} का संदर्भ:** {shloka.translation_hi} **{shloka.section_name_hi}** ({shloka.section_description}) में अध्याय {shloka.chapter} का यह श्लोक {shloka.section_description} की समझ में योगदान देता है। **संस्कृत:** {shloka.sanskrit}""" elif lang == "sa": return f"""**{shloka.shloka_id} संदर्भः:** {shloka.translation_en} **{shloka.section_name_sa}** ({shloka.section_description}) अध्यायः {shloka.chapter} एतत् श्लोकः {shloka.section_description} बोधाय। **मूलम्:** {shloka.sanskrit}""" else: return f"""**Context of {shloka.shloka_id}:** {shloka.translation_en} **{shloka.section_name}** ({shloka.section_description}) - Chapter {shloka.chapter}, this verse contributes to understanding {shloka.section_description}. **Sanskrit:** {shloka.sanskrit}""" def _build_brief_answer(self, shloka: ShlokaData, lang: str) -> str: if lang == "hi": return f"{shloka.translation_hi}\n\n**संदर्भ:** {shloka.section_name_hi} अध्याय {shloka.chapter}, श्लोक {shloka.verse}" elif lang == "sa": return f"{shloka.translation_en}\n\n**सन्दर्भः:** {shloka.section_name_sa} अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}" else: return f"{shloka.translation_en}\n\n**Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}" def _build_section_info_answer(self, shloka: ShlokaData, lang: str) -> str: if lang == "hi": return f"**{shloka.section_name_hi}** चरक संहिता का एक अनुभाग है जो {shloka.section_description} से संबंधित है।\n\n**उदाहरण श्लोक:** {shloka.shloka_id} - {shloka.sanskrit}" elif lang == "sa": return f"**{shloka.section_name_sa}** चरकसंहितायाः अष्टाध्यायीभ्यः एकम् - {shloka.section_description}।\n\n**उदाहरणश्लोकः:** {shloka.shloka_id} - {shloka.sanskrit}" else: return f"**{shloka.section_name}** is one of the eight sections of Charaka Samhita dealing with {shloka.section_description}.\n\n**Example verse:** {shloka.shloka_id} - {shloka.sanskrit}" def _build_analytical_answer(self, shloka: ShlokaData) -> str: return f"""**Analysis:** {shloka.translation_en} **Key Teaching:** This verse from **{shloka.section_name}** addresses {shloka.section_description} through systematic exposition. **Sanskrit:** {shloka.sanskrit} **Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def _build_word_by_word_answer(self, shloka: ShlokaData) -> str: return f"""**Word-by-Word Analysis of {shloka.shloka_id}:** **Complete Verse:** {shloka.sanskrit} **Transliteration:** {shloka.transliteration} **Translation:** {shloka.translation_en} **Context:** {shloka.section_name} Chapter {shloka.chapter}, dealing with {shloka.section_description}.""" def _build_application_answer(self, shloka: ShlokaData) -> str: return f"""**Practical Application:** {shloka.translation_en} **Application:** This teaching from **{shloka.section_name}** ({shloka.section_description}) provides guidance for {shloka.section_description} in clinical practice. **Sanskrit:** {shloka.sanskrit} **Source:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}""" def generate_full_dataset(self): """Generate complete enriched dataset""" print("Generating full enriched dataset...") all_conversations = [] for idx, shloka in enumerate(self.shlokas): if idx % 100 == 0: print(f"Processing {idx}/{len(self.shlokas)} shlokas...") conversations = self.generate_all_conversation_types(shloka) for conv in conversations: enriched_conv = { "shloka_id": shloka.shloka_id, "conversation_type": conv["conversation_type"], "messages": conv["messages"], "metadata": { "sanskrit": shloka.sanskrit, "transliteration": shloka.transliteration, "section": shloka.section_name, "section_hi": shloka.section_name_hi, "section_sa": shloka.section_name_sa, "section_code": shloka.section_code, "section_description": shloka.section_description, "chapter": shloka.chapter, "verse": shloka.verse, "translation_en": shloka.translation_en, "translation_hi": shloka.translation_hi } } all_conversations.append(enriched_conv) return all_conversations def save_dataset(self, conversations: List[Dict], output_file: str): """Save in pretty-printed JSONL format""" print(f"Saving {len(conversations)} conversations to {output_file}...") with open(output_file, 'w', encoding='utf-8') as f: for conv in conversations: f.write(json.dumps(conv, ensure_ascii=False, indent=2) + '\n') # Calculate size import os size_mb = os.path.getsize(output_file) / (1024 * 1024) print(f"Saved: {size_mb:.1f} MB") # Statistics stats = {} for conv in conversations: ctype = conv["conversation_type"] stats[ctype] = stats.get(ctype, 0) + 1 print(f"\nTop 15 conversation types:") for ctype, count in sorted(stats.items(), key=lambda x: -x[1])[:15]: print(f" {ctype}: {count}") return stats def main(): print("=" * 60) print("COMPREHENSIVE GEMMA-4 DATASET GENERATOR") print("=" * 60) generator = ComprehensiveDatasetGenerator("shlokas_export_2026-04-30.json") print("\nLoading source data...") generator.load_source() print("\nGenerating all conversations...") conversations = generator.generate_full_dataset() print(f"\nTotal conversations generated: {len(conversations)}") print(f"Per shloka: ~{len(conversations) // len(generator.shlokas)} conversations") output_file = "gemma4_ayurveda_full_enriched.jsonl" print(f"\nSaving to {output_file}...") stats = generator.save_dataset(conversations, output_file) # Save statistics with open("gemma4_full_dataset_stats.json", 'w', encoding='utf-8') as f: json.dump({ "total_shlokas": len(generator.shlokas), "total_conversations": len(conversations), "conversations_per_shloka": len(conversations) // len(generator.shlokas), "conversation_types": len(stats), "type_distribution": stats }, f, indent=2) print("\n" + "=" * 60) print("DATASET GENERATION COMPLETE") print("=" * 60) if __name__ == "__main__": main()