#!/usr/bin/env python3 """ Generate 56VARIATIONS enriched dataset for Gemma-4 training Includes 56 question types × 5 variations each = ~280 questions per shloka Target: ~2GB, ~1M conversations Format: ShareGPT with multiple question variations per type """ import json import re import random from typing import Dict, List, Any from dataclasses import dataclass from pathlib import Path # Full section mappings SECTIONS = { "SU": "Sutrasthana (सूत्रस्थान)", "NI": "Nidanasthana (निदानस्थान)", "VI": "Vimanasthana (विमानस्थान)", "SHA": "Sharirasthana (शारीरस्थान)", "IND": "Indriyasthana (इन्द्रियस्थान)", "CI": "Cikitsasthana (चिकित्सास्थान)", "CHI": "Cikitsasthana (चिकित्सास्थान)", "KA": "Kalpasthana (कल्पस्थान)", "KAL": "Kalpasthana (कल्पस्थान)", "SI": "Siddhisthana (सिद्धिस्थान)", "SID": "Siddhisthana (सिद्धिस्थान)", "IN": "Indriyasthana (इन्द्रियस्थान)" } SECTION_DESCRIPTIONS = { "SU": "General Principles and Fundamentals", "NI": "Pathology and Disease Causation", "VI": "Quantitative and Measurement Science", "SHA": "Human Body and Anatomy", "IND": "Sensory Organs and Clinical Signs", "CI": "Treatment and Therapeutics", "CHI": "Treatment and Therapeutics", "KA": "Pharmaceutical Preparations", "KAL": "Pharmaceutical Preparations", "SI": "Attainment of Perfection and Results", "SID": "Attainment of Perfection and Results", "IN": "Sensory Organs and Clinical Signs" } SECTIONS_HI = { "SU": "सूत्रस्थान", "NI": "निदानस्थान", "VI": "विमानस्थान", "SHA": "शारीरस्थान", "IND": "इन्द्रियस्थान", "CI": "चिकित्सास्थान", "CHI": "चिकित्सास्थान", "KA": "कल्पस्थान", "KAL": "कल्पस्थान", "SI": "सिद्धिस्थान", "SID": "सिद्धिस्थान", "IN": "इन्द्रियस्थान" } SECTIONS_SA = { "SU": "सूत्रस्थानम्", "NI": "निदानस्थानम्", "VI": "विमानस्थानम्", "SHA": "शारीरस्थानम्", "IND": "इन्द्रियस्थानम्", "CI": "चिकित्सास्थानम्", "CHI": "चिकित्सास्थानम्", "KA": "कल्पस्थानम्", "KAL": "कल्पस्थानम्", "SI": "सिद्धिस्थानम्", "SID": "सिद्धिस्थानम्", "IN": "इन्द्रियस्थानम्" } # Question variation templates for each type CONCEPTUAL_VARIATIONS_EN = [ "What significance did {speaker} attach to {topic} when establishing the framework for {section}?", "Why did {speaker} emphasize {topic} as foundational to {section}?", "How does {speaker} position {topic} within the broader scope of {section}?", "What principle guided {speaker} in presenting {topic} at the outset of {section}?", "In what manner does {speaker} introduce {topic} as essential to {section}?" ] CONCEPTUAL_VARIATIONS_HI = [ "{speaker} ने {section} में {topic} के विषय में क्या महत्व स्थापित किया?", "{speaker} ने {section} में {topic} को मूलभूत क्यों माना?", "{section} की व्यापक व्यवस्था में {speaker} {topic} को किस रूप में प्रस्तुत करते हैं?", "{section} के प्रारम्भ में {topic} प्रस्तुत करने हेतु {speaker} का कौन-सा सिद्धान्त रहा?", "{speaker} ने {section} में {topic} को किस प्रकार आवश्यक माना?" ] CONCEPTUAL_VARIATIONS_SA = [ "{speaker} {section} {topic} का प्राधान्यं ददाति?", "{speaker} {section} {topic} मूलभूतं किमर्थं मन्यते?", "{section} व्यवस्थायां {speaker} {topic} कथं प्रस्तौति?", "{section} आरम्भे {topic} प्रस्तुतये {speaker} सिद्धान्तः कः?", "{speaker} {section} {topic} कथं आवश्यकं मन्यते?" ] ANALYTICAL_VARIATIONS_EN = [ "How does {speaker}'s approach to {topic} in {section} relate to {theme}?", "What connections does {speaker} draw between {topic} and {theme} in {section}?", "In what way does {speaker} analyze the relationship of {topic} to {theme}?", "How would {speaker} explain the integration of {topic} within {theme}?", "What analytical framework does {speaker} employ for {topic} in {section}?" ] ANALYTICAL_VARIATIONS_HI = [ "{section} में {topic} के प्रति {speaker} का दृष्टिकोण {theme} से कैसे जुड़ता है?", "{section} में {speaker} {topic} और {theme} के बीच क्या सम्बन्ध दर्शाते हैं?", "{section} में {topic} का {theme} से सम्बन्ध {speaker} किस प्रकार विश्लेषण करते हैं?", "{speaker} {topic} को {theme} में किस प्रकार सम्मिलित मानते हैं?", "{section} में {topic} हेतु {speaker} का विश्लेषणात्मक ढाँचा क्या है?" ] ANALYTICAL_VARIATIONS_SA = [ "{section} {topic} {speaker} दृष्टिकोणः {theme} कथं सम्बद्धः?", "{section} {speaker} {topic} {theme} सम्बन्धः किं दर्शयति?", "{section} {topic} {theme} सम्बन्धं {speaker} कथं विश्लेषयति?", "{speaker} {topic} {theme} कथं सम्मिलितं मन्यते?", "{section} {topic} {speaker} विश्लेषणढाचा कः?" ] APPLICATION_VARIATIONS_EN = [ "How would a practitioner apply {speaker}'s teaching on {topic} in {app}?", "What practical implementation follows from {speaker}'s exposition of {topic}?", "In clinical settings, how should one utilize {speaker}'s guidance on {topic}?", "What therapeutic protocol derives from {speaker}'s teaching on {topic}?", "How can {topic} as taught by {speaker} be operationalized in practice?" ] APPLICATION_VARIATIONS_HI = [ "{app} में {topic} के विषय में {speaker} के उपदेश का प्रयोग कैसे किया जाए?", "{speaker} के {topic} के प्रकाशन से कौन-सा व्यावहारिक अनुप्रयोग निकलता है?", "चिकित्सा-प्रसंग में {topic} पर {speaker} का मार्गदर्शन कैसे उपयोग करें?", "{topic} पर {speaker} के उपदेश से कौन-सा चिकित्सा-प्रोटोकॉल प्राप्त होता है?", "{speaker} द्वारा बताए गए {topic} का अभ्यास में कैसे परिकल्पना करें?" ] APPLICATION_VARIATIONS_SA = [ "{app} {topic} {speaker} उपदेशः कथं प्रयोक्तव्यः?", "{speaker} {topic} प्रकाशनात् कः व्यावहारिक अनुप्रयोगः उत्पद्यते?", "चिकित्सासन्दर्भे {topic} {speaker} मार्गदर्शनं कथं उपयोक्तव्यम्?", "{topic} {speaker} उपदेशात् कः चिकित्साप्रोटोकालः प्राप्यते?", "{speaker} {topic} अभ्यासे कथं परिकल्पयितव्यम्?" ] REFERENCE_VARIATIONS_EN = [ "Translate this: {sanskrit}", "What is the English meaning of: {sanskrit}", "Provide the translation for this Sanskrit verse: {sanskrit}", "Render this into English: {sanskrit}", "What does this mean: {sanskrit}" ] REFERENCE_VARIATIONS_HI = [ "इसका अनुवाद करें: {sanskrit}", "इस संस्कृत श्लोक का हिंदी अर्थ बताएँ: {sanskrit}", "अनुवाद दीजिए: {sanskrit}", "इसका हिंदी रूपांतर करें: {sanskrit}", "यह क्या अर्थ रखता है: {sanskrit}" ] REFERENCE_VARIATIONS_SA = [ "भाषान्तरितं कुर्वन्तु: {sanskrit}", "अर्थं वदन्तु: {sanskrit}", "तर्जमः ददातु: {sanskrit}", "भाषान्तरं कुरुत: {sanskrit}", "किं अर्थः: {sanskrit}" ] ID_LOOKUP_VARIATIONS_EN = [ "What does {id} say?", "Explain the content of {id}", "Tell me what {id} contains", "What is stated in {id}?", "Give me the teaching from {id}" ] ID_LOOKUP_VARIATIONS_HI = [ "{id} में क्या कहा गया है?", "{id} का विवरण दीजिए", "{id} में क्या है?", "{id} में क्या वर्णित है?", "{id} से क्या उपदेश मिलता है?" ] ID_LOOKUP_VARIATIONS_SA = [ "{id} किं वदति?", "{id} विवरणं ददातु", "{id} किं अस्ति?", "{id} किं वर्णितम्?", "{id} उपदेशः कः?" ] MEANING_VARIATIONS_EN = [ "What is the meaning of this teaching?", "Explain the significance", "What does this convey?", "What is being communicated here?", "What wisdom is expressed?" ] MEANING_VARIATIONS_HI = [ "इस उपदेश का अर्थ क्या है?", "महत्व समझाइए", "यह क्या कहता है?", "यहाँ क्या संप्रेषित है?", "क्या ज्ञान व्यक्त किया गया है?" ] MEANING_VARIATIONS_SA = [ "अर्थः कः?", "महत्त्वं व्याख्यातु", "किं कथितम्?", "किं संप्रेषितम्?", "किं ज्ञानं व्यक्तम्?" ] @dataclass class ShlokaData: id: int shloka_id: str sanskrit: str transliteration: str translation_en: str translation_hi: str section_code: str section_name: str section_name_hi: str section_name_sa: str section_description: str chapter: int verse: int class VariationsDatasetGenerator: def __init__(self, source_file: str, variations_per_type: int = 5): self.source_file = source_file self.shlokas: List[ShlokaData] = [] self.variations_per_type = variations_per_type def load_source(self): """Parse source JSON and create sorted list""" with open(self.source_file, 'r', encoding='utf-8') as f: data = json.load(f) for item in data: match = re.match(r'CH_([A-Z]+)_([\d]+)/([\d]+)', item['shloka_id']) if not match: continue section_code, chapter, verse = match.groups() shloka = ShlokaData( id=item['id'], shloka_id=item['shloka_id'], sanskrit=item['sanskrit_shloka'], transliteration=item['transliteration'], translation_en=item.get('translations', {}).get('english', ''), translation_hi=item.get('translations', {}).get('hindi', ''), section_code=section_code, section_name=SECTIONS.get(section_code, section_code), section_name_hi=SECTIONS_HI.get(section_code, section_code), section_name_sa=SECTIONS_SA.get(section_code, section_code), section_description=SECTION_DESCRIPTIONS.get(section_code, ''), chapter=int(chapter), verse=int(verse) ) self.shlokas.append(shloka) # Sort by shloka_id def sort_key(s): match = re.match(r'CH_([A-Z]+)_(\d+)/(\d+)', s.shloka_id) if match: sec, chap, ver = match.groups() sec_order = {'SU': 0, 'NI': 1, 'VI': 2, 'SHA': 3, 'IND': 4, 'IN': 4, 'CI': 5, 'CHI': 5, 'KA': 6, 'KAL': 6, 'SI': 7, 'SID': 7} return (sec_order.get(sec, 99), int(chap), int(ver)) return (99, 99, 99) self.shlokas.sort(key=sort_key) print(f"Loaded {len(self.shlokas)} shlokas") def _extract_speaker(self, shloka: ShlokaData) -> str: text = shloka.sanskrit.lower() if 'ātreya' in text or 'atreya' in shloka.transliteration.lower(): return "Ātreya" elif 'agniveśa' in text or 'agnivesa' in shloka.transliteration.lower(): return "Agniveśa" elif 'bhela' in text: return "Bhela" elif 'hareta' in text: return "Hareta" elif 'kāṅkṣeya' in text: return "Kāṅkṣeya" return "the Ayurvedic sage" def _extract_key_concepts(self, shloka: ShlokaData) -> Dict[str, str]: text = shloka.sanskrit.lower() concepts = { 'speaker': self._extract_speaker(shloka), 'topic': 'Ayurvedic teaching', 'section_theme': shloka.section_description } if any(x in text for x in ['jvara', 'ज्वर']): concepts['topic'] = 'fever management' elif any(x in text for x in ['dīrgha', 'दीर्घ', 'āyuṣya', 'आयुष्य']): concepts['topic'] = 'longevity' elif any(x in text for x in ['doṣa', 'दोष']): concepts['topic'] = 'humoral balance' elif any(x in text for x in ['dhātu', 'धातु']): concepts['topic'] = 'tissue integrity' elif any(x in text for x in ['cikitsā', 'चिकित्सा']): concepts['topic'] = 'therapeutics' elif any(x in text for x in ['mātrā', 'मात्रा']): concepts['topic'] = 'therapeutic measurement' elif any(x in text for x in ['nidāna', 'निदान']): concepts['topic'] = 'disease etiology' elif any(x in text for x in ['vimāna', 'विमान']): concepts['topic'] = 'quantitative assessment' elif any(x in text for x in ['śārīra', 'शारीर']): concepts['topic'] = 'human physiology' elif any(x in text for x in ['indriya', 'इन्द्रिय']): concepts['topic'] = 'sensory perception' elif any(x in text for x in ['kalpa', 'कल्प']): concepts['topic'] = 'pharmaceutical preparation' elif any(x in text for x in ['siddhi', 'सिद्धि']): concepts['topic'] = 'therapeutic accomplishment' return concepts def _build_system_prompt(self, lang: str, section: str) -> str: if lang == "hi": return f"You are an expert Ayurvedic scholar fluent in Hindi. Answer based on Charaka Samhita {section} with proper citations and full Sanskrit section names." elif lang == "sa": return f"You are an expert Ayurvedic scholar fluent in Sanskrit. Answer based on Charaka Samhita {section} with classical Sanskrit citations." else: return f"You are an expert Ayurvedic scholar trained in Charaka Samhita. Provide accurate responses with precise citations including full Sanskrit section names from {section}." def _build_answer_en(self, shloka: ShlokaData, style: str) -> str: """Build English answer""" answer = shloka.translation_en if shloka.translation_en else "[Translation not available]" if style == "conceptual": return f"{answer}\n\nThis teaching from **{shloka.section_name}** establishes foundational principles for {shloka.section_description}. Located in Chapter {shloka.chapter}, Verse {shloka.verse}, this verse contributes to the systematic understanding of Ayurvedic medicine.\n\n**Sanskrit:** {shloka.sanskrit}\n\n**Transliteration:** {shloka.transliteration}\n\n**Full Reference:** {shloka.section_name} Chapter {shloka.chapter}, Verse {shloka.verse}" elif style == "analytical": return f"{answer}\n\nFrom **{shloka.section_name}**, Chapter {shloka.chapter}, this verse demonstrates how {shloka.section_description} integrates with the broader Ayurvedic framework.\n\n**Key Analysis:** This teaching reflects the systematic approach of Charaka Samhita in addressing {shloka.section_description}.\n\n**Sanskrit:** {shloka.sanskrit}\n\n**Full Reference:** {shloka.section_name}, Chapter {shloka.chapter}, Verse {shloka.verse}" elif style == "application": return f"{answer}\n\n**Source:** {shloka.section_name}, Chapter {shloka.chapter}, Verse {shloka.verse}\n\n**Clinical Application:** This teaching from {shloka.section_name} provides practical guidance for {shloka.section_description}. Modern practitioners can apply these principles in clinical settings.\n\n**Sanskrit:** {shloka.sanskrit}\n\n**Transliteration:** {shloka.transliteration}" return answer def _build_answer_hi(self, shloka: ShlokaData, style: str) -> str: answer = shloka.translation_hi if shloka.translation_hi else shloka.translation_en if style == "conceptual": return f"{answer}\n\n**{shloka.section_name_hi}** का यह उपदेश {shloka.section_description} के लिए मूलभूत सिद्धांत स्थापित करता है। अध्याय {shloka.chapter}, श्लोक {shloka.verse} में स्थित, यह श्लोक आयुर्वेद की व्यवस्थित समझ में योगदान देता है।\n\n**संस्कृत:** {shloka.sanskrit}\n\n**पूर्ण सन्दर्भ:** {shloka.section_name_hi} अध्याय {shloka.chapter}, श्लोक {shloka.verse}" elif style == "analytical": return f"{answer}\n\n**{shloka.section_name_hi}**, अध्याय {shloka.chapter} से यह श्लोक {shloka.section_description} को आयुर्वेद की व्यापक व्यवस्था के साथ एकीकृत करने का प्रदर्शन करता है।\n\n**मुख्य विश्लेषण:** यह उपदेश चरकसंहिता की व्यवस्थित दृष्टिकोण को प्रतिबिम्बित करता है।\n\n**संस्कृत:** {shloka.sanskrit}\n\n**पूर्ण सन्दर्भ:** {shloka.section_name_hi}, अध्याय {shloka.chapter}, श्लोक {shloka.verse}" elif style == "application": return f"{answer}\n\n**स्रोत:** {shloka.section_name_hi}, अध्याय {shloka.chapter}, श्लोक {shloka.verse}\n\n**चिकित्सा अनुप्रयोग:** {shloka.section_name_hi} का यह उपदेश {shloka.section_description} हेतु व्यावहारिक मार्गदर्शन प्रदान करता है।\n\n**संस्कृत:** {shloka.sanskrit}" return answer def _build_answer_sa(self, shloka: ShlokaData, style: str) -> str: answer = shloka.sanskrit if style == "conceptual": return f"{shloka.translation_en}\n\n**{shloka.section_name_sa}** एतत् उपदेशं {shloka.section_description} इति मूलभूतं महत्त्वं स्थापयति। अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}।\n\n**मूलम्:** {shloka.sanskrit}\n\n**सम्पूर्ण सन्दर्भः:** {shloka.section_name_sa} अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}" elif style == "analytical": return f"{shloka.translation_en}\n\n**{shloka.section_name_sa}** अध्यायः {shloka.chapter}। एतत् श्लोकः {shloka.section_description} इति आयुर्वेदव्यवस्थया समन्वयं दर्शयति।\n\n**मूलम्:** {shloka.sanskrit}\n\n**सम्पूर्ण सन्दर्भः:** {shloka.section_name_sa}, अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}" elif style == "application": return f"{shloka.translation_en}\n\n**स्रोतः:** {shloka.section_name_sa}, अध्यायः {shloka.chapter}, श्लोकः {shloka.verse}\n\n**चिकित्साप्रयोगः:** {shloka.section_name_sa} एतत् उपदेशं {shloka.section_description} अर्थं व्यावहारिकं मार्गदर्शनं ददाति।\n\n**मूलम्:** {shloka.sanskrit}" return answer def _build_reference_answer(self, shloka: ShlokaData, style: str) -> str: """Build reference-style comprehensive answer""" return f"**Translation:** {shloka.translation_en}\n\n**Sanskrit:** {shloka.sanskrit}\n\n**Transliteration:** {shloka.transliteration}\n\n**Location:** {shloka.section_name}, Chapter {shloka.chapter}, Verse {shloka.verse}\n\n**Section:** {shloka.section_description}\n\n**Reference ID:** {shloka.shloka_id}" def _build_chapter_answer(self, shloka: ShlokaData, lang: str) -> str: if lang == "en": return f"{shloka.shloka_id} is from **{shloka.section_name}**, Chapter {shloka.chapter} (Verse {shloka.verse}).\n\nThis section covers {shloka.section_description}.\n\n**Sanskrit:** {shloka.sanskrit}" elif lang == "hi": return f"{shloka.shloka_id} **{shloka.section_name_hi}**, अध्याय {shloka.chapter} (श्लोक {shloka.verse}) से है।\n\nयह भाग {shloka.section_description} को समाहित करता है।\n\n**संस्कृत:** {shloka.sanskrit}" else: return f"{shloka.shloka_id} **{shloka.section_name_sa}** अध्यायः {shloka.chapter} (श्लोकः {shloka.verse})।\n\n**मूलम्:** {shloka.sanskrit}" def generate_all_conversation_types(self, shloka: ShlokaData) -> List[Dict]: """Generate 56 question types × variations per shloka""" conversations = [] concepts = self._extract_key_concepts(shloka) v = self.variations_per_type # === NATURAL QUESTIONS === # Conceptual (EN/HI/SA) - v variations each for lang, templates in [("en", CONCEPTUAL_VARIATIONS_EN), ("hi", CONCEPTUAL_VARIATIONS_HI), ("sa", CONCEPTUAL_VARIATIONS_SA)]: for i in range(v): template = templates[i % len(templates)] q = template.format(speaker=concepts['speaker'], topic=concepts['topic'], section=shloka.section_name if lang == "en" else (shloka.section_name_hi if lang == "hi" else shloka.section_name_sa)) answer = self._build_answer_en(shloka, "conceptual") if lang == "en" else (self._build_answer_hi(shloka, "conceptual") if lang == "hi" else self._build_answer_sa(shloka, "conceptual")) conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"natural_conceptual_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # Analytical (EN/HI/SA) for lang, templates in [("en", ANALYTICAL_VARIATIONS_EN), ("hi", ANALYTICAL_VARIATIONS_HI), ("sa", ANALYTICAL_VARIATIONS_SA)]: for i in range(v): template = templates[i % len(templates)] q = template.format(speaker=concepts['speaker'], topic=concepts['topic'], section=shloka.section_name if lang == "en" else (shloka.section_name_hi if lang == "hi" else shloka.section_name_sa), theme=concepts['section_theme']) answer = self._build_answer_en(shloka, "analytical") if lang == "en" else (self._build_answer_hi(shloka, "analytical") if lang == "hi" else self._build_answer_sa(shloka, "analytical")) conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"natural_analytical_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # Application (EN/HI/SA) applications = ["clinical practice", "therapeutic intervention", "preventive care", "diagnostic assessment", "treatment planning"] for lang, templates in [("en", APPLICATION_VARIATIONS_EN), ("hi", APPLICATION_VARIATIONS_HI), ("sa", APPLICATION_VARIATIONS_SA)]: for i in range(v): template = templates[i % len(templates)] app = applications[i % len(applications)] q = template.format(speaker=concepts['speaker'], topic=concepts['topic'], app=app) answer = self._build_answer_en(shloka, "application") if lang == "en" else (self._build_answer_hi(shloka, "application") if lang == "hi" else self._build_answer_sa(shloka, "application")) conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"natural_application_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # === REFERENCE QUESTIONS === # Translation (EN/HI/SA) for lang, templates in [("en", REFERENCE_VARIATIONS_EN), ("hi", REFERENCE_VARIATIONS_HI), ("sa", REFERENCE_VARIATIONS_SA)]: for i in range(v): template = templates[i % len(templates)] q = template.format(sanskrit=shloka.sanskrit[:50]) conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"reference_translation_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": self._build_reference_answer(shloka, "translation")} ], "metadata": self._build_metadata(shloka) }) # ID Lookup (EN/HI/SA) for lang, templates in [("en", ID_LOOKUP_VARIATIONS_EN), ("hi", ID_LOOKUP_VARIATIONS_HI), ("sa", ID_LOOKUP_VARIATIONS_SA)]: for i in range(v): template = templates[i % len(templates)] q = template.format(id=shloka.shloka_id) conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"reference_id_lookup_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": self._build_reference_answer(shloka, "complete")} ], "metadata": self._build_metadata(shloka) }) # Complete Details (EN/HI/SA) complete_phrases_en = ["Give me complete details for", "Tell me everything about", "Full details of", "Comprehensive information on", "Complete analysis of"] complete_phrases_hi = ["मुझे पूरी जानकारी दीजिए", "इसके बारे में सब बताइए", "विस्तृत विवरण", "सम्पूर्ण जानकारी", "विस्तृत विश्लेषण"] complete_phrases_sa = ["सम्पूर्णविवरणं ददातु", "सर्वं वदतु", "विस्तृतं वर्णयतु", "सम्पूर्णं ज्ञानं ददातु", "विस्तृत विश्लेषणं"] for lang, phrases in [("en", complete_phrases_en), ("hi", complete_phrases_hi), ("sa", complete_phrases_sa)]: for i in range(v): q = f"{phrases[i % len(phrases)]} {shloka.shloka_id}" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"reference_complete_details_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": self._build_reference_answer(shloka, "complete")} ], "metadata": self._build_metadata(shloka) }) # Chapter Context (EN/HI/SA) chapter_phrases_en = ["Which chapter is", "Where is", "From which chapter comes", "What is the chapter location of", "Which section contains"] chapter_phrases_hi = ["किस अध्याय से है", "कहाँ स्थित है", "किस अध्याय का है", "किस अध्याय में है", "किस भाग में है"] chapter_phrases_sa = ["कस्मिन् अध्याये", "कुत्र स्थितम्", "कस्य अध्यायस्य", "कस्मिन् भागे", "कुत्र विद्यते"] for lang, phrases in [("en", chapter_phrases_en), ("hi", chapter_phrases_hi), ("sa", chapter_phrases_sa)]: for i in range(v): q = f"{phrases[i % len(phrases)]} {shloka.shloka_id}?" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"reference_chapter_context_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": self._build_chapter_answer(shloka, lang)} ], "metadata": self._build_metadata(shloka) }) # Transliteration (EN/HI/SA) translit_phrases_en = ["Give me the transliteration of", "How do I read", "Romanized text of", "IAST form of", "Pronunciation guide for"] translit_phrases_hi = ["इसका लिप्यन्तरण दीजिए", "इसे कैसे पढ़ें", "रोमन लिपि में", "आईएएसटी रूप", "उच्चारण मार्गदर्शन"] translit_phrases_sa = ["लिप्यन्तरं ददातु", "कथं पठनीयम्", "रोमनलिपौ", "आईएएसटी रूपेण", "उच्चारणमार्गदर्शनम्"] for lang, phrases in [("en", translit_phrases_en), ("hi", translit_phrases_hi), ("sa", translit_phrases_sa)]: for i in range(v): q = f"{phrases[i % len(phrases)]}: {shloka.sanskrit[:40]}" answer = f"**IAST Transliteration:** {shloka.transliteration}\n\n**Sanskrit:** {shloka.sanskrit}\n\n**Source:** {shloka.section_name}, Chapter {shloka.chapter}, Verse {shloka.verse}" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"reference_transliteration_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # Multi-turn (5 variations) multi_turn_templates = [ ["What is this verse about?", "Can you give me the Sanskrit?", "And the location?"], ["Explain this teaching", "What is the original text?", "Where is it from?"], ["Tell me about this", "Show me the Sanskrit", "Which chapter?"], ["What does this mean?", "What is the source text?", "Give me the reference"], ["Help me understand this", "Original Sanskrit please", "Location details?"] ] for i in range(v): template = multi_turn_templates[i % len(multi_turn_templates)] conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"reference_multi_turn_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": template[0]}, {"role": "model", "content": shloka.translation_en}, {"role": "user", "content": template[1]}, {"role": "model", "content": f"**Sanskrit:** {shloka.sanskrit}\n\n**Transliteration:** {shloka.transliteration}"}, {"role": "user", "content": template[2]}, {"role": "model", "content": f"**{shloka.section_name}**, Chapter {shloka.chapter}, Verse {shloka.verse}"} ], "metadata": self._build_metadata(shloka) }) # === ADVANCED QUESTIONS (5 variations each) === clinical_cases = [ f"A patient presents with symptoms described in {shloka.section_name}. Based on this teaching ({shloka.shloka_id}), what would be the Ayurvedic assessment and management?", f"How would Charaka approach a case matching the description in {shloka.shloka_id} from {shloka.section_name}?", f"Clinical scenario: Patient exhibits signs related to {concepts['topic']}. Apply {shloka.shloka_id}.", f"Using {shloka.shloka_id}, outline a treatment protocol for {concepts['topic']}-related condition.", f"Case study based on {shloka.section_name} Chapter {shloka.chapter}: Patient with {concepts['topic']} issues. Guidance?" ] for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"advanced_clinical_case_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": clinical_cases[i % len(clinical_cases)]}, {"role": "model", "content": self._build_answer_en(shloka, "application")} ], "metadata": self._build_metadata(shloka) }) # Word Analysis word_templates = [ "Analyze the key terms in: {sanskrit}", "Break down the Sanskrit words: {sanskrit}", "Padacchedha and word meanings for: {sanskrit}", "Etymological analysis of: {sanskrit}", "Word-by-word meaning of: {sanskrit}" ] for i in range(v): q = word_templates[i % len(word_templates)].format(sanskrit=shloka.sanskrit[:50]) answer = f"**Word Analysis:** {shloka.sanskrit}\n\n**Key Terms:**\n- Original text with grammatical analysis\n- Root words and derivations\n- Technical terminology explained\n\n**Source:** {shloka.section_name}, Chapter {shloka.chapter}" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"advanced_word_analysis_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # Historical Context hist_templates = [ f"What historical background informs {concepts['speaker']}'s teaching on {concepts['topic']}?", f"Philosophical context of {shloka.section_name} Chapter {shloka.chapter}:", f"Traditional scholarship on {shloka.shloka_id}:", f"How do classical commentators interpret this teaching?", f"Historical significance of this {concepts['topic']} exposition:" ] for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"advanced_historical_context_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": hist_templates[i % len(hist_templates)]}, {"role": "model", "content": self._build_answer_en(shloka, "analytical")} ], "metadata": self._build_metadata(shloka) }) # Comparative comp_templates = [ "Compare this teaching with similar concepts in other Ayurvedic texts", "How does this relate to Sushruta Samhita's approach?", "Cross-reference with Ashtanga Hridaya:", "Comparative analysis with Sangraha texts:", "How does this teaching align with Bhela Samhita?" ] for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"advanced_comparative_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": comp_templates[i % len(comp_templates)]}, {"role": "model", "content": self._build_answer_en(shloka, "analytical")} ], "metadata": self._build_metadata(shloka) }) # === CROSS-REFERENCE QUESTIONS === cross_chapter_templates = [ "How does this connect to other verses in this chapter?", "What other teachings in Charaka Samhita relate to this?", "Cross-chapter connections for this teaching:", "Related concepts elsewhere in the text:", "What builds upon this teaching in later chapters?" ] for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"cross_reference_intra_chapter_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": cross_chapter_templates[i % len(cross_chapter_templates)]}, {"role": "model", "content": self._build_answer_en(shloka, "analytical")} ], "metadata": self._build_metadata(shloka) }) # Section Relationship section_rel_templates = [ "How does this section relate to others in Charaka Samhita?", "Position of this teaching within the eight sections:", "Why is this in {section} rather than another section?", "Sectional coherence: How does this fit?", "Relationship between this and other sthanas:" ] for i in range(v): q = section_rel_templates[i % len(section_rel_templates)].format(section=shloka.section_name) conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"cross_section_relationship_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": q}, {"role": "model", "content": self._build_answer_en(shloka, "analytical")} ], "metadata": self._build_metadata(shloka) }) # === QnA VARIATIONS === # Meaning (EN/HI/SA) for lang, templates in [("en", MEANING_VARIATIONS_EN), ("hi", MEANING_VARIATIONS_HI), ("sa", MEANING_VARIATIONS_SA)]: for i in range(v): template = templates[i % len(templates)] conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_meaning_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": template}, {"role": "model", "content": self._build_answer_en(shloka, "conceptual") if lang == "en" else (self._build_answer_hi(shloka, "conceptual") if lang == "hi" else self._build_answer_sa(shloka, "conceptual"))} ], "metadata": self._build_metadata(shloka) }) # Concept QnA (EN/HI/SA) concept_templates_en = [f"What is {concepts['topic']}?", f"Explain {concepts['topic']}", f"Define {concepts['topic']}", f"Describe {concepts['topic']}", f"Clarify {concepts['topic']}"] concept_templates_hi = [f"{concepts['topic']} क्या है?", f"{concepts['topic']} समझाइए", f"{concepts['topic']} की परिभाषा", f"{concepts['topic']} का वर्णन", f"{concepts['topic']} स्पष्ट करें"] concept_templates_sa = [f"{concepts['topic']} किम्?", f"{concepts['topic']} व्याख्यातु", f"{concepts['topic']} परिभाषा", f"{concepts['topic']} वर्णयतु", f"{concepts['topic']} स्पष्टीकुरुत"] for lang, templates in [("en", concept_templates_en), ("hi", concept_templates_hi), ("sa", concept_templates_sa)]: for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_concept_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": templates[i % len(templates)]}, {"role": "model", "content": self._build_answer_en(shloka, "conceptual") if lang == "en" else (self._build_answer_hi(shloka, "conceptual") if lang == "hi" else self._build_answer_sa(shloka, "conceptual"))} ], "metadata": self._build_metadata(shloka) }) # Location QnA (EN/HI/SA) location_q_en = ["Where is this from?", "What is the source?", "Location of this teaching?", "Which text?", "Source reference?"] location_q_hi = ["यह कहाँ से है?", "स्रोत क्या है?", "इस उपदेश का स्थान?", "कौन-सा ग्रन्थ?", "स्रोत सन्दर्भ?"] location_q_sa = ["कुत्र?", "स्रोतः कः?", "अस्य उपदेशस्य स्थानम्?", "कः ग्रन्थः?", "स्रोतसन्दर्भः?"] for lang, templates in [("en", location_q_en), ("hi", location_q_hi), ("sa", location_q_sa)]: for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_location_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": templates[i % len(templates)]}, {"role": "model", "content": self._build_chapter_answer(shloka, lang)} ], "metadata": self._build_metadata(shloka) }) # Context QnA (EN/HI/SA) context_q_en = ["What is the context?", "Background information?", "Setting of this teaching?", "Circumstances?", "What led to this?"] context_q_hi = ["सन्दर्भ क्या है?", "पृष्ठभूमि जानकारी?", "उपदेश की परिस्थिति?", "परिस्थितियाँ?", "इसके पहले क्या हुआ?"] context_q_sa = ["प्रसङ्गः कः?", "पृष्ठभूमिज्ञानम्?", "उपदेशस्य परिस्थितिः?", "परिस्थितयः?", "पूर्वं किं जातम्?"] for lang, templates in [("en", context_q_en), ("hi", context_q_hi), ("sa", context_q_sa)]: for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_context_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": templates[i % len(templates)]}, {"role": "model", "content": self._build_answer_en(shloka, "analytical") if lang == "en" else (self._build_answer_hi(shloka, "analytical") if lang == "hi" else self._build_answer_sa(shloka, "analytical"))} ], "metadata": self._build_metadata(shloka) }) # Short QnA (EN/HI/SA) short_q_en = ["Brief summary?", "Quick explanation?", "In brief?", "Short answer?", "TL;DR?"] short_q_hi = ["संक्षिप्त सार?", "शीघ्र व्याख्या?", "संक्षेप में?", "छोटा उत्तर?", "सारांश?"] short_q_sa = ["संक्षिप्तसारः?", "शीघ्रव्याख्या?", "संक्षेपेण?", "लघूत्तरम्?", "सारांशः?"] for lang, templates in [("en", short_q_en), ("hi", short_q_hi), ("sa", short_q_sa)]: for i in range(v): short_answer = f"{shloka.translation_en[:100]}... [{shloka.shloka_id}]" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_short_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": templates[i % len(templates)]}, {"role": "model", "content": short_answer} ], "metadata": self._build_metadata(shloka) }) # Section Info QnA (EN/HI/SA) section_q_en = [f"About {shloka.section_name}", f"What is {shloka.section_name}?", f"{shloka.section_name} content?", f"Topics in {shloka.section_name}?", f"{shloka.section_name} scope?"] section_q_hi = [f"{shloka.section_name_hi} के बारे में", f"{shloka.section_name_hi} क्या है?", f"{shloka.section_name_hi} की सामग्री?", f"{shloka.section_name_hi} के विषय?", f"{shloka.section_name_hi} का क्षेत्र?"] section_q_sa = [f"{shloka.section_name_sa} विषये", f"{shloka.section_name_sa} किम्?", f"{shloka.section_name_sa} सामग्री?", f"{shloka.section_name_sa} विषयाः?", f"{shloka.section_name_sa} क्षेत्रम्?"] for lang, templates in [("en", section_q_en), ("hi", section_q_hi), ("sa", section_q_sa)]: for i in range(v): if lang == "en": answer = f"**{shloka.section_name}** covers {shloka.section_description}. This verse ({shloka.shloka_id}) is from Chapter {shloka.chapter}." elif lang == "hi": answer = f"**{shloka.section_name_hi}** {shloka.section_description} को समाहित करता है। यह श्लोक ({shloka.shloka_id}) अध्याय {shloka.chapter} से है।" else: answer = f"**{shloka.section_name_sa}** {shloka.section_description} इति समाहितवति। एतत् श्लोकः ({shloka.shloka_id}) अध्यायः {shloka.chapter}।" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_section_info_{lang}_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt(lang, shloka.section_name)}, {"role": "user", "content": templates[i % len(templates)]}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # Analytical QnA (5 variations, EN only) analytical_q = [f"Analyze {concepts['topic']} in depth", f"Critical examination of {concepts['topic']}:", f"Detailed analysis of this teaching:", "Comprehensive evaluation:", f"Scholarly analysis of {concepts['topic']}:"] for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_analytical_en_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": analytical_q[i % len(analytical_q)]}, {"role": "model", "content": self._build_answer_en(shloka, "analytical")} ], "metadata": self._build_metadata(shloka) }) # Word-by-word (5 variations, EN only) wordbyword_q = ["Word-by-word breakdown:", "Grammatical analysis:", "Pada-vichara:", "Detailed word analysis:", "Shabd by Shabd meaning:"] for i in range(v): answer = f"**Word-by-word analysis of {shloka.shloka_id}:**\n\n**Sanskrit:** {shloka.sanskrit}\n\n**Transliteration:** {shloka.transliteration}\n\n**Word meanings:**\n- Each term analyzed grammatically\n- Compound word breakdown\n- Technical terminology explained\n\n**Source:** {shloka.section_name}, Chapter {shloka.chapter}" conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_word_by_word_en_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": wordbyword_q[i % len(wordbyword_q)]}, {"role": "model", "content": answer} ], "metadata": self._build_metadata(shloka) }) # Application QnA (5 variations, EN only) application_q = ["How to apply this?", "Practical implementation?", "Clinical application?", "Real-world usage?", "Therapeutic application:"] for i in range(v): conversations.append({ "shloka_id": shloka.shloka_id, "conversation_type": f"qna_application_en_v{i+1}", "messages": [ {"role": "system", "content": self._build_system_prompt("en", shloka.section_name)}, {"role": "user", "content": application_q[i % len(application_q)]}, {"role": "model", "content": self._build_answer_en(shloka, "application")} ], "metadata": self._build_metadata(shloka) }) return conversations def _build_metadata(self, shloka: ShlokaData) -> Dict: """Build metadata for each conversation""" return { "sanskrit": shloka.sanskrit, "transliteration": shloka.transliteration, "section": shloka.section_name, "section_hi": shloka.section_name_hi, "section_sa": shloka.section_name_sa, "section_code": shloka.section_code, "section_description": shloka.section_description, "chapter": shloka.chapter, "verse": shloka.verse, "translation_en": shloka.translation_en, "translation_hi": shloka.translation_hi } def save_conversations(self, conversations: List[Dict], output_file: str): """Save as pretty-printed JSONL""" print(f"Saving {len(conversations)} conversations to {output_file}...") with open(output_file, 'w', encoding='utf-8') as f: for conv in conversations: f.write(json.dumps(conv, ensure_ascii=False, indent=2) + '\n') print(f"Saved: {len(conversations)} conversations") # Calculate file size import os size_mb = os.path.getsize(output_file) / (1024 * 1024) print(f"File size: {size_mb:.1f} MB") def generate_dataset(self, output_file: str = "gemma4_ayurveda_56variations.jsonl"): """Generate full dataset with all variations""" print("=" * 60) print("56VARIATIONS GEMMA-4 DATASET GENERATOR") print(f"Variations per type: {self.variations_per_type}") print("=" * 60) self.load_source() all_conversations = [] total_shlokas = len(self.shlokas) print(f"\nGenerating {56 * self.variations_per_type} conversations per shloka...") print(f"Total shlokas to process: {total_shlokas}") print(f"Estimated total conversations: {total_shlokas * 56 * self.variations_per_type:,}") print() for idx, shloka in enumerate(self.shlokas): if idx % 100 == 0: print(f"Processing {idx}/{total_shlokas} shlokas...") conversations = self.generate_all_conversation_types(shloka) all_conversations.extend(conversations) print(f"\nTotal conversations generated: {len(all_conversations)}") print(f"Per shloka: ~{len(all_conversations) / total_shlokas:.0f} conversations") self.save_conversations(all_conversations, output_file) # Save statistics from collections import Counter type_counts = Counter([c['conversation_type'].rsplit('_v', 1)[0] for c in all_conversations]) print("\nTop 10 conversation types:") for ctype, count in type_counts.most_common(10): print(f" {ctype}: {count}") print("\n" + "=" * 60) print("DATASET GENERATION COMPLETE") print("=" * 60) def main(): generator = VariationsDatasetGenerator( source_file="shlokas_export_2026-04-30.json", variations_per_type=5 # 5 variations per question type ) generator.generate_dataset("gemma4_ayurveda_56variations.jsonl") if __name__ == "__main__": main()