#!/usr/bin/env python3
"""
Create comprehensive documentation of all 56 question types with full examples
Extracting from the actual 56variations dataset
"""

import json
from pathlib import Path

def extract_one_full_example_per_type(filepath):
    """Extract one full example (with v1) for each question type"""
    print(f"Scanning {filepath}...")

    with open(filepath, 'r', encoding='utf-8') as f:
        content = f.read()

    # Parse all objects
    objects = content.split('\n}\n{')

    examples = {}
    print(f"Total objects to parse: {len(objects)}")

    for i, obj in enumerate(objects):
        if i == 0:
            obj = obj + '\n}'
        else:
            obj = '{' + obj + '\n}'

        try:
            data = json.loads(obj)
            conv_type = data.get('conversation_type', '')

            # Only take v1 examples (first variation)
            if conv_type.endswith('_v1'):
                examples[conv_type] = data

        except Exception as e:
            continue

        if i % 100000 == 0 and i > 0:
            print(f"  Processed {i} objects, found {len(examples)} _v1 types...")

    print(f"\nTotal _v1 examples found: {len(examples)}")
    return examples

def create_comprehensive_markdown(examples, output_file="56_TYPES_FULL_EXAMPLES.md"):
    """Create comprehensive markdown with all examples"""

    # Get metadata from first example
    first_key = list(examples.keys())[0]
    first = examples[first_key]
    metadata = first.get('metadata', {})

    md = f"""# 56 Question Types - Complete Examples Documentation

**Dataset:** `gemma4_ayurveda_56variations.jsonl`
**Format:** ShareGPT (system/user/model)
**Generated:** 2024-05-14

---

## Source Shloka Information

| Field | Value |
|-------|-------|
| **Shloka ID** | {first.get('shloka_id', 'N/A')} |
| **Sanskrit** | `{metadata.get('sanskrit', 'N/A')}` |
| **Transliteration** | `{metadata.get('transliteration', 'N/A')}` |
| **Translation (EN)** | {metadata.get('translation_en', 'N/A')} |
| **Section** | {metadata.get('section', 'N/A')} |
| **Chapter** | {metadata.get('chapter', 'N/A')} |
| **Verse** | {metadata.get('verse', 'N/A')} |

---

## Understanding the Format

Each question type has **5 variations** (v1, v2, v3, v4, v5) with different phrasings:
- **Natural questions:** Treat content as knowledge, NOT text (no "shloka", "verse" references)
- **Reference questions:** Include text references for scholarly use
- **Advanced:** Clinical, analytical, historical contexts
- **Cross-reference:** Connections between sections/chapters
- **QnA:** Quick question-answer formats

---

## Category 1: Natural Conceptual Questions (15 types)

Questions about conceptual understanding - what does this teaching mean?

### 1.1 English (5 variations)

"""

    # Natural Conceptual EN
    for i in range(1, 6):
        key = f"natural_conceptual_en_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**System:** {msgs[0]['content'][:100]}...

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 1.2 Hindi (5 variations)

"""

    # Natural Conceptual HI
    for i in range(1, 6):
        key = f"natural_conceptual_hi_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**System:** {msgs[0]['content'][:100]}...

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 1.3 Sanskrit (5 variations)

"""

    # Natural Conceptual SA
    for i in range(1, 6):
        key = f"natural_conceptual_sa_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**System:** {msgs[0]['content'][:100]}...

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Natural Analytical
    md += """---

## Category 2: Natural Analytical Questions (15 types)

Questions about analysis and relationships - how does this connect?

### 2.1 English (5 variations)

"""

    for i in range(1, 6):
        key = f"natural_analytical_en_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 2.2 Hindi (5 variations)

"""

    for i in range(1, 6):
        key = f"natural_analytical_hi_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 2.3 Sanskrit (5 variations)

"""

    for i in range(1, 6):
        key = f"natural_analytical_sa_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Natural Application
    md += """---

## Category 3: Natural Application Questions (15 types)

Questions about practical use - how do I apply this?

### 3.1 English (5 variations)

"""

    for i in range(1, 6):
        key = f"natural_application_en_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 3.2 Hindi (5 variations)

"""

    for i in range(1, 6):
        key = f"natural_application_hi_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 3.3 Sanskrit (5 variations)

"""

    for i in range(1, 6):
        key = f"natural_application_sa_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Reference Translation
    md += """---

## Category 4: Reference Questions (40 types)

Questions that reference specific text - for scholarly/academic use.

### 4.1 Translation (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"reference_translation_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Reference ID Lookup
    md += """### 4.2 ID Lookup (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"reference_id_lookup_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Reference Complete Details
    md += """### 4.3 Complete Details (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"reference_complete_details_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Reference Chapter Context
    md += """### 4.4 Chapter Context (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"reference_chapter_context_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Reference Transliteration
    md += """### 4.5 Transliteration (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"reference_transliteration_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Reference Multi-turn
    md += """### 4.6 Multi-turn (5 types)

Multi-turn conversations with follow-up questions.

"""

    for i in range(1, 6):
        key = f"reference_multi_turn_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

"""
            # Show all messages
            for msg in msgs[1:]:  # Skip system
                md += f"**{msg['role'].upper()}:** {msg['content'][:200]}...\n\n"
            md += "---\n\n"

    # Advanced
    md += """---

## Category 5: Advanced Questions (20 types)

Deep analytical and clinical questions.

### 5.1 Clinical Case (5 types)

"""

    for i in range(1, 6):
        key = f"advanced_clinical_case_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 5.2 Word Analysis (5 types)

"""

    for i in range(1, 6):
        key = f"advanced_word_analysis_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 5.3 Historical Context (5 types)

"""

    for i in range(1, 6):
        key = f"advanced_historical_context_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 5.4 Comparative (5 types)

"""

    for i in range(1, 6):
        key = f"advanced_comparative_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Cross-Reference
    md += """---

## Category 6: Cross-Reference Questions (10 types)

Questions about relationships between sections.

### 6.1 Intra-chapter (5 types)

"""

    for i in range(1, 6):
        key = f"cross_reference_intra_chapter_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 6.2 Section Relationship (5 types)

"""

    for i in range(1, 6):
        key = f"cross_section_relationship_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # QnA Variations
    md += """---

## Category 7: QnA Variations (45 types)

Quick question-answer formats.

### 7.1 Meaning (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"qna_meaning_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.2 Concept (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"qna_concept_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.3 Location (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"qna_location_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.4 Context (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"qna_context_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.5 Short (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"qna_short_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.6 Section Info (3 languages × 5 = 15 types)

"""

    for lang in ['en', 'hi', 'sa']:
        md += f"**{lang.upper()}**\n\n"
        for i in range(1, 6):
            key = f"qna_section_info_{lang}_v{i}"
            if key in examples:
                ex = examples[key]
                msgs = ex['messages']
                md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.7 Analytical (5 types)

"""

    for i in range(1, 6):
        key = f"qna_analytical_en_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.8 Word-by-word (5 types)

"""

    for i in range(1, 6):
        key = f"qna_word_by_word_en_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    md += """### 7.9 Application (5 types)

"""

    for i in range(1, 6):
        key = f"qna_application_en_v{i}"
        if key in examples:
            ex = examples[key]
            msgs = ex['messages']
            md += f"""
#### `{key}`

**User:** {msgs[1]['content']}

**Model:** {msgs[2]['content'][:300]}...

---

"""

    # Summary
    md += f"""---

## Summary Statistics

| Category | Base Types | Variations | Total per Shloka |
|----------|-----------|-----------|------------------|
| Natural Conceptual | 3 (EN/HI/SA) | 5 each | 15 |
| Natural Analytical | 3 (EN/HI/SA) | 5 each | 15 |
| Natural Application | 3 (EN/HI/SA) | 5 each | 15 |
| Reference Translation | 3 (EN/HI/SA) | 5 each | 15 |
| Reference ID Lookup | 3 (EN/HI/SA) | 5 each | 15 |
| Reference Complete Details | 3 (EN/HI/SA) | 5 each | 15 |
| Reference Chapter Context | 3 (EN/HI/SA) | 5 each | 15 |
| Reference Transliteration | 3 (EN/HI/SA) | 5 each | 15 |
| Reference Multi-turn | 1 | 5 | 5 |
| Advanced Clinical | 1 | 5 | 5 |
| Advanced Word Analysis | 1 | 5 | 5 |
| Advanced Historical | 1 | 5 | 5 |
| Advanced Comparative | 1 | 5 | 5 |
| Cross Intra-chapter | 1 | 5 | 5 |
| Cross Section Rel | 1 | 5 | 5 |
| QnA Meaning | 3 (EN/HI/SA) | 5 each | 15 |
| QnA Concept | 3 (EN/HI/SA) | 5 each | 15 |
| QnA Location | 3 (EN/HI/SA) | 5 each | 15 |
| QnA Context | 3 (EN/HI/SA) | 5 each | 15 |
| QnA Short | 3 (EN/HI/SA) | 5 each | 15 |
| QnA Section Info | 3 (EN/HI/SA) | 5 each | 15 |
| QnA Analytical | 1 | 5 | 5 |
| QnA Word-by-word | 1 | 5 | 5 |
| QnA Application | 1 | 5 | 5 |
| **TOTAL** | **52 base** | **5 each** | **~260** |

---

## File Information

- **Dataset File:** `gemma4_ayurveda_56variations.jsonl`
- **Dataset Size:** 3.6 GB
- **Total Conversations:** 894,140
- **Source Shlokas:** 3,439
- **Format:** ShareGPT (system/user/model + metadata)
- **Languages:** English, Hindi, Sanskrit

---

*Documentation generated: 2024-05-14*
"""

    with open(output_file, 'w', encoding='utf-8') as f:
        f.write(md)

    size_mb = len(md.encode('utf-8')) / 1024 / 1024
    print(f"\nSaved: {output_file}")
    print(f"Size: {size_mb:.2f} MB")
    print(f"Total _v1 examples documented: {len(examples)}")

def main():
    print("="*60)
    print("56 QUESTION TYPES DOCUMENTATION GENERATOR")
    print("="*60)

    examples = extract_one_full_example_per_type("gemma4_ayurveda_56variations.jsonl")
    create_comprehensive_markdown(examples)

    print("\n" + "="*60)
    print("COMPLETE")
    print("="*60)

if __name__ == "__main__":
    main()