AI 훈련 데이터를 위한 JSONL

{"system": "You are a medical information assistant. Provide accurate, general health information while always recommending consulting a healthcare professional.", "messages": [{"role": "user", "content": "What are the symptoms of vitamin D deficiency?"}, {"role": "assistant", "content": "Common symptoms of vitamin D deficiency include fatigue and tiredness, bone pain and muscle weakness, mood changes including depression, slow wound healing, and frequent infections. However, many people with mild deficiency may not notice symptoms. A blood test is the only way to confirm your vitamin D levels. I recommend speaking with your doctor who can order the appropriate test and recommend supplementation if needed."}]}
{"system": "You are a medical information assistant. Provide accurate, general health information while always recommending consulting a healthcare professional.", "messages": [{"role": "user", "content": "Is it safe to take ibuprofen daily?"}, {"role": "assistant", "content": "Taking ibuprofen daily for extended periods can carry risks including stomach ulcers and gastrointestinal bleeding, increased risk of heart attack and stroke, kidney problems with long-term use, and potential interactions with other medications. For occasional pain relief, short-term use at recommended doses is generally considered safe for most adults. However, if you find yourself needing daily pain relief, please consult your doctor. They can evaluate the underlying cause and recommend a safer long-term treatment plan."}]}

{"system": "You are a cooking assistant specializing in Italian cuisine.", "messages": [{"role": "user", "content": "How do I make carbonara?"}, {"role": "assistant", "content": "Authentic carbonara uses just five ingredients: spaghetti, guanciale (cured pork cheek), eggs, Pecorino Romano cheese, and black pepper. The key technique is tempering the egg mixture with the hot pasta water so it creates a creamy sauce without scrambling."}, {"role": "user", "content": "Can I use bacon instead of guanciale?"}, {"role": "assistant", "content": "While guanciale gives the most authentic flavor, pancetta is a closer substitute. Bacon works in a pinch but its smoky flavor changes the dish significantly. If using bacon, choose an unsmoked variety. Whichever you use, cut it into small pieces and render the fat slowly over medium-low heat until crispy."}]}

import json
import sys
from collections import Counter

def validate_training_data(path: str) -> dict:
    """Validate a JSONL file for AI fine-tuning."""
    stats = Counter()
    errors = []

    with open(path, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            stats['total'] += 1

            # Check valid JSON
            try:
                data = json.loads(line)
            except json.JSONDecodeError as e:
                errors.append(f'Line {line_num}: Invalid JSON - {e}')
                stats['invalid_json'] += 1
                continue

            # Check messages field exists
            if 'messages' not in data:
                errors.append(f'Line {line_num}: Missing "messages" field')
                stats['missing_messages'] += 1
                continue

            messages = data['messages']

            # Check message structure
            for i, msg in enumerate(messages):
                if 'role' not in msg:
                    errors.append(f'Line {line_num}, msg {i}: Missing "role"')
                    stats['missing_role'] += 1
                if 'content' not in msg:
                    errors.append(f'Line {line_num}, msg {i}: Missing "content"')
                    stats['missing_content'] += 1
                elif not msg['content'].strip():
                    errors.append(f'Line {line_num}, msg {i}: Empty content')
                    stats['empty_content'] += 1

            # Check has at least one assistant message
            roles = [m.get('role') for m in messages]
            if 'assistant' not in roles:
                errors.append(f'Line {line_num}: No assistant message')
                stats['no_assistant'] += 1

            stats['valid'] += 1

    return {'stats': dict(stats), 'errors': errors[:50]}

result = validate_training_data('training.jsonl')
print(f"Total: {result['stats'].get('total', 0)}")
print(f"Valid: {result['stats'].get('valid', 0)}")
if result['errors']:
    print(f"\nFirst {len(result['errors'])} errors:")
    for err in result['errors']:
        print(f'  {err}')

import json
import hashlib

def deduplicate_jsonl(input_path: str, output_path: str) -> dict:
    """Remove duplicate training examples from a JSONL file."""
    seen_hashes = set()
    total = 0
    unique = 0

    with open(input_path, 'r') as fin, open(output_path, 'w') as fout:
        for line in fin:
            line = line.strip()
            if not line:
                continue
            total += 1

            # Hash the normalized JSON to catch formatting differences
            data = json.loads(line)
            canonical = json.dumps(data, sort_keys=True)
            content_hash = hashlib.sha256(canonical.encode()).hexdigest()

            if content_hash not in seen_hashes:
                seen_hashes.add(content_hash)
                fout.write(json.dumps(data) + '\n')
                unique += 1

    duplicates = total - unique
    print(f'Total: {total}, Unique: {unique}, Removed: {duplicates}')
    return {'total': total, 'unique': unique, 'duplicates': duplicates}

deduplicate_jsonl('training.jsonl', 'training_deduped.jsonl')