JSONL Schema 验证：确保数据质量

import json
from jsonschema import validate, ValidationError

def validate_jsonl(file_path, schema):
    errors = []
    with open(file_path, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, start=1):
            line = line.strip()
            if not line:
                continue
            try:
                record = json.loads(line)
                validate(instance=record, schema=schema)
            except json.JSONDecodeError as e:
                errors.append({
                    'line': line_num,
                    'error': f'Invalid JSON: {e.msg}'
                })
            except ValidationError as e:
                errors.append({
                    'line': line_num,
                    'error': e.message,
                    'path': list(e.absolute_path)
                })
    return errors

# Load schema and validate
with open('schema.json', 'r') as f:
    schema = json.load(f)

errors = validate_jsonl('data.jsonl', schema)
if errors:
    print(f'Found {len(errors)} validation errors:')
    for err in errors:
        print(f"  Line {err['line']}: {err['error']}")
else:
    print('All records are valid!')

guide-jsonl-schema-validation.jsonlSchemaValidation.methods.nodejs.code

# Install ajv-cli globally
npm install -g ajv-cli ajv-formats

# Validate each line of a JSONL file against a schema
line_num=0
errors=0
while IFS= read -r line; do
  line_num=$((line_num + 1))
  if [ -z "$line" ]; then continue; fi
  echo "$line" | ajv validate -s schema.json -d /dev/stdin --all-errors 2>/dev/null
  if [ $? -ne 0 ]; then
    echo "  Error on line $line_num"
    errors=$((errors + 1))
  fi
done < data.jsonl

if [ $errors -eq 0 ]; then
  echo "All records are valid!"
else
  echo "Found $errors invalid records"
  exit 1
fi