JSONLスキーマ検証：データ品質を確保する

import json
from jsonschema import validate, ValidationError

def validate_jsonl(file_path, schema):
    errors = []
    with open(file_path, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, start=1):
            line = line.strip()
            if not line:
                continue
            try:
                record = json.loads(line)
                validate(instance=record, schema=schema)
            except json.JSONDecodeError as e:
                errors.append({
                    'line': line_num,
                    'error': f'Invalid JSON: {e.msg}'
                })
            except ValidationError as e:
                errors.append({
                    'line': line_num,
                    'error': e.message,
                    'path': list(e.absolute_path)
                })
    return errors

# Load schema and validate
with open('schema.json', 'r') as f:
    schema = json.load(f)

errors = validate_jsonl('data.jsonl', schema)
if errors:
    print(f'Found {len(errors)} validation errors:')
    for err in errors:
        print(f"  Line {err['line']}: {err['error']}")
else:
    print('All records are valid!')

import { createReadStream } from 'node:fs';
import { createInterface } from 'node:readline';
import Ajv from 'ajv';
import addFormats from 'ajv-formats';
import schema from './schema.json' assert { type: 'json' };

const ajv = new Ajv({ allErrors: true });
addFormats(ajv);
const validate = ajv.compile(schema);

async function validateJsonl(filePath) {
  const errors = [];
  let lineNum = 0;

  const rl = createInterface({
    input: createReadStream(filePath, 'utf-8'),
    crlfDelay: Infinity,
  });

  for await (const line of rl) {
    lineNum++;
    const trimmed = line.trim();
    if (!trimmed) continue;

    try {
      const record = JSON.parse(trimmed);
      if (!validate(record)) {
        errors.push({
          line: lineNum,
          errors: validate.errors.map(e =>
            `${e.instancePath || '/'}: ${e.message}`
          ),
        });
      }
    } catch (e) {
      errors.push({ line: lineNum, errors: [`Invalid JSON: ${e.message}`] });
    }
  }

  return errors;
}

const errors = await validateJsonl('data.jsonl');
if (errors.length) {
  console.log(`Found ${errors.length} invalid records:`);
  errors.forEach(e =>
    console.log(`  Line ${e.line}: ${e.errors.join(', ')}`)
  );
  process.exit(1);
} else {
  console.log('All records are valid!');
}

# Install ajv-cli globally
npm install -g ajv-cli ajv-formats

# Validate each line of a JSONL file against a schema
line_num=0
errors=0
while IFS= read -r line; do
  line_num=$((line_num + 1))
  if [ -z "$line" ]; then continue; fi
  echo "$line" | ajv validate -s schema.json -d /dev/stdin --all-errors 2>/dev/null
  if [ $? -ne 0 ]; then
    echo "  Error on line $line_num"
    errors=$((errors + 1))
  fi
done < data.jsonl

if [ $errors -eq 0 ]; then
  echo "All records are valid!"
else
  echo "Found $errors invalid records"
  exit 1
fi