JSONL Best Practices

# Valid JSONL - one complete JSON per line
{"id":1,"name":"Alice","tags":["admin","user"]}
{"id":2,"name":"Bob","tags":["user"]}

# INVALID - pretty-printed JSON spans multiple lines
{
  "id": 1,
  "name": "Alice"
}

# Python: always specify UTF-8 when reading/writing
with open('data.jsonl', 'w', encoding='utf-8') as f:
    f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Node.js: UTF-8 is the default for fs
fs.appendFileSync('data.jsonl', JSON.stringify(record) + '\n', 'utf-8');

# Correct: LF line endings (\n)
{"id":1}\n{"id":2}\n

# Avoid: CRLF line endings (\r\n)
{"id":1}\r\n{"id":2}\r\n

# Tip: configure Git to normalize line endings
# .gitattributes
*.jsonl text eol=lf

import json

def parse_jsonl_safe(path: str):
    """Parse JSONL with error tolerance."""
    valid, errors = [], []
    with open(path, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            try:
                valid.append(json.loads(line))
            except json.JSONDecodeError as e:
                errors.append({'line': line_num, 'error': str(e), 'raw': line})
    print(f'Parsed {len(valid)} records, {len(errors)} errors')
    return valid, errors

def validate_record(record: dict) -> list[str]:
    """Validate a JSONL record against expected schema."""
    issues = []
    required = ['id', 'name', 'timestamp']
    for field in required:
        if field not in record:
            issues.append(f'Missing required field: {field}')
    if 'id' in record and not isinstance(record['id'], int):
        issues.append(f'Field "id" should be int, got {type(record["id"]).__name__}')
    return issues

# Usage in pipeline
for record in parse_jsonl_safe('data.jsonl')[0]:
    issues = validate_record(record)
    if issues:
        log_warning(f'Record {record.get("id")}: {issues}')
    else:
        process(record)

# INVALID: trailing comma after last property
{"id": 1, "name": "Alice",}

# VALID: no trailing comma
{"id": 1, "name": "Alice"}

# INVALID: trailing comma in array
{"tags": ["admin", "user",]}

# VALID: no trailing comma in array
{"tags": ["admin", "user"]}

# INVALID: raw newline inside a string value breaks JSONL
{"id": 1, "bio": "Line one
Line two"}

# VALID: escaped newline keeps everything on one line
{"id": 1, "bio": "Line one\nLine two"}

# Tip: json.dumps() in Python handles this automatically
import json
record = {"bio": "Line one\nLine two"}
print(json.dumps(record))
# Output: {"bio": "Line one\nLine two"}

# Python: detect and convert encoding
import chardet

def normalize_to_utf8(input_path: str, output_path: str):
    """Detect encoding and convert to UTF-8."""
    with open(input_path, 'rb') as f:
        raw = f.read()
    detected = chardet.detect(raw)
    encoding = detected['encoding'] or 'utf-8'
    print(f'Detected encoding: {encoding}')
    text = raw.decode(encoding)
    with open(output_path, 'w', encoding='utf-8') as f:
        f.write(text)