JSONL en Python : Lecture, écriture et analyse

import json

records = []
with open('data.jsonl', 'r', encoding='utf-8') as f:
    for line in f:
        line = line.strip()
        if line:  # Skip empty lines
            records.append(json.loads(line))

print(f'Loaded {len(records)} records')
print(records[0])

import pandas as pd

# Read entire file into a DataFrame
df = pd.read_json('data.jsonl', lines=True)
print(df.head())
print(f'Shape: {df.shape}')

# For large files, read in chunks
chunks = pd.read_json('large.jsonl', lines=True, chunksize=10000)
for chunk in chunks:
    # Process each chunk (DataFrame)
    print(f'Chunk shape: {chunk.shape}')

import json
from typing import Iterator, Any

def read_jsonl(path: str) -> Iterator[dict[str, Any]]:
    """Read a JSONL file lazily, yielding one record at a time."""
    with open(path, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            try:
                yield json.loads(line)
            except json.JSONDecodeError as e:
                print(f'Skipping invalid JSON at line {line_num}: {e}')

# Process records one at a time
for record in read_jsonl('large_data.jsonl'):
    process(record)  # Only one record in memory at a time