JSONL vs Parquet: Choosing the Right Data Format

import pyarrow.parquet as pq
import pandas as pd

# Write a DataFrame to Parquet
df = pd.DataFrame({
    'id': [1, 2, 3],
    'name': ['Alice', 'Bob', 'Charlie'],
    'role': ['engineer', 'designer', 'manager'],
    'salary': [95000, 88000, 105000]
})
df.to_parquet('employees.parquet')

# Read specific columns (columnar advantage)
df = pq.read_table('employees.parquet', columns=['name', 'salary']).to_pandas()

import json
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
from pathlib import Path
from datetime import datetime

def jsonl_to_parquet(jsonl_dir: str, parquet_dir: str):
    """Convert accumulated JSONL files to partitioned Parquet."""
    records = []
    for jsonl_file in Path(jsonl_dir).glob('*.jsonl'):
        with open(jsonl_file, 'r') as f:
            for line in f:
                line = line.strip()
                if line:
                    records.append(json.loads(line))

    if not records:
        return

    df = pd.DataFrame(records)
    # Add partition column
    df['date'] = datetime.now().strftime('%Y-%m-%d')

    table = pa.Table.from_pandas(df)
    pq.write_to_dataset(
        table,
        root_path=parquet_dir,
        partition_cols=['date'],
        compression='zstd'
    )
    print(f'Converted {len(records)} records to Parquet')

jsonl_to_parquet('raw_events/', 'data_lake/')