Comment gérer les gros fichiers JSONL (1 Go+)

import json

def process_large_jsonl(filepath: str) -> int:
    """Process a large JSONL file line by line."""
    count = 0
    errors = 0
    
    with open(filepath, 'r', encoding='utf-8') as f:
        for line_num, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            try:
                record = json.loads(line)
                # Process your record here
                count += 1
            except json.JSONDecodeError as e:
                errors += 1
                print(f'Line {line_num}: {e}')
    
    print(f'Processed {count} records, {errors} errors')
    return count

import { createReadStream } from 'fs';
import { createInterface } from 'readline';

async function processLargeJsonl(filepath) {
  const rl = createInterface({
    input: createReadStream(filepath, 'utf-8'),
    crlfDelay: Infinity,
  });

  let count = 0;
  for await (const line of rl) {
    const trimmed = line.trim();
    if (!trimmed) continue;
    try {
      const record = JSON.parse(trimmed);
      // Process your record here
      count++;
    } catch (err) {
      console.error(`Parse error: ${err.message}`);
    }
  }
  console.log(`Processed ${count} records`);
}

# Count lines in a JSONL file
wc -l data.jsonl

# View first 10 records
head -n 10 data.jsonl

# View last 5 records
tail -n 5 data.jsonl

# Pretty-print first record
head -n 1 data.jsonl | jq .

# Filter records with jq
jq -c 'select(.age > 30)' data.jsonl

# Extract specific fields
jq -c {name, email} data.jsonl

# Split into files of 100,000 lines each
split -l 100000 data.jsonl chunk_

# Split into files of approximately 100MB each
split -b 100m data.jsonl chunk_

# Add .jsonl extension to split files
for f in chunk_*; do mv "$f" "$f.jsonl"; done

import json

def split_jsonl(input_path: str, lines_per_file: int = 100000):
    file_num = 0
    line_count = 0
    out_file = None
    
    with open(input_path, 'r') as f:
        for line in f:
            if line_count % lines_per_file == 0:
                if out_file:
                    out_file.close()
                file_num += 1
                out_file = open(f'part_{file_num:04d}.jsonl', 'w')
            out_file.write(line)
            line_count += 1
    
    if out_file:
        out_file.close()
    print(f'Split into {file_num} files')