Compression JSONL : gzip vs zstd vs Brotli

import gzip
import json

# === gzip (integre) ===
# Ecrire du JSONL compresse
with gzip.open('data.jsonl.gz', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Lire du JSONL compresse
with gzip.open('data.jsonl.gz', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === zstd (pip install pyzstd) ===
import pyzstd

# Ecrire du JSONL compresse
with pyzstd.open('data.jsonl.zst', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Lire du JSONL compresse
with pyzstd.open('data.jsonl.zst', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === Brotli (pip install brotli) ===
import brotli

# Compresser un fichier JSONL entier
with open('data.jsonl', 'rb') as f:
    raw = f.read()
compressed = brotli.compress(raw, quality=6)
with open('data.jsonl.br', 'wb') as f:
    f.write(compressed)

# Decompresser
with open('data.jsonl.br', 'rb') as f:
    raw = brotli.decompress(f.read())
for line in raw.decode('utf-8').splitlines():
    record = json.loads(line)

import { createReadStream, createWriteStream } from 'fs';
import { createGzip, createGunzip, createBrotliCompress,
         createBrotliDecompress } from 'zlib';
import { createInterface } from 'readline';
import { pipeline } from 'stream/promises';

// === compression gzip ===
await pipeline(
  createReadStream('data.jsonl'),
  createGzip({ level: 6 }),
  createWriteStream('data.jsonl.gz')
);

// === decompression gzip et analyse ===
const gunzip = createGunzip();
const rl = createInterface({
  input: createReadStream('data.jsonl.gz').pipe(gunzip),
});
for await (const line of rl) {
  if (line.trim()) {
    const record = JSON.parse(line);
    // traiter l'enregistrement
  }
}

// === compression Brotli ===
await pipeline(
  createReadStream('data.jsonl'),
  createBrotliCompress(),
  createWriteStream('data.jsonl.br')
);

// === decompression Brotli et analyse ===
const br = createBrotliDecompress();
const rl2 = createInterface({
  input: createReadStream('data.jsonl.br').pipe(br),
});
for await (const line of rl2) {
  if (line.trim()) {
    const record = JSON.parse(line);
  }
}

# === gzip ===
# Compresser (garde l'original par defaut avec -k)
gzip -k data.jsonl                # -> data.jsonl.gz
gzip -9 -k data.jsonl              # compression maximale

# Decompresser
gzip -d data.jsonl.gz
# ou : gunzip data.jsonl.gz

# === zstd ===
# Installer : brew install zstd / apt install zstd
zstd data.jsonl                    # -> data.jsonl.zst
zstd -3 data.jsonl                 # niveau 3 (defaut)
zstd --fast data.jsonl             # compression la plus rapide

# Decompresser
zstd -d data.jsonl.zst
# ou : unzstd data.jsonl.zst

# === Brotli ===
# Installer : brew install brotli / apt install brotli
brotli data.jsonl                  # -> data.jsonl.br
brotli -q 6 data.jsonl             # qualite 6
brotli -q 11 data.jsonl            # compression maximale

# Decompresser
brotli -d data.jsonl.br

# === Piping avec jq ===
# Compresser du JSONL filtre
cat data.jsonl | jq -c 'select(.level == "error")' | gzip > errors.jsonl.gz

# Decompresser et compter les lignes
zstd -dc data.jsonl.zst | wc -l