Compressão JSONL: gzip vs zstd vs Brotli

import gzip
import json

# === gzip (integrado) ===
# Escreve JSONL comprimido
with gzip.open('data.jsonl.gz', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Lê JSONL comprimido
with gzip.open('data.jsonl.gz', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === zstd (pip install pyzstd) ===
import pyzstd

# Escreve JSONL comprimido
with pyzstd.open('data.jsonl.zst', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Lê JSONL comprimido
with pyzstd.open('data.jsonl.zst', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === Brotli (pip install brotli) ===
import brotli

# Comprime um arquivo JSONL inteiro
with open('data.jsonl', 'rb') as f:
    raw = f.read()
compressed = brotli.compress(raw, quality=6)
with open('data.jsonl.br', 'wb') as f:
    f.write(compressed)

# Descomprime
with open('data.jsonl.br', 'rb') as f:
    raw = brotli.decompress(f.read())
for line in raw.decode('utf-8').splitlines():
    record = json.loads(line)

import { createReadStream, createWriteStream } from 'fs';
import { createGzip, createGunzip, createBrotliCompress,
         createBrotliDecompress } from 'zlib';
import { createInterface } from 'readline';
import { pipeline } from 'stream/promises';

// === comprimir com gzip ===
await pipeline(
  createReadStream('data.jsonl'),
  createGzip({ level: 6 }),
  createWriteStream('data.jsonl.gz')
);

// === descomprimir gzip e parsear ===
const gunzip = createGunzip();
const rl = createInterface({
  input: createReadStream('data.jsonl.gz').pipe(gunzip),
});
for await (const line of rl) {
  if (line.trim()) {
    const record = JSON.parse(line);
    // processa registro
  }
}

// === comprimir com Brotli ===
await pipeline(
  createReadStream('data.jsonl'),
  createBrotliCompress(),
  createWriteStream('data.jsonl.br')
);

// === descomprimir Brotli e parsear ===
const br = createBrotliDecompress();
const rl2 = createInterface({
  input: createReadStream('data.jsonl.br').pipe(br),
});
for await (const line of rl2) {
  if (line.trim()) {
    const record = JSON.parse(line);
  }
}

# === gzip ===
# Comprimir (mantém original por padrão com -k)
gzip -k data.jsonl                # -> data.jsonl.gz
gzip -9 -k data.jsonl              # compressão máxima

# Descomprimir
gzip -d data.jsonl.gz
# ou: gunzip data.jsonl.gz

# === zstd ===
# Instalar: brew install zstd / apt install zstd
zstd data.jsonl                    # -> data.jsonl.zst
zstd -3 data.jsonl                 # nível 3 (padrão)
zstd --fast data.jsonl             # compressão mais rápida

# Descomprimir
zstd -d data.jsonl.zst
# ou: unzstd data.jsonl.zst

# === Brotli ===
# Instalar: brew install brotli / apt install brotli
brotli data.jsonl                  # -> data.jsonl.br
brotli -q 6 data.jsonl             # qualidade 6
brotli -q 11 data.jsonl            # compressão máxima

# Descomprimir
brotli -d data.jsonl.br

# === Piping com jq ===
# Comprimir JSONL filtrado
cat data.jsonl | jq -c 'select(.level == "error")' | gzip > errors.jsonl.gz

# Descomprimir e contar linhas
zstd -dc data.jsonl.zst | wc -l