Kompresja JSONL: gzip vs zstd vs Brotli

import gzip
import json

# === gzip (wbudowany) ===
# Zapis skompresowanego JSONL
with gzip.open('data.jsonl.gz', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Odczyt skompresowanego JSONL
with gzip.open('data.jsonl.gz', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === zstd (pip install pyzstd) ===
import pyzstd

# Zapis skompresowanego JSONL
with pyzstd.open('data.jsonl.zst', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Odczyt skompresowanego JSONL
with pyzstd.open('data.jsonl.zst', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === Brotli (pip install brotli) ===
import brotli

# Kompresja calego pliku JSONL
with open('data.jsonl', 'rb') as f:
    raw = f.read()
compressed = brotli.compress(raw, quality=6)
with open('data.jsonl.br', 'wb') as f:
    f.write(compressed)

# Dekompresja
with open('data.jsonl.br', 'rb') as f:
    raw = brotli.decompress(f.read())
for line in raw.decode('utf-8').splitlines():
    record = json.loads(line)

import { createReadStream, createWriteStream } from 'fs';
import { createGzip, createGunzip, createBrotliCompress,
         createBrotliDecompress } from 'zlib';
import { createInterface } from 'readline';
import { pipeline } from 'stream/promises';

// === kompresja gzip ===
await pipeline(
  createReadStream('data.jsonl'),
  createGzip({ level: 6 }),
  createWriteStream('data.jsonl.gz')
);

// === dekompresja gzip i parsowanie ===
const gunzip = createGunzip();
const rl = createInterface({
  input: createReadStream('data.jsonl.gz').pipe(gunzip),
});
for await (const line of rl) {
  if (line.trim()) {
    const record = JSON.parse(line);
    // przetwarzaj rekord
  }
}

// === kompresja Brotli ===
await pipeline(
  createReadStream('data.jsonl'),
  createBrotliCompress(),
  createWriteStream('data.jsonl.br')
);

// === dekompresja Brotli i parsowanie ===
const br = createBrotliDecompress();
const rl2 = createInterface({
  input: createReadStream('data.jsonl.br').pipe(br),
});
for await (const line of rl2) {
  if (line.trim()) {
    const record = JSON.parse(line);
  }
}

# === gzip ===
# Kompresja (domyslnie zachowuje oryginal z -k)
gzip -k data.jsonl                # -> data.jsonl.gz
gzip -9 -k data.jsonl              # maksymalna kompresja

# Dekompresja
gzip -d data.jsonl.gz
# lub: gunzip data.jsonl.gz

# === zstd ===
# Instalacja: brew install zstd / apt install zstd
zstd data.jsonl                    # -> data.jsonl.zst
zstd -3 data.jsonl                 # poziom 3 (domyslny)
zstd --fast data.jsonl             # najszybsza kompresja

# Dekompresja
zstd -d data.jsonl.zst
# lub: unzstd data.jsonl.zst

# === Brotli ===
# Instalacja: brew install brotli / apt install brotli
brotli data.jsonl                  # -> data.jsonl.br
brotli -q 6 data.jsonl             # jakosc 6
brotli -q 11 data.jsonl            # maksymalna kompresja

# Dekompresja
brotli -d data.jsonl.br

# === Potokowanie z jq ===
# Kompresja przefiltrowanego JSONL
cat data.jsonl | jq -c 'select(.level == "error")' | gzip > errors.jsonl.gz

# Dekompresja i liczenie linii
zstd -dc data.jsonl.zst | wc -l