JSONL-compressie: gzip vs zstd vs Brotli

import gzip
import json

# === gzip (ingebouwd) ===
# Schrijf gecomprimeerde JSONL
with gzip.open('data.jsonl.gz', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Lees gecomprimeerde JSONL
with gzip.open('data.jsonl.gz', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === zstd (pip install pyzstd) ===
import pyzstd

# Schrijf gecomprimeerde JSONL
with pyzstd.open('data.jsonl.zst', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Lees gecomprimeerde JSONL
with pyzstd.open('data.jsonl.zst', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === Brotli (pip install brotli) ===
import brotli

# Comprimeer een heel JSONL-bestand
with open('data.jsonl', 'rb') as f:
    raw = f.read()
compressed = brotli.compress(raw, quality=6)
with open('data.jsonl.br', 'wb') as f:
    f.write(compressed)

# Decomprimeer
with open('data.jsonl.br', 'rb') as f:
    raw = brotli.decompress(f.read())
for line in raw.decode('utf-8').splitlines():
    record = json.loads(line)

import { createReadStream, createWriteStream } from 'fs';
import { createGzip, createGunzip, createBrotliCompress,
         createBrotliDecompress } from 'zlib';
import { createInterface } from 'readline';
import { pipeline } from 'stream/promises';

// === gzip comprimeren ===
await pipeline(
  createReadStream('data.jsonl'),
  createGzip({ level: 6 }),
  createWriteStream('data.jsonl.gz')
);

// === gzip decomprimeren & parseren ===
const gunzip = createGunzip();
const rl = createInterface({
  input: createReadStream('data.jsonl.gz').pipe(gunzip),
});
for await (const line of rl) {
  if (line.trim()) {
    const record = JSON.parse(line);
    // verwerk record
  }
}

// === Brotli comprimeren ===
await pipeline(
  createReadStream('data.jsonl'),
  createBrotliCompress(),
  createWriteStream('data.jsonl.br')
);

// === Brotli decomprimeren & parseren ===
const br = createBrotliDecompress();
const rl2 = createInterface({
  input: createReadStream('data.jsonl.br').pipe(br),
});
for await (const line of rl2) {
  if (line.trim()) {
    const record = JSON.parse(line);
  }
}

# === gzip ===
# Comprimeren (behoudt origineel standaard met -k)
gzip -k data.jsonl                # -> data.jsonl.gz
gzip -9 -k data.jsonl              # maximale compressie

# Decomprimeren
gzip -d data.jsonl.gz
# of: gunzip data.jsonl.gz

# === zstd ===
# Installeren: brew install zstd / apt install zstd
zstd data.jsonl                    # -> data.jsonl.zst
zstd -3 data.jsonl                 # niveau 3 (standaard)
zstd --fast data.jsonl             # snelste compressie

# Decomprimeren
zstd -d data.jsonl.zst
# of: unzstd data.jsonl.zst

# === Brotli ===
# Installeren: brew install brotli / apt install brotli
brotli data.jsonl                  # -> data.jsonl.br
brotli -q 6 data.jsonl             # kwaliteit 6
brotli -q 11 data.jsonl            # maximale compressie

# Decomprimeren
brotli -d data.jsonl.br

# === Piping met jq ===
# Comprimeer gefilterde JSONL
cat data.jsonl | jq -c 'select(.level == "error")' | gzip > errors.jsonl.gz

# Decomprimeer en tel regels
zstd -dc data.jsonl.zst | wc -l