JSONL-Komprimierung: gzip vs zstd vs Brotli

import gzip
import json

# === gzip (eingebaut) ===
# Komprimiertes JSONL schreiben
with gzip.open('data.jsonl.gz', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Komprimiertes JSONL lesen
with gzip.open('data.jsonl.gz', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === zstd (pip install pyzstd) ===
import pyzstd

# Komprimiertes JSONL schreiben
with pyzstd.open('data.jsonl.zst', 'wt', encoding='utf-8') as f:
    for record in records:
        f.write(json.dumps(record, ensure_ascii=False) + '\n')

# Komprimiertes JSONL lesen
with pyzstd.open('data.jsonl.zst', 'rt', encoding='utf-8') as f:
    for line in f:
        record = json.loads(line)

# === Brotli (pip install brotli) ===
import brotli

# Gesamte JSONL-Datei komprimieren
with open('data.jsonl', 'rb') as f:
    raw = f.read()
compressed = brotli.compress(raw, quality=6)
with open('data.jsonl.br', 'wb') as f:
    f.write(compressed)

# Dekomprimieren
with open('data.jsonl.br', 'rb') as f:
    raw = brotli.decompress(f.read())
for line in raw.decode('utf-8').splitlines():
    record = json.loads(line)

import { createReadStream, createWriteStream } from 'fs';
import { createGzip, createGunzip, createBrotliCompress,
         createBrotliDecompress } from 'zlib';
import { createInterface } from 'readline';
import { pipeline } from 'stream/promises';

// === gzip komprimieren ===
await pipeline(
  createReadStream('data.jsonl'),
  createGzip({ level: 6 }),
  createWriteStream('data.jsonl.gz')
);

// === gzip dekomprimieren & parsen ===
const gunzip = createGunzip();
const rl = createInterface({
  input: createReadStream('data.jsonl.gz').pipe(gunzip),
});
for await (const line of rl) {
  if (line.trim()) {
    const record = JSON.parse(line);
    // Datensatz verarbeiten
  }
}

// === Brotli komprimieren ===
await pipeline(
  createReadStream('data.jsonl'),
  createBrotliCompress(),
  createWriteStream('data.jsonl.br')
);

// === Brotli dekomprimieren & parsen ===
const br = createBrotliDecompress();
const rl2 = createInterface({
  input: createReadStream('data.jsonl.br').pipe(br),
});
for await (const line of rl2) {
  if (line.trim()) {
    const record = JSON.parse(line);
  }
}

# === gzip ===
# Komprimieren (behält Original standardmäßig mit -k)
gzip -k data.jsonl                # -> data.jsonl.gz
gzip -9 -k data.jsonl              # max. Komprimierung

# Dekomprimieren
gzip -d data.jsonl.gz
# oder: gunzip data.jsonl.gz

# === zstd ===
# Installieren: brew install zstd / apt install zstd
zstd data.jsonl                    # -> data.jsonl.zst
zstd -3 data.jsonl                 # Stufe 3 (Standard)
zstd --fast data.jsonl             # Schnellste Komprimierung

# Dekomprimieren
zstd -d data.jsonl.zst
# oder: unzstd data.jsonl.zst

# === Brotli ===
# Installieren: brew install brotli / apt install brotli
brotli data.jsonl                  # -> data.jsonl.br
brotli -q 6 data.jsonl             # Qualität 6
brotli -q 11 data.jsonl            # Max. Komprimierung

# Dekomprimieren
brotli -d data.jsonl.br

# === Piping mit jq ===
# Gefiltertes JSONL komprimieren
cat data.jsonl | jq -c 'select(.level == "error")' | gzip > errors.jsonl.gz

# Dekomprimieren und Zeilen zählen
zstd -dc data.jsonl.zst | wc -l