Batch processing

import asyncio, os, time
from pathlib import Path

import httpx

API = "https://api.hanji.dev"
HEADERS = {"X-API-KEY": os.environ["HANJI_API_KEY"]}


async def upload(client: httpx.AsyncClient, path: Path) -> str:
    meta = (await client.post(
        f"{API}/v1/files",
        json={"filename": path.name, "size_bytes": path.stat().st_size},
    )).json()
    async with httpx.AsyncClient() as raw:
        await raw.put(
            meta["upload"]["url"],
            content=path.read_bytes(),
            headers={"Content-Type": "application/octet-stream"},
            timeout=600,
        )
    return meta["id"]


async def main(input_dir: str, output_dir: str) -> None:
    files = sorted(p for p in Path(input_dir).rglob("*.pdf") if p.is_file())
    Path(output_dir).mkdir(parents=True, exist_ok=True)

    async with httpx.AsyncClient(headers=HEADERS, timeout=60) as client:
        sem = asyncio.Semaphore(10)
        async def _bound(p): 
            async with sem: return await upload(client, p)
        file_ids = await asyncio.gather(*[_bound(p) for p in files])

        batch = (await client.post(
            f"{API}/v1/batches",
            headers={"Idempotency-Key": f"my-run-{int(time.time())}"},
            json={"source": {"type": "files", "file_ids": file_ids}},
        )).json()
        print("submitted batch", batch["id"], "with", batch["total_items"], "items")

        while True:
            state = (await client.get(f"{API}/v1/batches/{batch['id']}")).json()
            print(state["status"], state["counts"])
            if state["status"] in {"completed", "partially_failed", "failed", "cancelled", "expired"}:
                break
            await asyncio.sleep(3)

        for item in state["items"]:
            if item["status"] != "succeeded":
                continue
            r = await client.get(
                f"{API}{item['result_url']}", follow_redirects=True
            )
            (Path(output_dir) / f"{item['id']}.json").write_bytes(r.content)


asyncio.run(main("./pdfs", "./extracted"))

import { readFile, stat, mkdir, writeFile, readdir } from "node:fs/promises";
import { join } from "node:path";

const API = "https://api.hanji.dev";
const HEADERS = { "X-API-KEY": process.env.HANJI_API_KEY };

async function upload(path) {
  const size = (await stat(path)).size;
  const meta = await (await fetch(`${API}/v1/files`, {
    method: "POST",
    headers: { ...HEADERS, "Content-Type": "application/json" },
    body: JSON.stringify({ filename: path.split("/").pop(), size_bytes: size }),
  })).json();
  await fetch(meta.upload.url, { method: "PUT", body: await readFile(path) });
  return meta.id;
}

async function main(inputDir, outputDir) {
  await mkdir(outputDir, { recursive: true });
  const entries = await readdir(inputDir);
  const pdfs = entries.filter(f => f.endsWith(".pdf")).map(f => join(inputDir, f));
  const fileIds = await Promise.all(pdfs.map(upload));

  const batch = await (await fetch(`${API}/v1/batches`, {
    method: "POST",
    headers: { ...HEADERS, "Content-Type": "application/json", "Idempotency-Key": `my-run-${Date.now()}` },
    body: JSON.stringify({ source: { type: "files", file_ids: fileIds } }),
  })).json();
  console.log("submitted batch", batch.id, "with", batch.total_items, "items");

  let state = batch;
  const terminal = new Set(["completed", "partially_failed", "failed", "cancelled", "expired"]);
  while (!terminal.has(state.status)) {
    await new Promise(r => setTimeout(r, 3000));
    state = await (await fetch(`${API}/v1/batches/${batch.id}`, { headers: HEADERS })).json();
    console.log(state.status, state.counts);
  }

  for (const item of state.items) {
    if (item.status !== "succeeded") continue;
    const r = await fetch(`${API}${item.result_url}`, { headers: HEADERS, redirect: "follow" });
    await writeFile(join(outputDir, `${item.id}.json`), Buffer.from(await r.arrayBuffer()));
  }
}

main("./pdfs", "./extracted");

# 1. Reserve a slot, get a presigned PUT URL.
curl -sS -X POST https://api.hanji.dev/v1/files \
  -H "X-API-KEY: $HANJI_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"filename": "chart.pdf", "size_bytes": 18342}'
# → { "id": "file_xxx", "upload": { "url": "https://...?X-Amz-Signature=...", ... } }

# 2. Upload the file bytes to the presigned URL.
curl -X PUT --data-binary @chart.pdf "$UPLOAD_URL"

# 3. Submit the batch (one or many file_ids).
curl -sS -X POST https://api.hanji.dev/v1/batches \
  -H "X-API-KEY: $HANJI_API_KEY" \
  -H "Idempotency-Key: my-run-2026-05-07" \
  -H "Content-Type: application/json" \
  -d '{"source": {"type": "files", "file_ids": ["file_xxx"]}}'
# → { "id": "batch_yyy", "status": "pending", "total_items": 1 }

# 4. Poll until terminal.
curl -sS "https://api.hanji.dev/v1/batches/batch_yyy" -H "X-API-KEY: $HANJI_API_KEY"

# 5. Fetch a successful item's result JSON.
curl -L "https://api.hanji.dev/v1/batches/batch_yyy/items/item_zzz/result" \
  -H "X-API-KEY: $HANJI_API_KEY"