Long-running calls

import osimport timeimport uuidimport requestsAPI = "https://data.upscrape.com"HEADERS = {"Authorization": f"Bearer {os.environ['UPSCRAPE_API_KEY']}"}def run(capability, input, deadline_seconds=300):    # One key per piece of work: a retry after a lost answer gets the same job.    key = str(uuid.uuid4())    response = requests.post(        f"{API}/execute",        headers={**HEADERS, "Prefer": "wait=30", "Idempotency-Key": key},        json={"capability": capability, "input": input},    )    job = response.json()    if "error" in job and "job_id" not in job:        raise RuntimeError(job["error"]["code"])    deadline = time.monotonic() + deadline_seconds    delay = job.get("poll_after_ms", 2000) / 1000    while job["state"] not in ("completed", "failed"):        if time.monotonic() > deadline:            raise TimeoutError(f"job {job['job_id']} is still {job['state']}")        time.sleep(delay)        delay = min(delay * 2, 10)        job = requests.get(job["poll_url"], headers=HEADERS).json()    if job["state"] == "failed":        raise RuntimeError(job["error"]["code"])    return job["results"][0]["data"]data = run("web.crawl", {"url": "https://docs.stripe.com/payments", "max_pages": 10})print(len(data["pages"]), "pages")

import { randomUUID } from "node:crypto";const API = "https://data.upscrape.com";const headers = { Authorization: `Bearer ${process.env.UPSCRAPE_API_KEY}` };const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));async function run(capability, input, deadlineMs = 300_000) {  // One key per piece of work: a retry after a lost answer gets the same job.  const response = await fetch(`${API}/execute`, {    method: "POST",    headers: {      ...headers,      "Content-Type": "application/json",      Prefer: "wait=30",      "Idempotency-Key": randomUUID(),    },    body: JSON.stringify({ capability, input }),  });  let job = await response.json();  if (job.error && !job.job_id) throw new Error(job.error.code);  const deadline = Date.now() + deadlineMs;  let delay = job.poll_after_ms ?? 2000;  while (job.state !== "completed" && job.state !== "failed") {    if (Date.now() > deadline) throw new Error(`job ${job.job_id} is still ${job.state}`);    await sleep(delay);    delay = Math.min(delay * 2, 10_000);    job = await (await fetch(job.poll_url, { headers })).json();  }  if (job.state === "failed") throw new Error(job.error.code);  return job.results[0].data;}const data = await run("web.crawl", { url: "https://docs.stripe.com/payments", max_pages: 10 });console.log(data.pages.length, "pages");