OpenAI-compatible gateway reference

#!/usr/bin/env python3
"""Small OpenAI-compatible gateway for admission-control and streaming labs."""
import argparse, json, os, time, uuid, urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from threading import BoundedSemaphore, Lock

LIMIT = int(os.environ.get("MAX_CONCURRENCY", "4"))
SLOTS = BoundedSemaphore(LIMIT)
LOCK = Lock()
METRICS = {"requests": 0, "rejected": 0, "active": 0, "completed": 0}
BACKEND = os.environ.get("BACKEND_URL", "").rstrip("/")

class Handler(BaseHTTPRequestHandler):
    server_version = "InferenceFounderGateway/0.1"
    def log_message(self, fmt, *args): pass
    def send_json(self, code, obj):
        body = json.dumps(obj).encode()
        self.send_response(code); self.send_header("content-type", "application/json")
        self.send_header("content-length", str(len(body))); self.end_headers(); self.wfile.write(body)
    def do_GET(self):
        if self.path == "/healthz": return self.send_json(200, {"ok": True})
        if self.path == "/metrics":
            with LOCK: snapshot = dict(METRICS)
            return self.send_json(200, {**snapshot, "limit": LIMIT})
        self.send_json(404, {"error": "not_found"})
    def do_POST(self):
        if self.path != "/v1/chat/completions": return self.send_json(404, {"error": "not_found"})
        request_id = str(uuid.uuid4())
        length = int(self.headers.get("content-length", "0"))
        body = self.rfile.read(length)
        with LOCK: METRICS["requests"] += 1
        if not SLOTS.acquire(blocking=False):
            with LOCK: METRICS["rejected"] += 1
            return self.send_json(429, {"error": {"type": "overloaded", "request_id": request_id}})
        start = time.perf_counter()
        try:
            with LOCK: METRICS["active"] += 1
            if BACKEND:
                req = urllib.request.Request(BACKEND + "/v1/chat/completions", data=body, method="POST",
                    headers={"content-type": "application/json", "authorization": self.headers.get("authorization","")})
                with urllib.request.urlopen(req, timeout=120) as response:
                    payload = response.read(); self.send_response(response.status)
                    self.send_header("content-type", response.headers.get("content-type","application/json"))
                    self.send_header("x-request-id", request_id); self.end_headers(); self.wfile.write(payload)
            else:
                prompt = json.loads(body or "{}").get("messages", [{}])[-1].get("content", "")
                time.sleep(.035 + min(len(prompt), 1000) / 20000)
                self.send_json(200, {"id": request_id, "object": "chat.completion",
                    "choices": [{"index": 0, "message": {"role": "assistant",
                    "content": "Mock inference response. Replace BACKEND_URL with a vLLM endpoint."}, "finish_reason": "stop"}],
                    "usage": {"prompt_tokens": max(1, len(prompt)//4), "completion_tokens": 13},
                    "gateway_ms": round((time.perf_counter()-start)*1000,2)})
            with LOCK: METRICS["completed"] += 1
        finally:
            with LOCK: METRICS["active"] -= 1
            SLOTS.release()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(); parser.add_argument("--port", type=int, default=8080)
    args = parser.parse_args()
    print(f"gateway=http://127.0.0.1:{args.port} max_concurrency={LIMIT} backend={BACKEND or 'mock'}")
    ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever()