LLM API 호출에서 Context Length Exceeded 오류: 해결 방법, 트레이드오프, 모델 선택

{
  "error": {
    "message": "This model's maximum context length is 128000 tokens. However, your messages resulted in 142837 tokens.",
    "type": "invalid_request_error",
    "code": "context_length_exceeded"
  }
}

def sliding_window(messages: list, max_tokens: int, system_prompt: str) -> list:
    """Keep system prompt + most recent messages within token budget."""
    # Always keep system prompt
    result = [{"role": "system", "content": system_prompt}]
    token_count = count_tokens(system_prompt)

    # Add messages from newest to oldest
    for msg in reversed(messages):
        msg_tokens = count_tokens(msg["content"])
        if token_count + msg_tokens > max_tokens:
            break
        result.insert(1, msg)  # Insert after system prompt
        token_count += msg_tokens

    return result

async def summarize_context(messages: list, client) -> str:
    """Compress older messages into a summary."""
    context_text = "\n".join(
        f"{m['role']}: {m['content']}" for m in messages
    )

    response = await client.chat.completions.create(
        model="gpt-4o-mini",  # Cheaper model for summarization
        messages=[{
            "role": "user",
            "content": f"Summarize this conversation context in under 500 tokens. "
                       f"Preserve key decisions, facts, and pending actions:\n\n{context_text}"
        }],
        max_tokens=500
    )

    return response.choices[0].message.content

async def process_long_document(
    document: str,
    question: str,
    client,
    chunk_size: int = 50000
) -> str:
    """Process a long document by chunking."""
    chunks = split_into_chunks(document, chunk_size)
    chunk_results = []

    for i, chunk in enumerate(chunks):
        response = await client.chat.completions.create(
            model="gpt-4o",
            messages=[{
                "role": "user",
                "content": f"Analyze chunk {i+1}/{len(chunks)} of a document.\n"
                           f"Question: {question}\n\n"
                           f"Chunk content:\n{chunk}"
            }]
        )
        chunk_results.append(response.choices[0].message.content)

    # Merge chunk results
    merge_response = await client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": f"Merge these partial analyses into a final answer.\n"
                       f"Question: {question}\n\n"
                       f"Partial results:\n" + "\n---\n".join(chunk_results)
        }]
    )

    return merge_response.choices[0].message.content

Option A: 80K로 트렁케이트 + GPT-4o = $X
Option B: 150K 전체 입력 + Claude Sonnet = $Y
Option C: 150K 전체 입력 + Gemini Flash = $Z

비교: 품질 차이 vs 비용 차이

from openai import OpenAI

client = OpenAI(
    api_key="your-evolink-key",
    base_url="https://api.evolink.ai/v1"
)

# Let the Smart Router choose based on your workload
response = client.chat.completions.create(
    model="evolink/auto",
    messages=your_messages
)

import tiktoken

def check_context_length(messages: list, model: str, max_tokens: int) -> dict:
    """Pre-check whether messages fit within the model's context window."""
    encoder = tiktoken.encoding_for_model(model)
    total_tokens = sum(
        len(encoder.encode(m["content"])) for m in messages
    )

    if total_tokens > max_tokens:
        return {
            "fits": False,
            "total_tokens": total_tokens,
            "excess": total_tokens - max_tokens,
            "suggestion": "truncate" if total_tokens < max_tokens * 1.5
                          else "summarize_or_switch"
        }

    return {"fits": True, "total_tokens": total_tokens}