如何为聊天机器人或 AI Agent 接入内容审核

input moderation
  -> model or agent reasoning
  -> tool-call validation
  -> output moderation
  -> allow / review / block

def handle_user_message(user_input: str):
    input_check = client.moderations.create(
        model="evolink-moderation-1.0",
        input=user_input
    )

    if input_check.evolink_summary.risk_level == "high":
        return {
            "status": "blocked",
            "message": "I can't help with that request."
        }

    return run_chatbot(user_input)

def safe_chat(user_input: str):
    input_check = moderate(user_input)
    if input_check.evolink_summary.risk_level == "high":
        return "I can't help with that request."

    answer = llm.generate(user_input)

    output_check = moderate(answer)
    if output_check.evolink_summary.risk_level == "high":
        return "I generated a response that needs to be revised. Please try rephrasing."

    return answer

def validate_tool_call(tool_name: str, args: dict, user_context: dict):
    if tool_name == "send_email":
        if not args["to"].endswith("@company.com"):
            return False, "External email requires approval."

    if tool_name == "database_query":
        query = args.get("query", "")
        forbidden = ["DROP ", "DELETE ", "TRUNCATE "]
        if any(word in query.upper() for word in forbidden):
            return False, "Destructive database operation blocked."

    if tool_name == "refund_customer":
        if args["amount"] > user_context["approval_limit"]:
            return False, "Refund amount requires human approval."

    return True, None

def route_by_risk(content: str):
    result = client.moderations.create(
        model="evolink-moderation-1.0",
        input=content
    )

    risk = result.evolink_summary.risk_level

    if risk == "high":
        log_block(result)
        return {"action": "block"}

    if risk == "medium":
        queue_for_review(content, result)
        return {"action": "review"}

    return {"action": "allow"}

def handle_chatbot_turn(user_input: str, user_context: dict):
    # 1. Moderate input
    input_route = route_by_risk(user_input)
    if input_route["action"] == "block":
        return "I can't help with that request."
    if input_route["action"] == "review":
        log_review_event("input", user_input)

    # 2. Generate model response and proposed tool calls
    agent_result = agent.run(user_input, user_context)

    # 3. Validate tool calls before execution
    for call in agent_result.tool_calls:
        ok, reason = validate_tool_call(call.name, call.args, user_context)
        if not ok:
            log_security_event(call, reason)
            return "That action requires additional review."

    # 4. Execute approved tools
    tool_outputs = execute_tool_calls(agent_result.tool_calls)

    # 5. Generate final response
    final_answer = agent.format_response(tool_outputs)

    # 6. Moderate output
    output_route = route_by_risk(final_answer)
    if output_route["action"] == "block":
        return "I need to revise that response."
    if output_route["action"] == "review":
        queue_for_review(final_answer, output_route)

    return final_answer