fix(float16): Bfloat16 is only supported on GPUs with compute capability of at least 8.0. Your Tesla T4 GPU has compute capability 7.5. You can use float16 instead by explicitly setting the`dtype` flag in CLI, for example: --dtype=half.

runner.sh

@@ -59,7 +59,7 @@ python -u /app/openai_compatible_api_server.py \
     --port 7860 \
     --max-num-batched-tokens 32768 \
     --max-model-len 32768 \
-    --dtype bfloat16 \
+    --dtype float16 \
     --enforce-eager \
     --gpu-memory-utilization 0.9 \
     --enable-prefix-caching \