Dockerfile

FROM vllm/vllm-openai:v0.10.0
# FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu22.04

# ENV VLLM_LOGGING_LEVEL=DEBUG
# ENV HF_HOME=/tmp/.cache/huggingface
# ENV OMP_NUM_THREADS=1

# # https://github.com/vllm-project/vllm/blob/v0.10.0/docs/getting_started/installation/gpu/rocm.inc.md?plain=1#L124
# ENV VLLM_USE_TRITON_FLASH_ATTN=0

# # https://github.com/vllm-project/vllm/blob/v0.10.0/docs/getting_started/quickstart.md?plain=1#L213
# # `FLASH_ATTN` or `FLASHINFER` or `XFORMERS`.
# # https://github.com/vllm-project/vllm/blob/main/vllm/envs.py#L426-L435
# # Backend for attention computation
# # Available options:
# # - "TORCH_SDPA": use torch.nn.MultiheadAttention
# # - "FLASH_ATTN": use FlashAttention
# # - "XFORMERS": use XFormers
# # - "ROCM_FLASH": use ROCmFlashAttention
# # - "FLASHINFER": use flashinfer
# # - "FLASHMLA": use FlashMLA
# # Choose XFORMERS that most stable for T4
# ENV VLLM_ATTENTION_BACKEND=XFORMERS

# RUN apt-get update && apt-get install -y python3 python3-pip git
# RUN pip3 install --upgrade pip

# # Install vLLM
# RUN pip3 install vllm==0.10.0 --extra-index-url https://download.pytorch.org/whl/cu113

# RUN useradd -m appuser
# USER appuser

# RUN mkdir -p /tmp/.cache/huggingface

# Download at build time,
# to ensure during restart we won't have to wait for the download from HF (only wait for docker pull).
# In Docker Spaces, the secrets management is different for security reasons.
# Once you create a secret in the Settings tab,
# you can expose the secret by adding the following line in your Dockerfile:
#
# For example, if SECRET_EXAMPLE is the name of the secret you created in the Settings tab,
# you can read it at build time by mounting it to a file, then reading it with $(cat /run/secrets/SECRET_EXAMPLE).
# https://huggingface.co/docs/hub/en/spaces-sdks-docker#buildtime
#
# AFTER TRIAL AND ERROR WE GOT 16GB (16431849854 bytes) OF LAYERS :(
#
# RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true HF_TOKEN=$(cat /run/secrets/HF_TOKEN) python /app/download_model.py

EXPOSE 7860

# Export for runtime environment
# CMD vllm serve \
#   --model "meta-llama/Llama-3.2-3B-Instruct" \
#   --task generate \
#   --revision "0cb88a4f764b7a12671c53f0838cd831a0843b95" \
#   --code-revision "0cb88a4f764b7a12671c53f0838cd831a0843b95" \
#   --tokenizer-revision "0cb88a4f764b7a12671c53f0838cd831a0843b95" \
#   --seed 42 \
#   --host 0.0.0.0 \
#   --port 7860 \
#   --max-num-batched-tokens 32768 \
#   --max-model-len 32768 \
#   --dtype float16 \
#   --enforce-eager \
#   --gpu-memory-utilization 0.9 \
#   --enable-prefix-caching \
#   --disable-log-requests \
#   --trust-remote-code

CMD --model "meta-llama/Llama-3.2-3B-Instruct" \
  --task generate \
  --revision "0cb88a4f764b7a12671c53f0838cd831a0843b95" \
  --code-revision "0cb88a4f764b7a12671c53f0838cd831a0843b95" \
  --tokenizer-revision "0cb88a4f764b7a12671c53f0838cd831a0843b95" \
  --seed 42 \
  --host 0.0.0.0 \
  --port 7860 \
  --max-num-batched-tokens 32768 \
  --max-model-len 32768 \
  --dtype float16 \
  --enforce-eager \
  --gpu-memory-utilization 0.9 \
  --enable-prefix-caching \
  --disable-log-requests \
  --trust-remote-code