# ==========================================================
# FILE: ghostpack.py
# ==========================================================
#!/usr/bin/env python3
# ---------------------------------------------------------------------------
# RELEASE – GhostPack Image-to-Video Generator
# ---------------------------------------------------------------------------
import os, sys, argparse, traceback
import numpy as np, torch, einops, gradio as gr
from PIL import Image
from diffusers_helper.hf_login import login
from diffusers import AutoencoderKLHunyuanVideo
from transformers import (
LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer,
SiglipImageProcessor, SiglipVisionModel,
)
from diffusers_helper.hunyuan import (
encode_prompt_conds, vae_encode, vae_decode, vae_decode_fake,
)
from diffusers_helper.utils import (
save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw,
resize_and_center_crop, generate_timestamp,
)
from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
from diffusers_helper.memory import (
gpu, get_cuda_free_memory_gb, DynamicSwapInstaller,
unload_complete_models, load_model_as_complete,
fake_diffusers_current_device, move_model_to_device_with_memory_preservation,
offload_model_from_device_for_memory_preservation,
)
from diffusers_helper.thread_utils import AsyncStream, async_run
from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
from diffusers_helper.clip_vision import hf_clip_vision_encode
from diffusers_helper.bucket_tools import find_nearest_bucket
BASE = os.path.abspath(os.path.dirname(__file__))
CACHE = os.path.join(BASE, "hf_download")
os.makedirs(CACHE, exist_ok=True)
for v in ("HF_HOME", "TRANSFORMERS_CACHE", "HF_DATASETS_CACHE"): os.environ[v] = CACHE
os.environ["HF_HUB_DISABLE_SYMLINKS_WARNING"] = "1"
p = argparse.ArgumentParser()
p.add_argument("--share", action="store_true")
p.add_argument("--server", default="0.0.0.0")
p.add_argument("--port", type=int, default=7860)
p.add_argument("--inbrowser", action="store_true")
args = p.parse_args()
free_gb = get_cuda_free_memory_gb(gpu)
hi_vram = free_gb > 60
print(f"[GhostPack] Free VRAM: {free_gb:.1f} GB | High-VRAM: {hi_vram}")
def llm(sub): return LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder=sub, torch_dtype=torch.float16).cpu().eval()
def clip(sub): return CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder=sub, torch_dtype=torch.float16).cpu().eval()
text_enc = llm("text_encoder")
text_enc2 = clip("text_encoder_2")
tok = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder="tokenizer")
tok2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder="tokenizer_2")
vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder="vae", torch_dtype=torch.float16).cpu().eval()
feat_ext = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder="feature_extractor")
img_enc = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder="image_encoder", torch_dtype=torch.float16).cpu().eval()
trans = HunyuanVideoTransformer3DModelPacked.from_pretrained("lllyasviel/FramePackI2V_HY", torch_dtype=torch.bfloat16).cpu().eval()
trans.high_quality_fp32_output_for_inference = True
if not hi_vram:
vae.enable_slicing(); vae.enable_tiling()
else:
for m in (text_enc, text_enc2, img_enc, vae, trans): m.to(gpu)
trans.to(dtype=torch.bfloat16)
for m in (vae, img_enc, text_enc, text_enc2): m.to(dtype=torch.float16)
for m in (vae, img_enc, text_enc, text_enc2, trans): m.requires_grad_(False)
if not hi_vram:
DynamicSwapInstaller.install_model(trans, device=gpu)
DynamicSwapInstaller.install_model(text_enc, device=gpu)
OUT = os.path.join(BASE, "outputs")
os.makedirs(OUT, exist_ok=True)
stream = AsyncStream()
@torch.no_grad()
def worker(img, p, n_p, sd, secs, win, stp, cfg, gsc, rsc, keep, tea, crf):
sections = max(round((secs*30)/(win*4)), 1)
job = generate_timestamp()
stream.output_queue.push(("progress",(None,"",make_progress_bar_html(0,"Start"))))
try:
if not hi_vram: unload_complete_models(text_enc, text_enc2, img_enc, vae, trans)
stream.output_queue.push(("progress",(None,"",make_progress_bar_html(0,"Text enc"))))
if not hi_vram:
fake_diffusers_current_device(text_enc, gpu)
load_model_as_complete(text_enc2, gpu)
lv, cp = encode_prompt_conds(p, text_enc, text_enc2, tok, tok2)
lv_n, cp_n = (torch.zeros_like(lv), torch.zeros_like(cp)) if cfg==1 else encode_prompt_conds(n_p, text_enc, text_enc2, tok, tok2)
lv, m = crop_or_pad_yield_mask(lv,512)
lv_n, m_n= crop_or_pad_yield_mask(lv_n,512)
stream.output_queue.push(("progress",(None,"",make_progress_bar_html(0,"Image"))))
H,W,_ = img.shape; h,w = find_nearest_bucket(H,W,640)
img_np = resize_and_center_crop(img,w,h)
Image.fromarray(img_np).save(os.path.join(OUT,f"{job}.png"))
img_pt = torch.from_numpy(img_np).float()/127.5-1; img_pt = img_pt.permute(2,0,1)[None,:,None]
stream.output_queue.push(("progress",(None,"",make_progress_bar_html(0,"VAE"))))
if not hi_vram: load_model_as_complete(vae, gpu)
start_lat = vae_encode(img_pt, vae)
stream.output_queue.push(("progress",(None,"",make_progress_bar_html(0,"Vision"))))
if not hi_vram: load_model_as_complete(img_enc, gpu)
img_hidden = hf_clip_vision_encode(img_np, feat_ext, img_enc).last_hidden_state
to = trans.dtype
lv, lv_n, cp, cp_n, img_hidden = (x.to(to) for x in (lv, lv_n, cp, cp_n, img_hidden))
stream.output_queue.push(("progress",(None,"",make_progress_bar_html(0,"Sample"))))
gen = torch.Generator("cpu").manual_seed(sd)
frames = win*4-3
hist_lat = torch.zeros((1,16,1+2+16,h//8,w//8), dtype=torch.float32).cpu()
hist_px=None; total=0
pad_seq=[3]+[2]*(sections-3)+[1,0] if sections>4 else list(reversed(range(sections)))
for pad in pad_seq:
last = pad==0
if stream.input_queue.top()=="end": stream.output_queue.push(("end",None)); return
pad_sz=pad*win
idx=torch.arange(0,sum([1,pad_sz,win,1,2,16])).unsqueeze(0)
a,b,c,d,e,f = idx.split([1,pad_sz,win,1,2,16],1)
clean_idx = torch.cat([a,d],1)
pre=start_lat.to(hist_lat); post,two,four=hist_lat[:,:,:1+2+16].split([1,2,16],2)
clean=torch.cat([pre,post],2)
if not hi_vram:
unload_complete_models()
move_model_to_device_with_memory_preservation(trans,gpu,keep)
trans.initialize_teacache(tea,stp)
def cb(d):
pv = vae_decode_fake(d["denoised"])
pv = (pv*255).cpu().numpy().clip(0,255).astype(np.uint8)
pv = einops.rearrange(pv,"b c t h w->(b h)(t w)c")
cur = d["i"]+1
stream.output_queue.push(("progress",(pv,f"{total*4-3}f",make_progress_bar_html(int(100*cur/stp),f"{cur}/{stp}"))))
if stream.input_queue.top()=="end":
stream.output_queue.push(("end",None)); raise KeyboardInterrupt
new_lat = sample_hunyuan(
transformer=trans,sampler="unipc",width=w,height=h,frames=frames,
real_guidance_scale=cfg,distilled_guidance_scale=gsc,guidance_rescale=rsc,
num_inference_steps=stp,generator=gen,
prompt_embeds=lv,prompt_embeds_mask=m,prompt_poolers=cp,
negative_prompt_embeds=lv_n,negative_prompt_embeds_mask=m_n,negative_prompt_poolers=cp_n,
device=gpu,dtype=torch.bfloat16,image_embeddings=img_hidden,
latent_indices=c,clean_latents=clean,clean_latent_indices=clean_idx,
clean_latents_2x=two,clean_latent_2x_indices=e,clean_latents_4x=four,clean_latent_4x_indices=f,
callback=cb,
)
if last: new_lat=torch.cat([start_lat.to(new_lat),new_lat],2)
total+=new_lat.shape[2]; hist_lat=torch.cat([new_lat.to(hist_lat),hist_lat],2)
if not hi_vram:
offload_model_from_device_for_memory_preservation(trans,gpu,8)
load_model_as_complete(vae,gpu)
real=hist_lat[:,:,:total]
if hist_px is None:
hist_px = vae_decode(real,vae).cpu()
else:
sec_lat=win*2+1 if last else win*2
cur_px = vae_decode(real[:,:,:sec_lat],vae).cpu()
hist_px = soft_append_bcthw(cur_px,hist_px,win*4-3)
if not hi_vram: unload_complete_models()
mp4=os.path.join(OUT,f"{job}_{total}.mp4")
save_bcthw_as_mp4(hist_px,mp4,fps=30,crf=crf)
stream.output_queue.push(("file",mp4))
if last: break
except Exception:
traceback.print_exc(); stream.output_queue.push(("end",None))
def ui():
css = make_progress_bar_css()+"""
body,.gradio-container,.gr-block{background:#121212;color:#eee}
.gr-button,.gr-button-primary{background:#006400;border:#006400}
.gr-button:hover,.gr-button-primary:hover{background:#00aa00;border:#00aa00}
input,textarea,.gr-input,.gr-textbox,.gr-slider,.gr-number{background:#1e1e1e;color:#eee;border-color:#006400}
"""
quick=[["The girl dances gracefully, with clear movements, full of charm."],
["A character doing some simple body movements."]]
blk=gr.Blocks(css=css).queue()
with blk:
gr.Markdown("# 👻 GhostPack Demo")
with gr.Row():
with gr.Column():
img=gr.Image(sources="upload",type="numpy",label="Image",height=320)
prm=gr.Textbox(label="Prompt")
ds=gr.Dataset(samples=quick,label="Quick List",components=[prm])
ds.click(lambda x:x[0],inputs=[ds],outputs=prm)
with gr.Row():
b_go=gr.Button("Start"); b_end=gr.Button("End",interactive=False)
with gr.Group():
tea=gr.Checkbox(label="Use TeaCache",value=True)
npr=gr.Textbox(label="Negative Prompt",visible=False)
se=gr.Number(label="Seed",value=31337,precision=0)
sec=gr.Slider(label="Video Length (s)",minimum=1,maximum=120,value=5,step=0.1)
win=gr.Slider(label="Latent Window",minimum=1,maximum=33,value=9,step=1,visible=False)
stp=gr.Slider(label="Steps",minimum=1,maximum=100,value=25,step=1)
cfg=gr.Slider(label="CFG",minimum=1,maximum=32,value=1,step=0.01,visible=False)
gsc=gr.Slider(label="Distilled CFG",minimum=1,maximum=32,value=10,step=0.01)
rsc=gr.Slider(label="CFG Re-Scale",minimum=0,maximum=1,value=0,step=0.01,visible=False)
kee=gr.Slider(label="GPU Keep (GB)",minimum=6,maximum=128,value=6,step=0.1)
crf=gr.Slider(label="MP4 CRF",minimum=0,maximum=100,value=16,step=1)
with gr.Column():
pv=gr.Image(label="Next Latents",height=200,visible=False,interactive=False)
vid=gr.Video(label="Finished",autoplay=True,height=512,loop=True,show_share_button=False)
gr.Markdown("Ending actions appear first; wait for start.")
dsc=gr.Markdown("")
bar=gr.HTML("")
log=gr.Markdown("")
inputs=[img,prm,npr,se,sec,win,stp,cfg,gsc,rsc,kee,tea,crf]
def launch(*xs):
global stream
if xs[0] is None: raise gr.Error("Upload an image.")
yield None,None,"","","",gr.update(interactive=False),gr.update(interactive=True)
stream=AsyncStream()
async_run(worker,*xs)
out=None; log=""
while True:
flag,data=stream.output_queue.next()
if flag=="file":
out=data
yield out,gr.update(),gr.update(),gr.update(),log,gr.update(interactive=False),gr.update(interactive=True)
if flag=="progress":
pv,desc,html=data; log=desc
yield gr.update(),gr.update(visible=True,value=pv),desc,html,log,gr.update(interactive=False),gr.update(interactive=True)
if flag=="end":
yield out,gr.update(visible=False),gr.update(),"",log,gr.update(interactive=True),gr.update(interactive=False); break
b_go.click(launch,inputs,[vid,pv,dsc,bar,log,b_go,b_end])
b_end.click(lambda: stream.input_queue.push("end"))
blk.launch(server_name=args.server,server_port=args.port,share=args.share,inbrowser=args.inbrowser)
if __name__ == "__main__":
ui()