initial commit

.gitignore

@@ -0,0 +1,2 @@
+venv/
+output*

README.md

@@ -1,8 +1,8 @@
 ---
 title: Multi View Diffusion
-emoji: 📚
-colorFrom: blue
-colorTo: pink
+emoji: 🧊
+colorFrom: indigo
+colorTo: blue
 sdk: gradio
 sdk_version: 4.24.0
 app_file: app.py

app.py

@@ -0,0 +1,84 @@
+import gradio as gr
+
+import torch
+from diffusers import DiffusionPipeline
+from PIL import Image
+
+# Text-to-Multi-View Diffusion pipeline
+text_pipeline = DiffusionPipeline.from_pretrained(
+    "ashawkey/mvdream-sd2.1-diffusers",
+    custom_pipeline="dylanebert/multi_view_diffusion",
+    torch_dtype=torch.float16,
+    trust_remote_code=True,
+).to("cuda")
+
+# Image-to-Multi-View Diffusion pipeline
+image_pipeline = DiffusionPipeline.from_pretrained(
+    "ashawkey/imagedream-ipmv-diffusers",
+    custom_pipeline="dylanebert/multi_view_diffusion",
+    torch_dtype=torch.float16,
+    trust_remote_code=True,
+).to("cuda")
+
+
+def create_image_grid(images):
+    images = [Image.fromarray((img * 255).astype("uint8")) for img in images]
+
+    width, height = images[0].size
+    grid_img = Image.new("RGB", (2 * width, 2 * height))
+
+    grid_img.paste(images[0], (0, 0))
+    grid_img.paste(images[1], (width, 0))
+    grid_img.paste(images[2], (0, height))
+    grid_img.paste(images[3], (width, height))
+
+    return grid_img
+
+
+def text_to_mv(prompt):
+    images = text_pipeline(
+        prompt, guidance_scale=5, num_inference_steps=30, elevation=0
+    )
+    return create_image_grid(images)
+
+
+def image_to_mv(image, prompt):
+    image = image.astype("float32") / 255.0
+    images = image_pipeline(
+        prompt, image, guidance_scale=5, num_inference_steps=30, elevation=0
+    )
+    return create_image_grid(images)
+
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            with gr.Tab("Text Input"):
+                text_input = gr.Textbox(
+                    lines=2,
+                    show_label=False,
+                    placeholder="Enter a prompt here (e.g. 'a cat statue')",
+                )
+                text_btn = gr.Button("Generate Multi-View Images")
+            with gr.Tab("Image Input"):
+                image_input = gr.Image(
+                    label="Image Input",
+                    type="numpy",
+                )
+                optional_text_input = gr.Textbox(
+                    lines=2,
+                    show_label=False,
+                    placeholder="Enter an optional prompt here",
+                )
+                image_btn = gr.Button("Generate Multi-View Images")
+        with gr.Column():
+            output = gr.Image(label="Generated Images")
+
+    text_btn.click(fn=text_to_mv, inputs=text_input, outputs=output)
+    image_btn.click(
+        fn=image_to_mv, inputs=[image_input, optional_text_input], outputs=output
+    )
+
+
+if __name__ == "__main__":
+    demo.queue().launch()

requirements.txt

@@ -0,0 +1,103 @@
+accelerate==0.28.0
+aiofiles==23.2.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.3.0
+attrs==23.2.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+diffusers==0.27.2
+einops==0.7.0
+exceptiongroup==1.2.0
+executing==2.0.1
+fastapi==0.110.0
+ffmpy==0.3.2
+filelock==3.13.3
+fonttools==4.50.0
+fsspec==2024.3.1
+gradio==4.24.0
+gradio_client==0.14.0
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.22.2
+idna==3.6
+importlib_metadata==7.1.0
+importlib_resources==6.4.0
+Jinja2==3.1.3
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiui==0.2.7
+kiwisolver==1.4.5
+lazy_loader==0.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.8.3
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.2.1
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.4.99
+nvidia-nvtx-cu12==12.1.105
+objprint==0.2.3
+opencv-python==4.9.0.80
+orjson==3.10.0
+packaging==24.0
+pandas==2.2.1
+pillow==10.2.0
+psutil==5.9.8
+pydantic==2.6.4
+pydantic_core==2.16.3
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.34.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.18.0
+ruff==0.3.4
+safetensors==0.4.2
+scipy==1.12.0
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.36.3
+sympy==1.12
+tokenizers==0.15.2
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.2.2
+torchaudio==2.2.2
+torchvision==0.17.2
+tqdm==4.66.2
+transformers==4.39.2
+triton==2.2.0
+typer==0.11.1
+typing_extensions==4.10.0
+tzdata==2024.1
+urllib3==2.2.1
+uvicorn==0.29.0
+varname==0.13.0
+websockets==11.0.3
+xformers==0.0.25.post1
+zipp==3.18.1