feat: add aws canva + examples in readme + my current environment + disable HPS for now

README.md

@@ -6,5 +6,14 @@ Install dependencies with conda like that:
 conda env create -f environment.yml
 
 
-Create .env file with all the files you will need.
-python sample.py replicate draw_bench genai_bench geneval hps parti
+Create .env file with all the credentials you will need.
+
+This is how you can generate the images.
+```
+python sample.py replicate draw_bench genai_bench geneval hps parti
+```
+
+This is how you would evaluate the benchmarks once you have all images:
+```
+python evaluate.py replicate draw_bench genai_bench geneval hps parti
+```

api/__init__.py

@@ -8,6 +8,7 @@ from api.pruna_dev import PrunaDevAPI
 from api.replicate import ReplicateAPI
 from api.together import TogetherAPI
 from api.fal import FalAPI
+from api.aws import AWSBedrockAPI
 
 __all__ = [
     'create_api',
@@ -33,6 +34,7 @@ def create_api(api_type: str) -> FluxAPI:
             - "replicate"
             - "together"
             - "fal"
+            - "aws"
             
     Returns:
         FluxAPI: An instance of the requested API implementation
@@ -52,6 +54,7 @@ def create_api(api_type: str) -> FluxAPI:
         "replicate": ReplicateAPI,
         "together": TogetherAPI,
         "fal": FalAPI,
+        "aws": AWSBedrockAPI,
     }
     
     if api_type not in api_map:

api/aws.py

@@ -0,0 +1,73 @@
+import os
+import time
+import base64
+import json
+from pathlib import Path
+from typing import Any
+
+import boto3
+from dotenv import load_dotenv
+
+from api.flux import FluxAPI
+
+
+class AWSBedrockAPI(FluxAPI):
+    def __init__(self):
+        load_dotenv()
+        # AWS credentials should be set via environment variables
+        # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN
+        os.environ["AWS_ACCESS_KEY_ID"] = ""
+        os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+        os.environ["AWS_SESSION_TOKEN"] = ""
+        os.environ["AWS_REGION"] = "us-east-1"
+        self._client = boto3.client("bedrock-runtime")
+        self._model_id = "amazon.nova-canvas-v1:0"
+
+    @property
+    def name(self) -> str:
+        return "aws_nova_canvas"
+
+    def generate_image(self, prompt: str, save_path: Path) -> float:
+        start_time = time.time()
+        # Format the request payload
+        native_request = {
+            "taskType": "TEXT_IMAGE",
+            "textToImageParams": {"text": prompt},
+            "imageGenerationConfig": {
+                "seed": 0,
+                "quality": "standard",
+                "height": 1024,
+                "width": 1024,
+                "numberOfImages": 1,
+            },
+        }
+
+        try:
+            # Convert request to JSON and invoke the model
+            request = json.dumps(native_request)
+            response = self._client.invoke_model(
+                modelId=self._model_id,
+                body=request
+            )
+            
+            # Process the response
+            model_response = json.loads(response["body"].read())
+            if not model_response.get("images"):
+                raise Exception("No images returned from AWS Bedrock API")
+                
+            # Save the image
+            base64_image_data = model_response["images"][0]
+            self._save_image_from_base64(base64_image_data, save_path)
+            
+        except Exception as e:
+            raise Exception(f"Error generating image with AWS Bedrock: {str(e)}")
+            
+        end_time = time.time()
+        return end_time - start_time
+
+    def _save_image_from_base64(self, base64_data: str, save_path: Path):
+        """Save a base64 encoded image to the specified path."""
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        image_data = base64.b64decode(base64_data)
+        with open(save_path, "wb") as f:
+            f.write(image_data) 

benchmark/metrics/__init__.py

@@ -6,7 +6,7 @@ from benchmark.metrics.clip_iqa import CLIPIQAMetric
 from benchmark.metrics.image_reward import ImageRewardMetric
 from benchmark.metrics.sharpness import SharpnessMetric
 from benchmark.metrics.vqa import VQAMetric
-from benchmark.metrics.hps import HPSMetric
+#from benchmark.metrics.hps import HPSMetric
 
 def create_metric(metric_type: str) -> Type[ARNIQAMetric | CLIPMetric | CLIPIQAMetric | ImageRewardMetric | SharpnessMetric | VQAMetric]:
     """
@@ -34,7 +34,7 @@ def create_metric(metric_type: str) -> Type[ARNIQAMetric | CLIPMetric | CLIPIQAM
         "image_reward": ImageRewardMetric,
         "sharpness": SharpnessMetric,
         "vqa": VQAMetric,
-        "hps": HPSMetric,
+        #"hps": HPSMetric,
     }
     
     if metric_type not in metric_map:

nils_installs.txt

@@ -0,0 +1,177 @@
+accelerate==1.7.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.12
+aiosignal==1.3.2
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.9.0
+args==0.1.0
+asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1733250440834/work
+attrs==25.3.0
+beautifulsoup4==4.13.4
+boto3==1.38.33
+botocore==1.38.33
+braceexpand==0.1.7
+Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1749229842835/work
+certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1746569525376/work/certifi
+cffi @ file:///home/conda/feedstock_root/build_artifacts/cffi_1725560558132/work
+charset-normalizer @ file:///home/conda/feedstock_root/build_artifacts/charset-normalizer_1746214863626/work
+click==8.1.8
+clint==0.5.1
+clip==0.2.0
+colorama @ file:///home/conda/feedstock_root/build_artifacts/colorama_1733218098505/work
+comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1733502965406/work
+contourpy==1.3.2
+cycler==0.12.1
+datasets==3.6.0
+debugpy @ file:///home/conda/feedstock_root/build_artifacts/debugpy_1744321241074/work
+decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1740384970518/work
+diffusers==0.31.0
+dill==0.3.8
+distro==1.9.0
+einops==0.8.1
+eval_type_backport==0.2.2
+exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1746947292760/work
+executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1745502089858/work
+fairscale==0.4.13
+fal_client==0.7.0
+filelock==3.18.0
+fire==0.4.0
+fonttools==4.58.2
+frozenlist==1.7.0
+fsspec==2025.3.0
+ftfy==6.3.1
+gdown==5.2.0
+h11==0.16.0
+h2 @ file:///home/conda/feedstock_root/build_artifacts/h2_1738578511449/work
+hf-xet==1.1.3
+hpack @ file:///home/conda/feedstock_root/build_artifacts/hpack_1737618293087/work
+hpsv2==1.2.0
+httpcore==1.0.9
+httpx==0.28.1
+httpx-sse==0.4.0
+huggingface-hub==0.32.5
+hyperframe @ file:///home/conda/feedstock_root/build_artifacts/hyperframe_1737618333194/work
+idna @ file:///home/conda/feedstock_root/build_artifacts/idna_1733211830134/work
+image-reward==1.5
+importlib_metadata==8.7.0
+iniconfig==2.1.0
+iopath==0.1.10
+ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1719845459717/work
+ipython @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipython_1748713870/work
+ipython_pygments_lexers @ file:///home/conda/feedstock_root/build_artifacts/ipython_pygments_lexers_1737123620466/work
+jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1733300866624/work
+Jinja2==3.1.6
+jiter==0.10.0
+jmespath==1.0.1
+joblib==1.5.1
+jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1733440914442/work
+jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1748333051527/work
+kiwisolver==1.4.8
+lightning-utilities==0.14.3
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+matplotlib==3.10.3
+matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1733416936468/work
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.4.4
+multiprocess==0.70.16
+nest_asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1733325553580/work
+networkx==3.5
+numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1749430504934/work/dist/numpy-2.3.0-cp312-cp312-linux_x86_64.whl#sha256=3c4437a0cbe50dbae872ad4cd8dc5316009165bce459c4ffe2c46cd30aba13d4
+nvidia-cublas-cu12==12.6.4.1
+nvidia-cuda-cupti-cu12==12.6.80
+nvidia-cuda-nvrtc-cu12==12.6.77
+nvidia-cuda-runtime-cu12==12.6.77
+nvidia-cudnn-cu12==9.5.1.17
+nvidia-cufft-cu12==11.3.0.4
+nvidia-cufile-cu12==1.11.1.6
+nvidia-curand-cu12==10.3.7.77
+nvidia-cusolver-cu12==11.7.1.2
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-nccl-cu12==2.26.2
+nvidia-nvjitlink-cu12==12.6.85
+nvidia-nvtx-cu12==12.6.77
+omegaconf==2.3.0
+open_clip_torch==2.32.0
+openai==1.85.0
+opencv-python==4.11.0
+opencv-python-headless==4.11.0
+packaging==25.0
+pandas==2.3.0
+parso @ file:///home/conda/feedstock_root/build_artifacts/parso_1733271261340/work
+pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1733301927746/work
+pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1733327343728/work
+pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1746646208260/work
+piq==0.8.0
+platformdirs @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_platformdirs_1746710438/work
+pluggy==1.6.0
+portalocker==3.1.1
+prompt_toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1744724089886/work
+propcache==0.3.2
+protobuf==3.20.3
+psutil==7.0.0
+ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1733302279685/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl#sha256=92c32ff62b5fd8cf325bec5ab90d7be3d2a8ca8c8a3813ff487a8d2002630d1f
+pure_eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1733569405015/work
+pyarrow==20.0.0
+pycocoevalcap==1.2
+pycocotools==2.0.10
+pycparser @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_pycparser_1733195786/work
+pydantic==2.11.5
+pydantic_core==2.33.2
+Pygments==2.19.1
+pyparsing==3.2.3
+PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1733217236728/work
+pytest==7.2.0
+pytest-split==0.8.0
+python-dateutil==2.9.0.post0
+python-dotenv @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_python-dotenv_1742948348/work
+pytz==2025.2
+PyYAML==6.0.2
+pyzmq @ file:///home/conda/feedstock_root/build_artifacts/pyzmq_1743831245863/work
+regex==2024.11.6
+replicate==1.0.7
+requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1749498106507/work
+rich==14.0.0
+s3transfer==0.13.0
+safetensors==0.5.3
+scikit-learn==1.7.0
+scipy==1.15.3
+sentencepiece==0.2.0
+setuptools==80.9.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.7
+stack_data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1733569443808/work
+sympy==1.14.0
+t2v_metrics==1.2
+tabulate==0.9.0
+termcolor==3.1.0
+threadpoolctl==3.6.0
+tiktoken==0.9.0
+timm==0.6.13
+together==1.5.11
+tokenizers==0.15.2
+torch==2.7.1
+torchmetrics==1.7.2
+torchvision==0.22.1
+tornado @ file:///home/conda/feedstock_root/build_artifacts/tornado_1748003300911/work
+tqdm @ file:///home/conda/feedstock_root/build_artifacts/tqdm_1735661334605/work
+traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1733367359838/work
+transformers==4.36.1
+triton==3.3.1
+typer==0.15.4
+typing-inspection==0.4.1
+typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_typing_extensions_1748959427/work
+tzdata==2025.2
+urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1744323578849/work
+wcwidth==0.2.13
+webdataset==0.2.111
+wheel==0.45.1
+xxhash==3.5.0
+yarl==1.20.1
+zipp==3.23.0
+zstandard==0.23.0