Add fallback path handling for SPA and start descriptions

README.md

@@ -30,7 +30,7 @@ conda activate omniseal-benchmark-backend
 ```bash
 cd frontend
 npm install
-npm run build
+npm run build -- --mode prod
 ```
 
 3. Run backend server from project root. This would serve frontend files from port http://localhost:7860

backend/app.py

@@ -3,6 +3,7 @@ from backend.config import (
     get_dataset_config,
     get_datasets,
 )
+from backend.descriptions import DESCRIPTIONS, MODEL_DESCRIPTIONS
 from backend.examples import (
     get_examples_tab,
 )
@@ -92,6 +93,22 @@ def example_files(type):
     return Response(json.dumps(result), mimetype="application/json")
 
 
+@app.route("/descriptions")
+def descriptions():
+    """
+    Serve descriptions and model descriptions from descriptions.py
+    """
+    return Response(
+        json.dumps(
+            {
+                "descriptions": DESCRIPTIONS,
+                "model_descriptions": MODEL_DESCRIPTIONS,
+            }
+        ),
+        mimetype="application/json",
+    )
+
+
 # Add a proxy endpoint to bypass CORS issues
 @app.route("/proxy/<path:url>")
 def proxy(url):
@@ -197,5 +214,11 @@ def get_chart(config, df):
     return Response(json.dumps(chart_data), mimetype="application/json")
 
 
+@app.errorhandler(404)
+def not_found(e):
+    # Serve index.html for any 404 (SPA fallback)
+    return send_from_directory(app.static_folder, "index.html")
+
+
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=7860, debug=True, use_reloader=True)

backend/descriptions.py

@@ -0,0 +1,164 @@
+DESCRIPTIONS = {
+    "snr": {
+        "full_name": "Signal-to-Noise Ratio",
+        "description": "Measures the ratio between the power of a signal and the power of background noise, expressed in decibels (dB). Higher values indicate better quality, with less noise relative to the signal.",
+        "link": "https://en.wikipedia.org/wiki/Signal-to-noise_ratio",
+    },
+    "sisnr": {
+        "full_name": "Scale-Invariant Signal-to-Noise Ratio",
+        "description": "A variant of SNR that is invariant to scaling of the signal, making it better for comparing audio quality across different amplitude levels. Higher values indicate better quality.",
+        "link": "https://arxiv.org/abs/1811.02508",
+    },
+    "stoi": {
+        "full_name": "Short-Time Objective Intelligibility",
+        "description": "Measures the intelligibility of speech by comparing temporal envelopes of clean and degraded speech. Values range from 0 to 1, with higher values indicating better intelligibility.",
+        "link": "https://ieeexplore.ieee.org/document/5495701",
+    },
+    "pesq": {
+        "full_name": "Perceptual Evaluation of Speech Quality",
+        "description": "An ITU-T standard for measuring speech quality, designed to mimic human perception. Scores range from -0.5 to 4.5, with higher values indicating better perceived quality.",
+        "link": "https://www.itu.int/rec/T-REC-P.862",
+    },
+    "psnr": {
+        "full_name": "Peak Signal-to-Noise Ratio",
+        "description": "Measures the ratio between the maximum possible power of a signal and the power of corrupting noise. Used primarily for image and video quality assessment, with higher values (in dB) indicating better quality.",
+        "link": "https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio",
+    },
+    "ssim": {
+        "full_name": "Structural SIMilarity Index",
+        "description": "Measures the perceived similarity between two images based on structural information, contrast, and luminance. Values range from -1 to 1, with 1 indicating perfect similarity.",
+        "link": "https://en.wikipedia.org/wiki/Structural_similarity",
+    },
+    "msssim": {
+        "full_name": "Multi-Scale Structural Similarity Index",
+        "description": "An extension of SSIM that measures image quality at multiple scales, providing a more robust quality assessment that better mimics human visual perception. Values range from 0 to 1, with higher values indicating better quality.",
+        "link": "https://ieeexplore.ieee.org/document/1292216",
+    },
+    "lpips": {
+        "full_name": "Learned Perceptual Image Patch Similarity",
+        "description": "A perceptual similarity metric that uses deep neural networks to better approximate human perception. Lower values indicate greater similarity, with 0 representing identical images.",
+        "link": "https://arxiv.org/abs/1801.03924",
+    },
+    "vmaf": {
+        "full_name": "Video Multi-method Assessment Fusion",
+        "description": "A machine learning-based video quality metric developed by Netflix that combines multiple quality measurements to better correlate with human perception. Scores range from 0 to 100, with higher values indicating better quality.",
+        "link": "https://github.com/Netflix/vmaf",
+    },
+    "decoder_time": {
+        "full_name": "Decoder Time",
+        "description": "The time (in seconds) taken by the watermark decoder to extract the watermark from the media. Lower values indicate more efficient decoding.",
+        "link": "",
+    },
+    "bit_acc": {
+        "full_name": "Bit Accuracy",
+        "description": "The percentage of bits correctly decoded from a watermark. Values range from 0 to 1, with 1 indicating perfect extraction of the watermark message.",
+        "link": "",
+    },
+    "word_acc": {
+        "full_name": "Word Accuracy",
+        "description": "A binary metric indicating whether the entire watermark message was correctly decoded (True) or not (False).",
+        "link": "",
+    },
+    "log10_p_value": {
+        "full_name": "Log10 P-Value",
+        "description": "The logarithm (base 10) of the probability that a decoded watermark could have occurred by chance. More negative values indicate stronger confidence that a real watermark was detected. This metric provides a better comparison that bit accuracy because it fairly compares different message sizes.",
+        "link": "",
+    },
+    "tpr": {
+        "full_name": "True Positive Rate",
+        "description": "The proportion of watermarked media correctly identified as containing a watermark. Also known as sensitivity or recall. Values range from 0 to 1, with higher values indicating better detection performance.",
+        "link": "https://en.wikipedia.org/wiki/Sensitivity_and_specificity",
+    },
+    "fpr": {
+        "full_name": "False Positive Rate",
+        "description": "The proportion of unwatermarked media incorrectly identified as containing a watermark. Values range from 0 to 1, with lower values indicating better detection performance.",
+        "link": "https://en.wikipedia.org/wiki/False_positive_rate",
+    },
+    "watermark_det_score": {
+        "full_name": "Watermark Detection Score",
+        "description": "A confidence score indicating the system's certainty that a watermark is present. Values typically range from 0 to 1, with higher values indicating greater confidence in watermark detection.",
+        "link": "",
+    },
+}
+
+MODEL_DESCRIPTIONS = {
+    "audioseal": {
+        "full_name": "AudioSeal",
+        "description": "AudioSeal is the first audio watermarking technique designed specifically for localized detection of AI-generated speech.",
+        "paper_link": "https://arxiv.org/abs/2401.17264",
+        "github_link": "https://github.com/facebookresearch/audioseal",
+    },
+    "wavmark": {
+        "full_name": "WavMark",
+        "description": "WavMark uses invertible networks to hide 32 bits in 1-second audio segments. Detection is performed by sliding along the audio in 0.05-second steps and decoding the message for each window. If the first 10 decoded bits match a synchronization pattern, the rest of the payload is saved (22 bits), and the window can directly slide 1 second (instead of 0.05 seconds).",
+        "paper_link": "https://arxiv.org/abs/2308.12770",
+        "github_link": "https://github.com/swesterfeld/audiowmark",
+    },
+    "timbre": {
+        "full_name": "Timbre",
+        "description": "Timbre embeds the watermark into the frequency domain, which is inherently robust against common data processing methods.",
+        "paper_link": "https://arxiv.org/abs/2312.03410",
+        "github_link": "https://github.com/TimbreWatermarking/TimbreWatermarking",
+    },
+    "wam": {
+        "full_name": "Watermark Anything Model",
+        "description": "The Watermark Anything Model (WAM) is designed for localized image watermarking.",
+        "paper_link": "https://arxiv.org/abs/2411.07231",
+        "github_link": "https://github.com/facebookresearch/watermark-anything",
+    },
+    "trustmark": {
+        "full_name": "TrustMark",
+        "description": "TrustMark is a GAN-based watermarking method.",
+        "paper_link": "https://arxiv.org/abs/2311.18297",
+        "github_link": "https://github.com/adobe/trustmark",
+    },
+    "ssl": {
+        "full_name": "Self-Supervised Latent Spaces",
+        "description": "This approach revisits watermarking techniques using pre-trained deep networks and self-supervised methods to embed marks and binary messages into latent spaces.",
+        "paper_link": "https://arxiv.org/abs/2112.09581",
+        "github_link": "https://github.com/facebookresearch/ssl_watermarking",
+    },
+    "fnns": {
+        "full_name": "Fixed Neural Network Steganography",
+        "description": "This approach revisits steganography through adversarial perturbation: it modifies the image such that a fixed decoder correctly outputs the desired message (similar to SSL but with a different network).",
+        "paper_link": "https://openreview.net/pdf?id=hcMvApxGSzZ",
+        "github_link": "https://github.com/varshakishore/FNNS",
+    },
+    "hidden": {
+        "full_name": "Hiding Data With Deep Networks",
+        "description": "First deep watermarking approach from 2018. We use the model trained and open-sourced here, which uses the same architecture and a similar training procedure. Note that this implementation uses a Just Noticeable Difference heatmap to modulate the watermark distortion for less visibility instead of using a perceptual loss during training like in the original paper.",
+        "paper_link": "https://arxiv.org/abs/1807.09937",
+        # "github_link": "https://github.com/ando-khachatryan/HiDDeN",
+    },
+    "dct_dwt": {
+        "full_name": "Combined DCT-DWT",
+        "description": "The algorithm watermarks a given image using a combination of the Discrete Wavelet Transform (DWT) and the Discrete Cosine Transform (DCT). Performance evaluation results show that combining the two transforms improved the performance of the watermarking algorithms that are based solely on the DWT transform.",
+        "paper_link": "https://pdfs.semanticscholar.org/1c47/f281c00cffad4e30deff48a922553cb04d17.pdf",
+        # "github_link": "",
+    },
+    # Fill this in, these are autogenned
+    "cine": {
+        "full_name": "CINE-Watermarking",
+        "description": "A network inversion-based watermarking method for high-resolution images, offering strong robustness against various image transformations while maintaining visual quality.",
+        "paper_link": "https://arxiv.org/abs/2402.19414",
+        "github_link": "",
+    },
+    "mbrs": {
+        "full_name": "Model-Based Robust Steganography (MBRS)",
+        "description": "A deep learning approach for robust image steganography that maintains high image quality while being resilient to common image distortions.",
+        "paper_link": "https://arxiv.org/abs/2204.12677",
+        "github_link": "https://github.com/jzyustc/mbrs",
+    },
+    "videoseal": {
+        "full_name": "VideoSeal",
+        "description": "A neural video watermarking system designed to embed imperceptible watermarks that are robust against common video manipulations and processing operations.",
+        "paper_link": "https://arxiv.org/abs/2405.20977",
+        "github_link": "https://github.com/facebookresearch/audioseal",
+    },
+    "rivagan": {
+        "full_name": "RivaGAN",
+        "description": "A GAN-based approach for robust invisible video watermarking that maintains high visual quality while providing resistance against common video attacks and transformations.",
+        "paper_link": "https://arxiv.org/abs/2102.05551",
+        "github_link": "https://github.com/eyalnakar/RivaGAN",
+    },
+}

frontend/.env.prod

@@ -0,0 +1 @@
+VITE_API_SERVER_URL=""

frontend/dist/index.html

@@ -5,7 +5,7 @@
     <link rel="icon" type="image/svg+xml" href="/vite.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>🥇 Omni Seal Bench Watermarking Leaderboard</title>
-    <script type="module" crossorigin src="/assets/index-UqKblgA6.js"></script>
+    <script type="module" crossorigin src="/assets/index-B4l7vkQT.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-CUEVu_CX.css">
   </head>
   <body>

frontend/src/API.ts

@@ -36,6 +36,13 @@ class API {
     if (!response.ok) throw new Error('Failed to fetch datasets')
     return response.json()
   }
+
+  // Fetch descriptions and model descriptions from the backend
+  static async fetchDescriptions(): Promise<{ descriptions: any; model_descriptions: any }> {
+    const response = await fetch(VITE_API_SERVER_URL + '/descriptions')
+    if (!response.ok) throw new Error('Failed to fetch descriptions')
+    return response.json()
+  }
 }
 
 export default API

frontend/src/App.tsx

@@ -10,6 +10,7 @@ import {
 import LeaderboardPage from './components/LeaderboardPage'
 import Examples from './components/Examples'
 import Docs from './components/Docs'
+import Descriptions from './Descriptions'
 
 const TABS = [
   { label: 'Audio', type: 'audio-leaderboard', path: '/audio-leaderboard' },
@@ -44,6 +45,11 @@ function AppContent() {
   const location = useLocation()
   const [theme, setTheme] = useState<'dark' | 'light'>('dark')
 
+  // Load descriptions on app load
+  useEffect(() => {
+    Descriptions.getInstance().load()
+  }, [])
+
   useEffect(() => {
     document.documentElement.setAttribute('data-theme', theme)
   }, [theme])

frontend/src/Descriptions.ts

@@ -0,0 +1,59 @@
+import API from './API'
+
+class Descriptions {
+  private static instance: Descriptions | null = null
+  private descriptions: Record<string, any> = {}
+  private modelDescriptions: Record<string, any> = {}
+  private loaded: boolean = false
+  private loadingPromise: Promise<void> | null = null
+
+  private constructor() {}
+
+  static getInstance(): Descriptions {
+    if (!Descriptions.instance) {
+      Descriptions.instance = new Descriptions()
+    }
+    return Descriptions.instance
+  }
+
+  async load(): Promise<void> {
+    if (this.loaded) return
+    if (this.loadingPromise) return this.loadingPromise
+    this.loadingPromise = API.fetchDescriptions().then((data) => {
+      this.descriptions = data.descriptions || {}
+      this.modelDescriptions = data.model_descriptions || {}
+      this.loaded = true
+    })
+    return this.loadingPromise
+  }
+
+  getFullName(name: string): string | undefined {
+    return this.descriptions[name]?.full_name
+  }
+
+  getDescription(name: string): string | undefined {
+    return this.descriptions[name]?.description
+  }
+
+  getUrl(name: string): string | undefined {
+    return this.descriptions[name]?.link
+  }
+
+  getModelDescription(name: string): string | undefined {
+    return this.modelDescriptions[name]?.description
+  }
+
+  getModelFullName(name: string): string | undefined {
+    return this.modelDescriptions[name]?.full_name
+  }
+
+  getModelPaperUrl(name: string): string | undefined {
+    return this.modelDescriptions[name]?.paper_link
+  }
+
+  getModelGithubUrl(name: string): string | undefined {
+    return this.modelDescriptions[name]?.github_link
+  }
+}
+
+export default Descriptions