Upload 15 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+LUT-Fuse-main/assets/framework.png filter=lfs diff=lfs merge=lfs -text

LUT-Fuse-main/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Yibing Zhang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LUT-Fuse-main/README.md

@@ -0,0 +1,84 @@
+<h1 align="center">[ICCV 2025] LUT-Fuse</h1>
+<p align="center">
+  <em>Towards Extremely Fast Infrared and Visible Image Fusion via Distillation to Learnable Look-Up Tables</em>
+</p>
+
+<p align="center">
+  <a href="https://github.com/zyb5/LUT-Fuse" style="text-decoration:none;">
+    <img src="https://img.shields.io/badge/GitHub-Code-black?logo=github" alt="Code" />
+  </a>
+  <a href="https://arxiv.org/abs/2509.00346" style="text-decoration:none; margin-left:8px;">
+    <img src="https://img.shields.io/badge/arXiv-Paper-B31B1B?logo=arxiv" alt="Paper" />
+  </a>
+</p>
+
+<p align="center">
+  <img src="assets/framework.png" alt="LUT-Fuse Framework" width="90%">
+</p>
+
+---
+
+## ⚙️ Environment
+
+```
+conda create -n lutfuse python=3.8
+conda activate lutfuse
+```
+
+```
+conda install pytorch==2.0.0 torchvision==0.15.0 pytorch-cuda=11.8 -c pytorch -c nvidia
+pip install -r requirements.txt
+```
+
+## 📂 Dataset
+
+You should list your dataset as followed rule:
+
+```
+|dataset
+  |train
+    |Infrared
+    |Visible
+    |Fuse_ref
+  |test
+    |Infrared
+    |Visible
+    |Fuse_ref
+```
+
+## 💾 Checkpoints
+
+We provide our **pretrained checkpoints** directly in this repository for convenience.  
+You can find them under [`./ckpts`](./ckpts).
+
+- **Fusion LUT weights:** `ckpts/fine_tuned_lut.npy`  
+- **Context generator weights:** `ckpts/generator_context.pth`
+
+## 🧪 Test
+
+```
+CUDA_VISIBLE_DEVICES=0 python test_lut.py
+```
+
+## 🚀 Train
+
+```
+CUDA_VISIBLE_DEVICES=0 python fine_tune_lut.py
+```
+
+## 📖 Citation
+
+If you find our work or dataset useful for your research, please cite our paper.
+
+```bibtex
+@inproceedings{yi2025LUT-Fuse,
+  title={LUT-Fuse: Towards Extremely Fast Infrared and Visible Image Fusion via Distillation to Learnable Look-Up Tables},
+  author={Yi, Xunpeng and Zhang, Yibing and Xiang, Xinyu and Yan, Qinglong and Xu, Han and Ma, Jiayi},
+  booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
+  year={2025}
+}
+```
+
+If you have any questions, please send an email to zhangyibing@whu.edu.cn
+
+

LUT-Fuse-main/ckpts/fine_tuned_lut.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f70326cb33b27285e5157594d31a79ee94b67b043828cb98a7d60aeec920e4
+size 262272

LUT-Fuse-main/ckpts/fine_tuned_lut_original.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ffa74433659b3a0d8e39ee1f2f4fda2d776ad2967a4e1d9190a67e314099a43
+size 262272

LUT-Fuse-main/ckpts/generator_context.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fcacc0280f6ee37d42ae7b82166484f9fe1660526c647c560e051ff23885324
+size 38143

LUT-Fuse-main/ckpts/generator_context_original.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7a4ec41712435defb95d3c9cfc23839cf79bc00ae22ba47cc5755d480525780
+size 37559

LUT-Fuse-main/data/o_fusion_dataset.py

@@ -0,0 +1,64 @@
+from PIL import Image
+import torch
+from torch.utils.data import Dataset
+import os
+from glob import glob
+from torchvision.transforms import RandomCrop
+import torchvision.transforms.functional as F
+
+class RandomCropPair:
+    def __init__(self, size):
+        self.size = size  # 裁剪尺寸 (h, w)
+
+    def __call__(self, vis_img, ir_img, fuse_image):
+        # 获取随机裁剪参数
+        i, j, h, w = RandomCrop.get_params(vis_img, output_size=self.size)
+        # 对可见光和红外图像使用相同的裁剪参数
+        vis_img = F.crop(vis_img, i, j, h, w)
+        ir_img = F.crop(ir_img, i, j, h, w)
+        fuse_image = F.crop(fuse_image, i, j, h, w)
+
+        vis_img = F.to_tensor(vis_img)
+        ir_img = F.to_tensor(ir_img)
+        fuse_image = F.to_tensor(fuse_image)
+        return vis_img, ir_img, fuse_image
+
+class DistillDataSet(Dataset):
+    def __init__(self, visible_path, infrared_path, other_fuse_path, phase="train", transform=None):
+        self.phase = phase
+        self.visible_files = sorted(glob(os.path.join(visible_path, "*.*")))
+        self.infrared_files = sorted(glob(os.path.join(infrared_path, "*.*")))
+        self.other_fuse_files = sorted(glob(os.path.join(other_fuse_path, "*.*")))
+        self.transform = transform
+
+    def __len__(self):
+        l = len(self.infrared_files)
+        return l
+
+    def __getitem__(self, item):
+        image_A_path = self.visible_files[item]
+        image_B_path = self.infrared_files[item]
+        other_fuse_path = self.other_fuse_files[item]
+        image_A = Image.open(image_A_path).convert(mode='RGB')
+        image_B = Image.open(image_B_path).convert(mode='L')   ##########
+        other_fuse = Image.open(other_fuse_path).convert(mode='RGB')
+
+        if self.transform is not None:
+            if isinstance(self.transform, RandomCropPair):
+                image_A, image_B, other_fuse = self.transform(image_A, image_B, other_fuse)
+            else:
+                image_A = self.transform(image_A)
+                image_B = self.transform(image_B)
+                other_fuse = self.transform(other_fuse)
+
+        name = image_A_path.replace("\\", "/").split("/")[-1].split(".")[0]
+
+        return image_A, image_B, other_fuse, name
+
+    @staticmethod
+    def collate_fn(batch):
+        images_A, images_B, other_fuse, name = zip(*batch)
+        images_A = torch.stack(images_A, dim=0)
+        images_B = torch.stack(images_B, dim=0)
+        other_fuse = torch.stack(other_fuse, dim=0)
+        return images_A, images_B, other_fuse, name

LUT-Fuse-main/data/simple_dataset.py

@@ -0,0 +1,218 @@
+from PIL import Image
+import torch
+from torch.utils.data import Dataset
+import os
+from glob import glob
+import transforms as T
+from torchvision.transforms import RandomCrop
+import torchvision.transforms.functional as F
+
+# class SimpleDataSet(Dataset):
+#     def __init__(self, visible_path, visible_gt_path, infrared_path, phase="train", transform=None):
+#         self.phase = phase
+#         self.visible_files = sorted(glob(os.path.join(visible_path, "*")))
+#         self.visible_gt_files = sorted(glob(os.path.join(visible_gt_path, "*")))
+#         self.infrared_files = sorted(glob(os.path.join(infrared_path, "*")))
+#         self.transform = transform
+#
+#     def __len__(self):
+#         l = len(self.infrared_files)
+#         return l
+#
+#     def __getitem__(self, item):
+#         image_A_path = self.visible_files[item]
+#         image_A_gt_path = self.visible_gt_files[item]
+#         image_B_path = self.infrared_files[item]
+#         image_A = Image.open(image_A_path).convert(mode='RGB')
+#         image_A_gt = Image.open(image_A_gt_path).convert(mode='RGB')
+#         image_B = Image.open(image_B_path).convert(mode='L')   ##########
+#
+#         image_A = self.transform(image_A)
+#         image_A_gt = self.transform(image_A_gt)
+#         image_B = self.transform(image_B)
+#
+#         name = image_A_path.replace("\\", "/").split("/")[-1].split(".")[0]
+#
+#         return image_A, image_A_gt, image_B, name
+#
+#     @staticmethod
+#     def collate_fn(batch):
+#         images_A, image_A_gt, images_B, name = zip(*batch)
+#         images_A = torch.stack(images_A, dim=0)
+#         image_A_gt = torch.stack(image_A_gt, dim=0)
+#         images_B = torch.stack(images_B, dim=0)
+#         return images_A, image_A_gt, images_B, name
+
+# class RandomCropPair:
+#     def __init__(self, size):
+#         self.size = size  # 裁剪尺寸 (h, w)
+#
+#     def __call__(self, vis_img, ir_img):
+#         # 获取随机裁剪参数
+#         i, j, h, w = RandomCrop.get_params(vis_img, output_size=self.size)
+#         # 对可见光和红外图像使用相同的裁剪参数
+#         vis_img = F.crop(vis_img, i, j, h, w)
+#         ir_img = F.crop(ir_img, i, j, h, w)
+#
+#         vis_img = F.to_tensor(vis_img)
+#         ir_img = F.to_tensor(ir_img)
+#         return vis_img, ir_img
+#
+# class SimpleDataSet(Dataset):
+#     def __init__(self, visible_path, infrared_path, phase="train", transform=None):
+#         self.phase = phase
+#         self.visible_files = sorted(glob(os.path.join(visible_path, "*.*")))
+#         self.infrared_files = sorted(glob(os.path.join(infrared_path, "*.*")))
+#         self.transform = transform
+#
+#     def __len__(self):
+#         l = len(self.infrared_files)
+#         return l
+#
+#     def __getitem__(self, item):
+#         image_A_path = self.visible_files[item]
+#         image_B_path = self.infrared_files[item]
+#         image_A = Image.open(image_A_path).convert(mode='RGB')
+#         image_B = Image.open(image_B_path).convert(mode='L')   ##########
+#
+#         # image_A = self.transform(image_A)
+#         # image_B = self.transform(image_B)
+#
+#         if self.transform is not None:
+#             if isinstance(self.transform, RandomCropPair):
+#                 image_A, image_B = self.transform(image_A, image_B)
+#             else:
+#                 image_A = self.transform(image_A)
+#                 image_B = self.transform(image_B)
+#
+#         name = image_A_path.replace("\\", "/").split("/")[-1].split(".")[0]
+#
+#         return image_A, image_B, name
+#
+#     @staticmethod
+#     def collate_fn(batch):
+#         images_A, images_B, name = zip(*batch)
+#         images_A = torch.stack(images_A, dim=0)
+#         images_B = torch.stack(images_B, dim=0)
+#         return images_A, images_B, name
+#
+#
+# class RandomCropPair:
+#     def __init__(self, size):
+#         self.size = size  # 裁剪尺寸 (h, w)
+#
+#     def __call__(self, vis_blur_img, ir_blur_img, vis_gt_img, ir_gt_img):
+#         # 获取随机裁剪参数
+#         i, j, h, w = RandomCrop.get_params(vis_blur_img, output_size=self.size)
+#         # 对可见光和红外图像使用相同的裁剪参数
+#         vis_blur_img = F.crop(vis_blur_img, i, j, h, w)
+#         ir_blur_img = F.crop(ir_blur_img, i, j, h, w)
+#         vis_gt_img = F.crop(vis_gt_img, i, j, h, w)
+#         ir_gt_img = F.crop(ir_gt_img, i, j, h, w)
+#
+#         vis_blur_img = F.to_tensor(vis_blur_img)
+#         ir_blur_img = F.to_tensor(ir_blur_img)
+#         vis_gt_img = F.to_tensor(vis_gt_img)
+#         ir_gt_img = F.to_tensor(ir_gt_img)
+#         return vis_blur_img, ir_blur_img, vis_gt_img, ir_gt_img
+#
+# class SimpleDataSet(Dataset):
+#     def __init__(self, visible_blur_path, infrared_blur_path, visible_gt_path, infrared_gt_path, phase="train", transform=None):
+#         self.phase = phase
+#         self.visible_blur_files = sorted(glob(os.path.join(visible_blur_path, "*.*")))
+#         self.infrared_blur_files = sorted(glob(os.path.join(infrared_blur_path, "*.*")))
+#         self.visible_gt_files = sorted(glob(os.path.join(visible_gt_path, "*.*")))
+#         self.infrared_gt_files = sorted(glob(os.path.join(infrared_gt_path, "*.*")))
+#         self.transform = transform
+#
+#     def __len__(self):
+#         l = len(self.infrared_gt_files)
+#         return l
+#
+#     def __getitem__(self, item):
+#         image_A_blur_path = self.visible_blur_files[item]
+#         image_B_blur_path = self.infrared_blur_files[item]
+#         image_A_gt_path = self.visible_gt_files[item]
+#         image_B_gt_path = self.infrared_gt_files[item]
+#         image_A_blur = Image.open(image_A_blur_path).convert(mode='RGB')
+#         image_B_blur = Image.open(image_B_blur_path).convert(mode='L')   ##########
+#         image_A_gt = Image.open(image_A_gt_path).convert(mode='RGB')
+#         image_B_gt = Image.open(image_B_gt_path).convert(mode='L')  ##########
+#
+#         if self.transform is not None:
+#             if isinstance(self.transform, RandomCropPair):
+#                 image_A_blur, image_B_blur, image_A_gt, image_B_gt = self.transform(image_A_blur, image_B_blur, image_A_gt, image_B_gt)
+#             else:
+#                 image_A_blur = self.transform(image_A_blur)
+#                 image_B_blur = self.transform(image_B_blur)
+#                 image_A_gt = self.transform(image_A_gt)
+#                 image_B_gt = self.transform(image_B_gt)
+#
+#         name = image_A_blur_path.replace("\\", "/").split("/")[-1].split(".")[0]
+#
+#         return image_A_blur, image_B_blur, image_A_gt, image_B_gt, name
+#
+#     @staticmethod
+#     def collate_fn(batch):
+#         image_A_blur, image_B_blur, image_A_gt, image_B_gt, name = zip(*batch)
+#         image_A_blur = torch.stack(image_A_blur, dim=0)
+#         image_B_blur = torch.stack(image_B_blur, dim=0)
+#         image_A_gt = torch.stack(image_A_gt, dim=0)
+#         image_B_gt = torch.stack(image_B_gt, dim=0)
+#         return image_A_blur, image_B_blur, image_A_gt, image_B_gt, name
+
+
+class RandomCropPair:
+    def __init__(self, size):
+        self.size = size  # 裁剪尺寸 (h, w)
+
+    def __call__(self, vis_img, ir_img):
+        # 获取随机裁剪参数
+        i, j, h, w = RandomCrop.get_params(vis_img, output_size=self.size)
+        # 对可见光和红外图像使用相同的裁剪参数
+        vis_img = F.crop(vis_img, i, j, h, w)
+        ir_img = F.crop(ir_img, i, j, h, w)
+
+        vis_img = F.to_tensor(vis_img)
+        ir_img = F.to_tensor(ir_img)
+        return vis_img, ir_img
+
+class SimpleDataSet(Dataset):
+    def __init__(self, visible_path, infrared_path, phase="train", transform=None):
+        self.phase = phase
+        self.visible_files = sorted(glob(os.path.join(visible_path, "*.*")))
+        self.infrared_files = sorted(glob(os.path.join(infrared_path, "*.*")))
+        self.transform = transform
+
+    def __len__(self):
+        l = len(self.infrared_files)
+        return l
+
+    def __getitem__(self, item):
+        image_A_path = self.visible_files[item]
+        image_B_path = self.infrared_files[item]
+        image_A = Image.open(image_A_path).convert(mode='RGB')
+        image_B = Image.open(image_B_path).convert(mode='L')   ##########
+
+        # image_A = self.transform(image_A)
+        # image_B = self.transform(image_B)
+
+        if self.transform is not None:
+            if isinstance(self.transform, RandomCropPair):
+                image_A, image_B = self.transform(image_A, image_B)
+            else:
+                image_A = self.transform(image_A)
+                image_B = self.transform(image_B)
+
+        name = image_A_path.replace("\\", "/").split("/")[-1].split(".")[0]
+
+        return image_A, image_B, name
+
+    @staticmethod
+    def collate_fn(batch):
+        images_A, images_B, name = zip(*batch)
+        images_A = torch.stack(images_A, dim=0)
+        images_B = torch.stack(images_B, dim=0)
+        return images_A, images_B, name
+
+

LUT-Fuse-main/fine_tune_lut.py

@@ -0,0 +1,264 @@
+import torch
+import torch.nn.functional as F
+import torch.optim as optim
+import numpy as np
+import torch.nn as nn
+from torch.utils.tensorboard import SummaryWriter
+
+from data.o_fusion_dataset import DistillDataSet
+from data.o_fusion_dataset import RandomCropPair
+import datetime
+import os
+
+import transforms as T
+from scripts.loss_lut import fusion_loss
+from itertools import chain
+from scripts.calculate import OptimizableLUT, Generator_for_info, apply_fusion_4d_with_interpolation
+
+cuda = True if torch.cuda.is_available() else False
+Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
+
+
+class TV_4D(nn.Module):
+    def __init__(self, dim=16, output_channels=3):
+        super(TV_4D, self).__init__()
+
+        self.weight_r = torch.ones(dim, dim, dim, dim - 1, output_channels, dtype=torch.float)
+        self.weight_r[..., (0, dim - 2), :] *= 2.0
+
+        self.weight_g = torch.ones(dim, dim, dim - 1, dim, output_channels, dtype=torch.float)
+        self.weight_g[..., (0, dim - 2), :, :] *= 2.0
+
+        self.weight_b = torch.ones(dim, dim - 1, dim, dim, output_channels, dtype=torch.float)
+        self.weight_b[..., (0, dim - 2), :, :, :] *= 2.0
+
+        self.weight_ir = torch.ones(dim - 1, dim, dim, dim, output_channels, dtype=torch.float)
+        self.weight_ir[(0, dim - 2), :, :, :, :] *= 2.0
+
+        self.relu = torch.nn.ReLU()
+
+    def forward(self, LUT):
+        device = LUT.device
+
+        self.weight_r = self.weight_r.to(device)
+
+        self.weight_g = self.weight_g.to(device)
+        self.weight_b = self.weight_b.to(device)
+        self.weight_ir = self.weight_ir.to(device)
+
+        dif_r = LUT[ :, :, :, :-1, :] - LUT[ :, :, :, 1:, :]  
+        dif_g = LUT[ :, :, :-1, :, :] - LUT[ :, :, 1:, :, :]  
+        dif_b = LUT[ :, :-1, :, :, :] - LUT[ :, 1:, :, :, :]  
+        dif_ir = LUT[ :-1, :, :, :, :] - LUT[ 1:, :, :, :, :]  
+
+        tv = (torch.mean(torch.mul(dif_r ** 2, self.weight_r)) + torch.mean(torch.mul(dif_g ** 2, self.weight_g)) +
+              torch.mean(torch.mul(dif_b ** 2, self.weight_b)) + torch.mean(torch.mul(dif_ir ** 2, self.weight_ir)))
+
+        mn = (torch.mean(self.relu(dif_r)) + torch.mean(self.relu(dif_g)) +
+              torch.mean(self.relu(dif_b)) + torch.mean(self.relu(dif_ir)))
+
+        return tv, mn
+
+
+def fine_tune_lut(lut_model, Generator_context, train_loader, val_loader, device, epochs, learning_rate, save_dir="ww"):
+    TV4 = TV_4D().to(device)
+    best_val_loss = 1e5
+    Generator_context.train()
+    loss_fuction = fusion_loss()
+    optimizer = optim.Adam(chain(lut_model.parameters(), Generator_context.parameters()), lr=learning_rate)
+    # optimizer = optim.Adam(lut_model.parameters(), lr=learning_rate)
+
+    for epoch in range(epochs):
+        lut_model.train()
+
+        train_loss = 0
+        # train_loss_max = 0
+        # train_loss_text = 0
+        train_loss_l1 = 0
+        train_loss_ssim = 0
+        train_loss_tv0 = 0
+        train_loss_mn0 = 0
+
+        for step, data in enumerate(train_loader):
+            I_A, I_B, fuse, _ = data
+            # optimizer.zero_grad()
+
+            if torch.cuda.is_available():
+                I_A = I_A.to(device)
+                I_B = I_B.to(device)
+                high_quality = fuse.to(device)
+                loss_fuction = loss_fuction.to(device)
+
+            lut = lut_model()
+
+            tv0, mn0 = TV4(lut)
+            loss_tv0 = tv0
+            loss_mn0 = mn0
+
+            outputs = apply_fusion_4d_with_interpolation(I_A * 255., I_B * 255., lut, Generator_context)
+
+            l1 = F.l1_loss(outputs, high_quality)
+            ssim = loss_fuction(I_A, I_B, outputs)
+            loss_all = l1 + ssim + 10.0 * loss_mn0 + 0.0001 * loss_tv0 #+ text_loss + loss_max
+
+            loss_all.backward()
+            optimizer.step()
+
+            train_loss += loss_all.item()
+            train_loss_l1 += l1.item()
+            train_loss_ssim += ssim.item()
+            # train_loss_text += text_loss.item()
+            # train_loss_max += loss_max.item()
+            train_loss_tv0 += loss_tv0.item()
+            train_loss_mn0 += loss_mn0.item()
+            # train_loss_color += loss_color.item()
+
+        tb_writer.add_scalar("train_total_loss", train_loss/len(train_loader), epoch)
+        tb_writer.add_scalar("train_loss_l1", train_loss_l1/len(train_loader), epoch)
+        tb_writer.add_scalar("train_loss_ssim", train_loss_ssim / len(train_loader), epoch)
+        # tb_writer.add_scalar("train_loss_text", train_loss_text / len(train_loader), epoch)
+        # tb_writer.add_scalar("train_loss_max", train_loss_max / len(train_loader), epoch)
+        tb_writer.add_scalar("train_loss_tv0", train_loss_tv0/len(train_loader), epoch)
+        tb_writer.add_scalar("train_loss_mn0", train_loss_mn0/len(train_loader), epoch)
+
+        print(f"Epoch {epoch + 1}/{epochs} - Loss: {train_loss / len(train_loader):.6f} - loss_l1: {train_loss_l1 / len(train_loader):.6f} - loss_ssim: {train_loss_ssim / len(train_loader):.6f} - loss_tv: {train_loss_tv0 / len(train_loader):.6f} - loss_tv: {train_loss_tv0 / len(train_loader):.6f} ")
+        # print(f"Epoch {epoch + 1}/{epochs} - Loss: {train_loss / len(train_loader):.6f} - l1: {train_loss_l1 / len(train_loader):.6f} - loss_text: {train_loss_text / len(train_loader):.6f} - loss_max: {train_loss_max / len(train_loader):.6f}")
+        # print(f"Epoch {epoch + 1}/{epochs} - Loss: {train_loss / len(train_loader):.6f} - l1: {train_loss_l1 / len(train_loader):.6f} - tv: {train_loss_tv0 / len(train_loader):.6f} - mn: {train_loss_mn0 / len(train_loader):.6f}")
+
+        if (epoch + 1) % 10 == 0:
+            val_loss, val_loss_l1, val_loss_ssim, val_loss_tv0, val_loss_mn0 = validate_lut(lut_model, val_loader, device)
+            tb_writer.add_scalar("val_total_loss", val_loss/len(val_loader), epoch)
+            tb_writer.add_scalar("val_loss_l1", val_loss_l1/len(val_loader), epoch)
+            tb_writer.add_scalar("val_loss_ssim", val_loss_ssim / len(val_loader), epoch)
+            # tb_writer.add_scalar("val_loss_text", val_loss_text / len(val_loader), epoch)
+            # tb_writer.add_scalar("val_loss_max", val_loss_max / len(val_loader), epoch)
+            tb_writer.add_scalar("val_loss_tv0", val_loss_tv0/len(val_loader), epoch)
+            tb_writer.add_scalar("val_loss_mn0", val_loss_mn0/len(val_loader), epoch)
+            # print(f"Validation - Epoch {epoch} - Loss: {val_loss / len(val_loader):.6f} - l1: {val_loss_l1 / len(val_loader):.6f} - tv: {val_loss_tv0 / len(val_loader):.6f} - mn: {val_loss_mn0 / len(val_loader):.6f}")
+
+            if val_loss < best_val_loss :
+                best_val_loss = val_loss
+                filename = f"fine_tuned_ygcy_epoch{epoch}_valloss{val_loss:.6f}.npy"
+                full_path = os.path.join(filefold_path, filename)
+                save_lut(lut_model, full_path)
+
+                context_filename = f"generator_context_epoch{epoch}_valloss{val_loss:.6f}.pth"
+                generator_context_save_path = os.path.join(filefold_path, context_filename)
+                save_generator_context(Generator_context, save_path=generator_context_save_path)
+
+            print(f"Validation - Epoch {epoch} - Loss: {val_loss / len(val_loader):.6f} - l1: {val_loss_l1 / len(val_loader):.6f} - loss_ssim: {val_loss_ssim / len(val_loader):.6f} - loss_tv0: {val_loss_tv0 / len(val_loader):.6f} - loss_mn0: {val_loss_mn0 / len(val_loader):.6f}")
+
+
+def save_lut(lut_module, path):
+
+    lut_weights = lut_module().detach().cpu().numpy()
+    np.save(path, lut_weights)
+    print(f"Fine-tuned LUT saved to {path}")
+
+
+def validate_lut(lut_module, val_loader, device):
+    train_loss = 0
+    train_loss_mn0 = 0
+    train_loss_tv0 = 0
+    train_loss_ssim = 0
+    train_loss_l1 = 0
+    # train_loss_text = 0
+    TV4 = TV_4D().to(device)
+
+    loss_fuction = fusion_loss()
+    Generator_context.eval()
+
+    lut = lut_module()
+    with torch.no_grad():
+        for step, data in enumerate(val_loader):
+            I_A, I_B, fuse, task = data
+            if torch.cuda.is_available():
+                I_A = I_A.to(device)
+                I_B = I_B.to(device)
+                high_quality = fuse.to(device)
+                loss_fuction = loss_fuction.to(device)
+
+            outputs = apply_fusion_4d_with_interpolation(I_A * 255., I_B * 255., lut, Generator_context)
+            tv0, mn0 = TV4(lut)
+            loss_tv0 = tv0
+            loss_mn0 = mn0
+            l1 = F.l1_loss(outputs, high_quality)
+            loss_ssim = loss_fuction(I_A, I_B, outputs)
+            loss_all = l1 + loss_ssim + 0.1 * loss_mn0 + 10.0 * loss_tv0 #+ text_loss + max_loss
+
+            train_loss += loss_all.item()
+            train_loss_l1 += l1.item()
+            train_loss_ssim += loss_ssim.item()
+            # train_loss_text += text_loss.item()
+            # train_loss_max += max_loss.item()
+            train_loss_tv0 += loss_tv0.item()
+            train_loss_mn0 += loss_mn0.item()
+
+    return train_loss, train_loss_l1 , train_loss_tv0, train_loss_mn0
+
+
+def save_generator_context(generator_context, save_path="generator_context.pth"):
+    torch.save(generator_context.state_dict(), save_path)
+    print(f"Generator_for_info weights saved to {save_path}")
+
+if __name__ == "__main__":
+
+    if os.path.exists("./finetune_lut_exp") is False:                       
+        os.makedirs("./finetune_lut_exp")
+    file_name = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    filefold_path = "./finetune_lut_exp/finetune_lut_{}".format(file_name)
+    file_log_path = os.path.join(filefold_path, "log")
+    os.makedirs(file_log_path)
+
+    tb_writer = SummaryWriter(log_dir=file_log_path)
+
+    DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    lut_filepath = "ckpts/fine_tuned_lut_original.npy"   
+    lut_tensor = torch.tensor(np.load(lut_filepath).astype(np.float32), device=DEVICE) 
+    lut = OptimizableLUT(lut_tensor) 
+
+    context_file = "ckpts/generator_context_original.pth"
+    Generator_context = Generator_for_info().to(DEVICE)        
+    Generator_context.load_state_dict(torch.load(context_file))
+    # Generator_context.eval()
+
+    batch_size = 6
+    visible_path = " "
+    infrared_path = " "
+    train_fusion_path = " "
+    test_visible_path = " "
+    test_infrared_path = " "
+    test_fusion_path = " "
+    nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8])
+    data_transform = {
+        "train": RandomCropPair(size=(128, 128)),
+        "val": T.Compose([T.Resize_16(),
+                          T.ToTensor()])}
+
+    train_dataset = DistillDataSet(visible_path=visible_path,
+                                  infrared_path=infrared_path,
+                                  other_fuse_path=train_fusion_path,
+                                  phase="train",
+                                  transform=data_transform["train"])
+    train_loader = torch.utils.data.DataLoader(train_dataset,
+                                               batch_size=batch_size,
+                                               shuffle=True,
+                                               pin_memory=True,
+                                               num_workers=nw,
+                                               collate_fn=train_dataset.collate_fn)
+
+    val_dataset = DistillDataSet(visible_path=test_visible_path,
+                                infrared_path=test_infrared_path,
+                                other_fuse_path=test_fusion_path,
+                                phase="val",
+                                transform=data_transform["val"])
+    val_loader = torch.utils.data.DataLoader(val_dataset,
+                                             batch_size=1,
+                                             shuffle=False,
+                                             pin_memory=True,
+                                             num_workers=nw,
+                                             collate_fn=val_dataset.collate_fn)
+
+    fine_tune_lut(lut, Generator_context, train_loader, val_loader, DEVICE, epochs=496, learning_rate=5e-5)
+

LUT-Fuse-main/requirements.txt

@@ -0,0 +1,104 @@
+absl-py==2.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+backcall==0.2.0
+Brotli
+cachetools==5.5.0
+certifi
+charset-normalizer
+contourpy==1.1.1
+cycler==0.12.1
+decorator==5.1.1
+einops==0.8.0
+et_xmlfile==2.0.0
+executing==2.1.0
+filelock
+flatbuffers==24.3.25
+fonttools==4.54.1
+fsspec==2024.12.0
+ftfy==6.2.3
+gast==0.4.0
+gmpy2
+google-auth==2.35.0
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+grpcio==1.66.2
+h5py==3.11.0
+huggingface-hub==0.27.0
+idna
+imageio==2.35.1
+importlib_metadata==8.5.0
+importlib_resources==6.4.5
+ipython==8.12.3
+jedi==0.19.1
+Jinja2
+keras==2.13.1
+kiwisolver==1.4.7
+lazy_loader==0.4
+libclang==18.1.1
+Markdown==3.7
+MarkupSafe
+matplotlib==3.7.5
+matplotlib-inline==0.1.7
+mkl-fft
+mkl-random
+mkl-service==2.4.0
+mpmath
+networkx
+numpy
+nvidia-ml-py==12.535.161
+nvitop==1.3.2
+oauthlib==3.2.2
+opencv-python==4.9.0.80
+openpyxl==3.1.5
+opt_einsum==3.4.0
+packaging==24.1
+pandas==2.0.3
+parso==0.8.4
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow
+prompt_toolkit==3.0.48
+protobuf==4.25.5
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+Pygments==2.18.0
+pyparsing==3.1.4
+PySocks
+python-dateutil==2.9.0.post0
+pytorch-msssim==1.0.0
+pytz==2024.2
+PyWavelets==1.4.1
+PyYAML==6.0.2
+regex==2024.9.11
+requests
+requests-oauthlib==2.0.0
+rsa==4.9
+safetensors==0.4.5
+scikit-image==0.21.0
+scipy==1.10.1
+seaborn==0.13.2
+six==1.16.0
+stack-data==0.6.3
+sympy
+tensorboard==2.13.0
+tensorboard-data-server==0.7.2
+termcolor==2.4.0
+tifffile==2023.7.10
+tokenizers==0.20.3
+torch==2.0.0
+torchvision==0.15.0
+tqdm==4.66.5
+traitlets==5.14.3
+transformers==4.46.3
+triton==2.0.0
+typing_extensions==4.5.0
+tzdata==2024.2
+urllib3
+wcwidth==0.2.13
+Werkzeug==3.0.4
+wrapt==1.16.0
+zipp==3.20.2

LUT-Fuse-main/scripts/calculate.py

@@ -0,0 +1,180 @@
+import torch
+import torch.nn.functional as F
+import numpy as np
+from PIL import Image
+from torchvision.transforms import ToPILImage
+import time
+from data.simple_dataset import RandomCropPair
+from data.simple_dataset import SimpleDataSet
+import torch.nn as nn
+import transforms as T
+import os
+
+
+def rgb_to_ycbcr(img):
+    return torch.stack(
+        (0. / 256. + img[:, 0, :, :] * 0.299000 + img[:, 1, :, :] * 0.587000 + img[:, 2, :, :] * 0.114000,
+               128. / 256. - img[:, 0, :, :] * 0.168736 - img[:, 1, :, :] * 0.331264 + img[:, 2, :, :] * 0.500000,
+               128. / 256. + img[:, 0, :, :] * 0.500000 - img[:, 1, :, :] * 0.418688 - img[:, 2, :, :] * 0.081312),
+               dim=1)
+
+
+def ycbcr_to_rgb(img):
+    return torch.stack(
+         (img[:, 0, :, :] + (img[:, 2, :, :] - 0.5) * 1.402,
+                img[:, 0, :, :] - (img[:, 1, :, :] - 0.5) * 0.344136 - (img[:, 2, :, :] - 0.5) * 0.714136,
+                img[:, 0, :, :] + (img[:, 1, :, :] - 0.5) * 1.772),
+                dim=1)
+
+
+def load_lookup_table(filepath):
+    try:
+        lut = np.load(filepath).astype(np.float32)
+        lut = torch.tensor(lut, device="cuda")  # 将查找表移到 GPU 上
+        return lut
+    except Exception as e:
+        print(f"加载查找表时出错: {e}")
+        return None
+
+
+def generator_block(in_filters, out_filters, normalization=False):
+    """Returns downsampling layers of each discriminator block"""
+    layers = [nn.Conv2d(in_filters, out_filters, 3, stride=1, padding=1)]
+    layers.append(nn.LeakyReLU(0.2))
+    if normalization:
+        layers.append(nn.InstanceNorm2d(out_filters, affine=True))
+        # layers.append(nn.BatchNorm2d(out_filters))
+    return layers
+
+
+class Generator_for_info(nn.Module):
+    def __init__(self, in_channels=4):
+        super(Generator_for_info, self).__init__()
+
+        self.input_layer = nn.Sequential(
+            nn.Conv2d(in_channels, 16, 3, stride=1, padding=1),
+            nn.LeakyReLU(0.2),
+            nn.InstanceNorm2d(16, affine=True),)
+
+        self.mid_layer = nn.Sequential(
+            *generator_block(16, 16, normalization=True),
+            *generator_block(16, 16, normalization=True),
+            *generator_block(16, 16, normalization=True),)
+
+        self.output_layer = nn.Sequential(
+            nn.Dropout(p=0.5),
+            nn.Conv2d(16, 1, 3, stride=1, padding=1),
+            nn.Sigmoid())
+
+    def forward(self, img_input):
+        x = self.input_layer(img_input)
+        identity = x
+        out = self.mid_layer(x)
+        out += identity
+        out = self.output_layer(out)
+        return out
+
+
+def apply_fusion_4d_with_interpolation(visible_img, infrared_img, lut, get_context):
+
+    image_cat = torch.cat((visible_img, infrared_img), dim=1)  # [0, 255]
+    context = get_context(image_cat)
+
+    context_scaled = (context *255./ 16.0).squeeze(1)  # [0, 16]
+    infrared_scaled = infrared_img / 16.0  # [0, 16]
+
+    ycbcr_vis = rgb_to_ycbcr(visible_img / 255.)  # [0, 1]
+    ycbcr_vis_scaled = ycbcr_vis * 255.0 / 16.0  # [0, 16]
+
+    y_vi_scaled = ycbcr_vis_scaled[:, 0, :, :]  # [b, 1, h, w]   # [0, 16]
+    cb_cr = ycbcr_vis[:, 1:, :, :]  # [0, 1]
+    ir_scaled = infrared_scaled[:, 0, :, :]  # [0, 16]
+
+    # 获取floor和ceil索引
+    ir_floor = torch.floor(ir_scaled).long()
+    ir_ceil = torch.clamp(ir_floor + 1, 0, lut.shape[3] - 1)
+    ir_alpha = ir_scaled - ir_floor
+
+    y_vi_floor = torch.floor(y_vi_scaled).long()
+    y_vi_ceil = torch.clamp(y_vi_floor + 1, 0, lut.shape[0] - 1)
+    y_vi_alpha = y_vi_scaled - y_vi_floor
+
+    c_floor = torch.floor(context_scaled).long()
+    c_ceil = torch.clamp(c_floor + 1, 0, lut.shape[2] - 1)
+    c_alpha = context_scaled - c_floor
+
+    sobel_x = torch.tensor([[1, 0, -1], [2, 0, -2], [1, 0, -1]], dtype=torch.float32).unsqueeze(0).unsqueeze(0).to(
+        visible_img.device)
+    sobel_y = torch.tensor([[1, 2, 1], [0, 0, 0], [-1, -2, -1]], dtype=torch.float32).unsqueeze(0).unsqueeze(0).to(
+        visible_img.device)
+
+    # 计算 x 和 y 方向的梯度
+    grad_x = torch.nn.functional.conv2d(ycbcr_vis[:, :1, :, :], sobel_x, padding=1)
+    grad_y = torch.nn.functional.conv2d(ycbcr_vis[:, :1, :, :], sobel_y, padding=1)
+
+    # 计算梯度幅值
+    gradient = torch.sqrt(grad_x ** 2 + grad_y ** 2)  # [b, 1, h, w]
+    min_val = gradient.min(dim=-1, keepdim=True).values.min(dim=-2, keepdim=True).values
+    max_val = gradient.max(dim=-1, keepdim=True).values.max(dim=-2, keepdim=True).values
+
+    gradient_normalized = (gradient - min_val) / (max_val - min_val + 1e-8)
+
+    gradient_scaled = (gradient_normalized * 255.)
+    gradient_scaled = (gradient_scaled / 16.0).squeeze(1)
+
+    g_floor = torch.floor(gradient_scaled).long()
+    g_ceil = torch.clamp(g_floor + 1, 0, lut.shape[1] - 1)
+    g_alpha = gradient_scaled - g_floor
+
+    ir_alpha = ir_alpha.unsqueeze(-1)
+    y_vi_alpha = y_vi_alpha.unsqueeze(-1)
+    g_alpha = g_alpha.unsqueeze(-1)
+    c_alpha = c_alpha.unsqueeze(-1)
+
+    def lerp(v1, v2, alpha):
+        out = v1 * (1 - alpha) + v2 * alpha
+        return out
+
+    fusion_result = (
+        lerp(
+            lerp(
+                lerp(
+                    lerp(lut[y_vi_floor, g_floor, c_floor, ir_floor], lut[y_vi_floor, g_floor, c_floor, ir_ceil],
+                         ir_alpha),
+                    lerp(lut[y_vi_floor, g_floor, c_ceil, ir_floor], lut[y_vi_floor, g_floor, c_ceil, ir_ceil],
+                         ir_alpha),
+                    c_alpha,
+                ),
+                lerp(
+                    lerp(lut[y_vi_floor, g_ceil, c_floor, ir_floor], lut[y_vi_floor, g_ceil, c_floor, ir_ceil],
+                         ir_alpha),
+                    lerp(lut[y_vi_floor, g_ceil, c_ceil, ir_floor], lut[y_vi_floor, g_ceil, c_ceil, ir_ceil], ir_alpha),
+                    c_alpha,
+                ),
+                g_alpha,
+            ),
+            lerp(
+                lerp(
+                    lerp(lut[y_vi_ceil, g_floor, c_floor, ir_floor], lut[y_vi_ceil, g_floor, c_floor, ir_ceil],
+                         ir_alpha),
+                    lerp(lut[y_vi_ceil, g_floor, c_ceil, ir_floor], lut[y_vi_ceil, g_floor, c_ceil, ir_ceil], ir_alpha),
+                    c_alpha,
+                ),
+                lerp(
+                    lerp(lut[y_vi_ceil, g_ceil, c_floor, ir_floor], lut[y_vi_ceil, g_ceil, c_floor, ir_ceil], ir_alpha),
+                    lerp(lut[y_vi_ceil, g_ceil, c_ceil, ir_floor], lut[y_vi_ceil, g_ceil, c_ceil, ir_ceil], ir_alpha),
+                    c_alpha,
+                ),
+                g_alpha,
+            ),
+            y_vi_alpha,
+        )
+    )
+
+    fusion_y = fusion_result.permute(0, 3, 1, 2)
+
+    fusion_ycbcr = torch.cat([fusion_y, cb_cr], dim=1)
+    fusion_rgb = ycbcr_to_rgb(fusion_ycbcr)  # fusion_rgb = fusion_ycbcr.permute(0, 3, 1, 2)
+
+    return fusion_rgb
+

LUT-Fuse-main/scripts/loss_lut.py

@@ -0,0 +1,196 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from math import exp
+
+
+class fusion_loss(nn.Module):
+    def __init__(self):
+        super(fusion_loss, self).__init__()
+        self.loss_func_ssim = L_SSIM(window_size=48)
+        self.loss_func_Grad = GradientMaxLoss()
+        self.loss_func_l1 = nn.L1Loss()  # 添加 L1 损失
+        self.loss_func_l2 = nn.MSELoss()  # 添加 L2 损失
+        self.loss_func_color = L_color()
+        self.loss_func_Max = L_Intensity_Max_RGB()
+
+    def forward(self, image_vi, image_ir, image_fused, max_ratio=4, consist_ratio=1, ssim_ir_ratio=1,
+                ssim_ratio=1, ir_compose=1, color_ratio=12, text_ratio=2, max_mode="l1", consist_mode="l1"):
+        image_visible_gray = self.rgb2gray(image_vi)
+        image_infrared_gray = self.rgb2gray(image_ir)
+        image_fused_gray = self.rgb2gray(image_fused)
+        # loss_text = text_ratio * self.loss_func_Grad(image_visible_gray, image_infrared_gray, image_fused_gray)
+        # loss_max = max_ratio * self.loss_func_Max(image_vi, image_ir, image_fused, max_mode)
+        loss_ssim = ssim_ratio * (self.loss_func_ssim(image_vi, image_fused) + ssim_ir_ratio * self.loss_func_ssim(image_ir, image_fused_gray))
+        return loss_ssim
+
+    def rgb2gray(self, image):
+        b, c, h, w = image.size()
+        if c == 1:
+            return image
+        image_gray = 0.299 * image[:, 0, :, :] + 0.587 * image[:, 1, :, :] + 0.114 * image[:, 2, :, :]
+        image_gray = image_gray.unsqueeze(dim=1)
+        return image_gray
+
+def gaussian(window_size, sigma):
+    gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)])
+    return gauss / gauss.sum()
+
+
+class L_Intensity_Max_RGB(nn.Module):
+    def __init__(self):
+        super(L_Intensity_Max_RGB, self).__init__()
+
+    def forward(self, image_visible, image_infrared, image_fused, max_mode="l1"):
+        gray_visible = torch.mean(image_visible, dim=1, keepdim=True)
+        gray_infrared = torch.mean(image_infrared, dim=1, keepdim=True)
+
+        mask = (gray_infrared > gray_visible).float()
+
+        fused_image = mask * image_infrared + (1 - mask) * image_visible
+        if max_mode == "l1":
+            Loss_intensity = F.l1_loss(fused_image, image_fused)
+        else:
+            Loss_intensity = F.mse_loss(fused_image, image_fused)
+        return Loss_intensity
+
+
+def create_window(window_size, channel=1):
+    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+    _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
+    window = _2D_window.expand(channel, 1, window_size, window_size).contiguous()
+    return window
+
+
+def ssim(img1, img2, window_size=24, window=None, size_average=True, val_range=None):
+
+    if val_range is None:
+        if torch.max(img1) > 128:
+            max_val = 255
+        else:
+            max_val = 1
+
+        if torch.min(img1) < -0.5:
+            min_val = -1
+        else:
+            min_val = 0
+        L = max_val - min_val
+    else:
+        L = val_range
+
+    padd = 0
+    (_, channel, height, width) = img1.size()
+    if window is None:
+        real_size = min(window_size, height, width)
+        window = create_window(real_size, channel=channel).to(img1.device)
+
+    mu1 = F.conv2d(img1, window, padding=padd, groups=channel)
+    mu2 = F.conv2d(img2, window, padding=padd, groups=channel)
+
+    mu1_sq = mu1.pow(2)
+    mu2_sq = mu2.pow(2)
+    mu1_mu2 = mu1 * mu2
+
+    sigma1_sq = F.conv2d(img1 * img1, window, padding=padd, groups=channel) - mu1_sq
+    sigma2_sq = F.conv2d(img2 * img2, window, padding=padd, groups=channel) - mu2_sq
+    sigma12 = F.conv2d(img1 * img2, window, padding=padd, groups=channel) - mu1_mu2
+
+    C1 = (0.01 * L) ** 2
+    C2 = (0.03 * L) ** 2
+
+    v1 = 2.0 * sigma12 + C2
+    v2 = sigma1_sq + sigma2_sq + C2
+    cs = torch.mean(v1 / v2)  # contrast sensitivity
+
+    ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
+
+    if size_average:
+        ret = ssim_map.mean()
+    else:
+        ret = ssim_map.mean(1).mean(1).mean(1)
+
+    return 1 - ret
+
+class GradientMaxLoss(nn.Module):
+    def __init__(self):
+        super(GradientMaxLoss, self).__init__()
+        self.sobel_x = nn.Parameter(torch.FloatTensor([[-1, 0, 1],
+                                                       [-2, 0, 2],
+                                                       [-1, 0, 1]]).view(1, 1, 3, 3), requires_grad=False).cuda()
+        self.sobel_y = nn.Parameter(torch.FloatTensor([[-1, -2, -1],
+                                                       [0, 0, 0],
+                                                       [1, 2, 1]]).view(1, 1, 3, 3), requires_grad=False).cuda()
+        self.padding = (1, 1, 1, 1)
+
+    def forward(self, image_A, image_B, image_fuse):
+        gradient_A_x, gradient_A_y = self.gradient(image_A)
+        gradient_B_x, gradient_B_y = self.gradient(image_B)
+        gradient_fuse_x, gradient_fuse_y = self.gradient(image_fuse)
+        loss = F.l1_loss(gradient_fuse_x, torch.max(gradient_A_x, gradient_B_x)) + F.l1_loss(gradient_fuse_y, torch.max(gradient_A_y, gradient_B_y))
+        return loss
+
+    def gradient(self, image):
+        image = F.pad(image, self.padding, mode='replicate')
+        gradient_x = F.conv2d(image, self.sobel_x, padding=0)
+        gradient_y = F.conv2d(image, self.sobel_y, padding=0)
+        return torch.abs(gradient_x), torch.abs(gradient_y)
+
+class L_SSIM(torch.nn.Module):
+    def __init__(self, window_size=11, size_average=True, val_range=None):
+        super(L_SSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.val_range = val_range
+
+        self.channel = 1
+        self.window = create_window(window_size)
+
+    def forward(self, img1, img2):
+        (_, channel, _, _) = img1.size()
+        (_, channel_2, _, _) = img2.size()
+
+        if channel != channel_2 and channel == 1:
+            img1 = torch.concat([img1, img1, img1], dim=1)
+            channel = 3
+
+        if channel == self.channel and self.window.dtype == img1.dtype:
+            window = self.window.cuda()
+        else:
+            window = create_window(self.window_size, channel).to(img1.device).type(img1.dtype)
+            self.window = window.cuda()
+            self.channel = channel
+
+        return ssim(img1, img2, window=window, window_size=self.window_size, size_average=self.size_average)
+
+
+class L_color(nn.Module):
+    def __init__(self):
+        super(L_color, self).__init__()
+
+    def forward(self, image_visible, image_fused):
+        ycbcr_visible = self.rgb_to_ycbcr(image_visible)
+        ycbcr_fused = self.rgb_to_ycbcr(image_fused)
+
+        cb_visible = ycbcr_visible[:, 1, :, :]
+        cr_visible = ycbcr_visible[:, 2, :, :]
+        cb_fused = ycbcr_fused[:, 1, :, :]
+        cr_fused = ycbcr_fused[:, 2, :, :]
+
+        loss_cb = F.l1_loss(cb_visible, cb_fused)
+        loss_cr = F.l1_loss(cr_visible, cr_fused)
+
+        loss_color = loss_cb + loss_cr
+        return loss_color
+
+    def rgb_to_ycbcr(self, image):
+        r = image[:, 0, :, :]
+        g = image[:, 1, :, :]
+        b = image[:, 2, :, :]
+
+        y = 0.299 * r + 0.587 * g + 0.114 * b
+        cb = -0.168736 * r - 0.331264 * g + 0.5 * b
+        cr = 0.5 * r - 0.418688 * g - 0.081312 * b
+
+        ycbcr_image = torch.stack((y, cb, cr), dim=1)
+        return ycbcr_image
+

LUT-Fuse-main/test_lut.py

@@ -0,0 +1,93 @@
+import torch
+import torch.nn.functional as F
+import numpy as np
+from PIL import Image
+from torchvision.transforms import ToPILImage
+import time
+from data.simple_dataset import RandomCropPair
+from data.simple_dataset import SimpleDataSet
+import torch.nn as nn
+import transforms as T
+import os
+from scripts.calculate import load_lookup_table, Generator_for_info, apply_fusion_4d_with_interpolation
+
+
+def main():
+    lut_filepath = " "
+    context_file = " "
+    infrared_dir = " "
+    visible_dir = " "
+    save_dir = " "
+    os.makedirs(save_dir, exist_ok=True)
+    # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    lut = load_lookup_table(lut_filepath).to(device)
+    if lut is None:
+        return
+
+    get_context = Generator_for_info()
+    get_context.load_state_dict(torch.load(context_file))
+    get_context = get_context.to(device)
+    get_context.eval()
+
+    data_transform = {
+        "train": RandomCropPair(size=(96, 96)),
+        "val": T.Compose([T.Resize_16(),
+                          T.ToTensor()])}
+
+    val_dataset = SimpleDataSet(visible_path=visible_dir,
+                                infrared_path=infrared_dir,
+                                phase="val",
+                                transform=data_transform["val"])
+    val_loader = torch.utils.data.DataLoader(val_dataset,
+                                             batch_size=1,
+                                             shuffle=False,
+                                             pin_memory=True,
+                                             num_workers=1,
+                                             collate_fn=val_dataset.collate_fn)
+
+    infrared_files = sorted(os.listdir(infrared_dir))
+    visible_files = sorted(os.listdir(visible_dir))
+
+    assert len(infrared_files) == len(visible_files), "The number of images in the infrared and visible folders do not match!"
+    target_size = (128, 128)
+    times = []
+
+    for step, data in enumerate(val_loader):
+        I_A, I_B, task = data
+
+        if torch.cuda.is_available():
+            I_A = I_A.to("cuda")
+            I_B = I_B.to("cuda")
+
+        torch.cuda.synchronize()
+        start_time = time.time()
+        outputs = apply_fusion_4d_with_interpolation(I_A * 255., I_B * 255., lut, get_context)
+        torch.cuda.synchronize()
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        times.append(elapsed_time)
+
+        if not os.path.splitext(task[0])[1]:
+            task_with_extension = task[0] + ".png"
+        else:
+            task_with_extension = task[0]
+        save_path = os.path.join(save_dir, task_with_extension)
+        fusion_result = outputs.squeeze(0).clamp(0, 1).cpu()
+        fusion_result_image = ToPILImage()(fusion_result)
+        fusion_result_image.save(save_path)
+
+
+    warmup_skip = 25
+    if len(times) > warmup_skip:
+        times_after_warmup = times[warmup_skip:]
+        avg_time = np.mean(times_after_warmup)
+        std_time = np.std(times_after_warmup)
+        print(f"Processing completed! after skipping the first {warmup_skip} images，avg_time: {avg_time:.4f} seconds，std_time: {std_time:.4f} seconds")
+    else:
+        print(f"Not enough images to skip the first {warmup_skip} ！Total images: {len(times)}")
+
+if __name__ == "__main__":
+    main()
+

LUT-Fuse-main/transforms.py

@@ -0,0 +1,107 @@
+import numpy as np
+import random
+
+import torch
+from torchvision import transforms as T
+from torchvision.transforms import functional as F
+
+
+def pad_if_smaller(img, size, fill=0):
+    min_size = min(img.size)
+    if min_size < size:
+        ow, oh = img.size
+        padh = size - oh if oh < size else 0
+        padw = size - ow if ow < size else 0
+        img = F.pad(img, (0, 0, padw, padh), fill=fill)
+    return img
+
+
+class Compose(object):
+    def __init__(self, transforms):
+        self.transforms = transforms
+
+    def __call__(self, image):
+        for t in self.transforms:
+            image= t(image)
+        return image
+
+
+class Resize(object):
+    def __init__(self, size):
+        self.size = size
+
+    def __call__(self, image):
+        image = F.resize(image, self.size)
+        return image
+
+class Resize_16(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, image):
+        width, height = image.size
+        new_width = (width // 16) * 16
+        new_height = (height // 16) * 16
+
+        image = F.resize(image, (new_height, new_width))
+
+        return image
+
+
+class Resize_20(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, image):
+        width, height = image.size
+        new_width = (width // 20) * 20
+        new_height = (height // 20) * 20
+
+        image = F.resize(image, (new_height, new_width))
+
+        return image
+
+
+class RandomHorizontalFlip(object):
+    def __init__(self, flip_prob):
+        self.flip_prob = flip_prob
+
+    def __call__(self, image):
+        if random.random() < self.flip_prob:
+            image = F.hflip(image)
+        return image
+
+
+class RandomVerticalFlip(object):
+    def __init__(self, flip_prob):
+        self.flip_prob = flip_prob
+
+    def __call__(self, image):
+        if random.random() < self.flip_prob:
+            image = F.vflip(image)
+        return image
+
+
+class RandomCrop(object):
+    def __init__(self, size):
+        self.size = size
+
+    def __call__(self, image):
+        image = pad_if_smaller(image, self.size)
+        crop_params = T.RandomCrop.get_params(image, (self.size, self.size))
+        image = F.crop(image, *crop_params)
+        return image
+
+class CenterCrop(object):
+    def __init__(self, size):
+        self.size = size
+
+    def __call__(self, image):
+        image = F.center_crop(image, self.size)
+
+        return image
+
+class ToTensor(object):
+    def __call__(self, image):
+        image = F.to_tensor(image)
+        return image