import ast
import argparse
import glob
import pickle
import gradio as gr
import numpy as np
import pandas as pd
block_css = """
#notice_markdown {
font-size: 104%
}
#notice_markdown th {
display: none;
}
#notice_markdown td {
padding-top: 6px;
padding-bottom: 6px;
}
#leaderboard_markdown {
font-size: 104%
}
#leaderboard_markdown td {
padding-top: 6px;
padding-bottom: 6px;
}
#leaderboard_dataframe td {
line-height: 0.1em;
}
footer {
display:none !important
}
.image-container {
display: flex;
align-items: center;
padding: 1px;
}
.image-container img {
margin: 0 30px;
height: 20px;
max-height: 100%;
width: auto;
max-width: 20%;
}
"""
def model_hyperlink(model_name, link):
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
def load_leaderboard_table_csv(filename, add_hyperlink=True):
lines = open(filename).readlines()
heads = [v.strip() for v in lines[0].split(",")]
rows = []
for i in range(1, len(lines)):
row = [v.strip() for v in lines[i].split(",")]
for j in range(len(heads)):
item = {}
for h, v in zip(heads, row):
if h != "Model" and h != "Link" and h != "Language Model" and h != "Open Source":
item[h] = int(v)
else:
item[h] = v
if add_hyperlink:
item["Model"] = model_hyperlink(item["Model"], item["Link"])
rows.append(item)
return rows
def get_arena_table(model_table_df):
# sort by rating
model_table_df = model_table_df.sort_values(by=["Final Score"], ascending=False)
values = []
for i in range(len(model_table_df)):
row = []
model_key = model_table_df.index[i]
model_name = model_table_df["Model"].values[model_key]
# rank
row.append(i + 1)
# model display name
row.append(model_name)
row.append(
model_table_df["Language Model"].values[model_key]
)
row.append(
model_table_df["Open Source"].values[model_key]
)
row.append(
model_table_df["Text Recognition"].values[model_key]
)
row.append(
model_table_df["Scene Text-Centric VQA"].values[model_key]
)
row.append(
model_table_df["Doc-Oriented VQA"].values[model_key]
)
row.append(
model_table_df["KIE"].values[model_key]
)
row.append(
model_table_df["HMER"].values[model_key]
)
row.append(
model_table_df["Final Score"].values[model_key]
)
values.append(row)
return values
def get_recog_table(model_table_df):
# sort by rating
values = []
for i in range(len(model_table_df)):
row = []
model_key = model_table_df.index[i]
model_name = model_table_df["Model"].values[model_key]
# rank
row.append(i + 1)
# model display name
row.append(model_name)
row.append(
model_table_df["Language Model"].values[model_key]
)
row.append(
model_table_df["Open Source"].values[model_key]
)
row.append(
model_table_df["Regular Text"].values[model_key]
)
row.append(
model_table_df["Irregular Text"].values[model_key]
)
row.append(
model_table_df["Artistic Text"].values[model_key]
)
row.append(
model_table_df["Handwriting"].values[model_key]
)
row.append(
model_table_df["Digit string"].values[model_key]
)
row.append(
model_table_df["Non-semantic Text"].values[model_key]
)
row.append(
model_table_df["ALL"].values[model_key]
)
values.append(row)
return values
def build_leaderboard_tab(leaderboard_table_file, text_recog_file, Inaccessible_model_file, show_plot=False):
if leaderboard_table_file:
data = load_leaderboard_table_csv(leaderboard_table_file)
data_recog = load_leaderboard_table_csv(text_recog_file)
data_Inaccessible = load_leaderboard_table_csv(Inaccessible_model_file)
model_table_df = pd.DataFrame(data)
model_table_df_Inaccessible = pd.DataFrame(data_Inaccessible)
recog_table_df = pd.DataFrame(data_recog)
md_head = f"""
# 🏆 OCRBench Leaderboard
| [GitHub](https://github.com/Yuliang-Liu/MultimodalOCR) | [Paper](https://arxiv.org/abs/2305.07895) |
"""
gr.Markdown(md_head, elem_id="leaderboard_markdown")
with gr.Tabs() as tabs:
# arena table
with gr.Tab("OCRBench", id=0):
arena_table_vals = get_arena_table(model_table_df)
md = "OCRBench is a comprehensive evaluation benchmark designed to assess the OCR capabilities of Large Multimodal Models. It consists of five components: Text Recognition, SceneText-Centric VQA, Document-Oriented VQA, Key Information Extraction, and Handwritten Mathematical Expression Recognition. The benchmark includes 1000 question-answer pairs, with all answers containing no fewer than 4 symbols to reduce false positives. All answers undergo manual verification and correction to ensure a more precise evaluation."
gr.Markdown(md, elem_id="leaderboard_markdown")
gr.Dataframe(
headers=[
"Rank",
"Name",
"Language Model",
"Open Source",
"Text Recognition",
"Scene Text-Centric VQA",
"Doc-Oriented VQA",
"KIE",
"HMER",
"Final Score",
],
datatype=[
"str",
"markdown",
"str",
"str",
"number",
"number",
"number",
"number",
"number",
"number",
],
value=arena_table_vals,
elem_id="arena_leaderboard_dataframe",
height=700,
column_widths=[60, 120,150,100, 150, 200, 180, 80, 80, 160],
wrap=True,
)
with gr.Tab("Text Recognition", id=1):
arena_table_vals = get_recog_table(recog_table_df)
md = "OCRBench is a comprehensive evaluation benchmark designed to assess the OCR capabilities of Large Multimodal Models. It consists of five components: Text Recognition, SceneText-Centric VQA, Document-Oriented VQA, Key Information Extraction, and Handwritten Mathematical Expression Recognition. The benchmark includes 1000 question-answer pairs, with all answers containing no fewer than 4 symbols to reduce false positives. All answers undergo manual verification and correction to ensure a more precise evaluation."
gr.Markdown(md, elem_id="leaderboard_markdown")
gr.Dataframe(
headers=[
"Rank",
"Name",
"Language Model",
"Open Source",
"Regular Text",
"Irregular Text",
"Artistic Text",
"Handwriting",
"Digit string",
"Non-semantic Text",
"ALL",
],
datatype=[
"str",
"markdown",
"str",
"str",
"number",
"number",
"number",
"number",
"number",
"number",
"number",
],
value=arena_table_vals,
elem_id="arena_leaderboard_dataframe",
height=700,
column_widths=[60, 120,150,100, 100, 100, 100, 100, 100,100, 80],
wrap=True,
)
with gr.Tab("Inaccessible Model", id=2):
arena_table_vals = get_arena_table(model_table_df_Inaccessible)
md = "The models on this list are neither open-source nor have API call interfaces available."
gr.Markdown(md, elem_id="leaderboard_markdown")
gr.Dataframe(
headers=[
"Rank",
"Name",
"Language Model",
"Open Source",
"Text Recognition",
"Scene Text-Centric VQA",
"Doc-Oriented VQA",
"KIE",
"HMER",
"Final Score",
],
datatype=[
"str",
"markdown",
"str",
"str",
"number",
"number",
"number",
"number",
"number",
"number",
],
value=arena_table_vals,
elem_id="arena_leaderboard_dataframe",
height=700,
column_widths=[60, 120,150,100, 150, 200, 180, 80, 80, 160],
wrap=True,
)
else:
pass
md_tail = f"""
# Notice
Sometimes, API calls to closed-source models may not succeed. In such cases, we will repeat the calls for unsuccessful samples until it becomes impossible to obtain a successful response. It is important to note that due to rigorous security reviews by OpenAI, GPT4V refuses to provide results for the 84 samples in OCRBench.
If you would like to include your model in the OCRBench leaderboard, please follow the evaluation instructions provided on [GitHub](https://github.com/Yuliang-Liu/MultimodalOCR), [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) or [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval), and send us the result screenshots as well as the output files (predictions) generated by your model. If you have any questions, feel free to contact us via email at zhangli123@hust.edu.cn. We will update the leaderboard in time."""
gr.Markdown(md_tail, elem_id="leaderboard_markdown")
def build_demo(leaderboard_table_file, recog_table_file, Inaccessible_model_file):
text_size = gr.themes.sizes.text_lg
with gr.Blocks(
title="OCRBench Leaderboard",
theme=gr.themes.Base(text_size=text_size),
css=block_css,
) as demo:
leader_components = build_leaderboard_tab(
leaderboard_table_file, recog_table_file,Inaccessible_model_file,show_plot=True
)
return demo
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--share", action="store_true")
parser.add_argument("--OCRBench_file", type=str, default="./OCRBench.csv")
parser.add_argument("--TextRecognition_file", type=str, default="./TextRecognition.csv")
parser.add_argument("--Inaccessible_model_file", type=str, default="./Inaccessible_model.csv")
args = parser.parse_args()
demo = build_demo(args.OCRBench_file, args.TextRecognition_file, args.Inaccessible_model_file)
demo.launch()