Hao Xu
commited on
Commit
Β·
d56bc01
1
Parent(s):
490d0b1
UI update
Browse files
app.py
CHANGED
@@ -31,7 +31,6 @@ def load_data(source, refresh=False):
|
|
31 |
|
32 |
def build_table(source, refresh=False):
|
33 |
data = load_data(source, refresh)
|
34 |
-
entries = []
|
35 |
|
36 |
if source == "core":
|
37 |
headers = ["Benchmark", "Category", "Pile-train Dirty (%)", "DCLM-baseline Dirty (%)", "CC-2025-05 Dirty (%)"]
|
@@ -40,19 +39,25 @@ def build_table(source, refresh=False):
|
|
40 |
|
41 |
html = """
|
42 |
<table id="benchmarkTable" style="border-collapse: collapse; width: 100%;">
|
43 |
-
<thead
|
|
|
44 |
"""
|
45 |
for col in headers:
|
46 |
-
html += f'
|
47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
48 |
|
49 |
for entry in data:
|
50 |
name = entry.get("Benchmark", "")
|
51 |
url = entry.get("URL", "#")
|
52 |
-
if url
|
53 |
-
hyperlink = f'<a href="{url}" target="_blank">{name}</a>'
|
54 |
-
else:
|
55 |
-
hyperlink = name
|
56 |
|
57 |
row = {
|
58 |
"Benchmark": hyperlink,
|
@@ -70,10 +75,12 @@ def build_table(source, refresh=False):
|
|
70 |
for col in headers:
|
71 |
val = row.get(col, "")
|
72 |
if isinstance(val, float) and val >= 0:
|
73 |
-
|
|
|
74 |
elif isinstance(val, float):
|
75 |
-
|
76 |
-
|
|
|
77 |
html += "</tr>\n"
|
78 |
|
79 |
html += "</tbody></table>"
|
@@ -83,52 +90,67 @@ def build_table(source, refresh=False):
|
|
83 |
let sortDirection = {};
|
84 |
|
85 |
function sortTable(header) {
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
sortDirection[columnIndex] = isAscending ? 'descending' : 'ascending';
|
92 |
|
93 |
-
|
94 |
-
|
95 |
-
th.querySelector('.triangle')
|
|
|
|
|
96 |
});
|
97 |
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
|
|
|
|
|
|
|
104 |
if (isNaN(cellA)) {
|
105 |
return isAscending ? cellA.localeCompare(cellB) : cellB.localeCompare(cellA);
|
106 |
}
|
107 |
return isAscending ? parseFloat(cellA) - parseFloat(cellB) : parseFloat(cellB) - parseFloat(cellA);
|
108 |
});
|
109 |
|
110 |
-
|
111 |
-
table.appendChild(rows[i]);
|
112 |
-
}
|
113 |
}
|
114 |
</script>
|
115 |
"""
|
116 |
|
117 |
html += """
|
118 |
<style>
|
119 |
-
|
|
|
|
|
|
|
120 |
display: inline-block;
|
|
|
|
|
|
|
|
|
|
|
121 |
width: 0;
|
122 |
height: 0;
|
|
|
123 |
border-left: 5px solid transparent;
|
124 |
border-right: 5px solid transparent;
|
125 |
-
margin-left: 5px;
|
126 |
-
transition: transform 0.2s;
|
127 |
}
|
128 |
-
.
|
|
|
|
|
|
|
|
|
|
|
|
|
129 |
border-bottom: 5px solid #000;
|
130 |
}
|
131 |
-
.
|
132 |
border-top: 5px solid #000;
|
133 |
}
|
134 |
</style>
|
@@ -231,9 +253,9 @@ with gr.Blocks() as interface:
|
|
231 |
gr.HTML(
|
232 |
'''<h1 text-align="center">π Benchmark Contamination Monitoring System</h1>
|
233 |
|
234 |
-
<p style='font-size: 16px;'>This system monitors potential contamination in benchmark datasets used for evaluating language models across various open-source corpora
|
235 |
<p style='font-size: 16px;'>The system is released along with our paper Infini-gram mini: Exact n-gram Search at the Internet Scale with FM-Index, which documents the methodology and findings in detail.</p>
|
236 |
-
<p style='font-size: 16px;'>We
|
237 |
'''
|
238 |
)
|
239 |
|
@@ -244,11 +266,9 @@ with gr.Blocks() as interface:
|
|
244 |
gr.Markdown('''
|
245 |
The **Benchmark Contamination Bulletin** presents contamination statistics for evaluation benchmarks across different data sources.
|
246 |
|
247 |
-
- Benchmarks analyzed in our
|
248 |
-
-
|
249 |
-
- The contamination rate represents the percentage of benchmark entries identified as *dirty* based on our detection criteria.
|
250 |
- The bulletin will be updated regularly to include contamination checks on newly released Common Crawl dumps.
|
251 |
-
- You can sort the results by clicking on the column headers.
|
252 |
''')
|
253 |
|
254 |
source_radio = gr.Radio(
|
|
|
31 |
|
32 |
def build_table(source, refresh=False):
|
33 |
data = load_data(source, refresh)
|
|
|
34 |
|
35 |
if source == "core":
|
36 |
headers = ["Benchmark", "Category", "Pile-train Dirty (%)", "DCLM-baseline Dirty (%)", "CC-2025-05 Dirty (%)"]
|
|
|
39 |
|
40 |
html = """
|
41 |
<table id="benchmarkTable" style="border-collapse: collapse; width: 100%;">
|
42 |
+
<thead>
|
43 |
+
<tr>
|
44 |
"""
|
45 |
for col in headers:
|
46 |
+
html += f'''
|
47 |
+
<th onclick="sortTable(this)" style="cursor: pointer; border: 1px solid #ddd; padding: 8px; text-align: right;">
|
48 |
+
{col}
|
49 |
+
<span class="tri-container">
|
50 |
+
<span class="triangle-up"></span>
|
51 |
+
<span class="triangle-down"></span>
|
52 |
+
</span>
|
53 |
+
</th>
|
54 |
+
'''
|
55 |
+
html += "</tr></thead><tbody>"
|
56 |
|
57 |
for entry in data:
|
58 |
name = entry.get("Benchmark", "")
|
59 |
url = entry.get("URL", "#")
|
60 |
+
hyperlink = f'<a href="{url}" target="_blank">{name}</a>' if url else name
|
|
|
|
|
|
|
61 |
|
62 |
row = {
|
63 |
"Benchmark": hyperlink,
|
|
|
75 |
for col in headers:
|
76 |
val = row.get(col, "")
|
77 |
if isinstance(val, float) and val >= 0:
|
78 |
+
val_display = f"{val:5.1f}"
|
79 |
+
html += f'<td style="border: 1px solid #ddd; padding: 8px; text-align: right;">{val_display}</td>'
|
80 |
elif isinstance(val, float):
|
81 |
+
html += f'<td style="border: 1px solid #ddd; padding: 8px; text-align: right;">N/A</td>'
|
82 |
+
else:
|
83 |
+
html += f'<td style="border: 1px solid #ddd; padding: 8px; text-align: right;">{val}</td>'
|
84 |
html += "</tr>\n"
|
85 |
|
86 |
html += "</tbody></table>"
|
|
|
90 |
let sortDirection = {};
|
91 |
|
92 |
function sortTable(header) {
|
93 |
+
const table = document.getElementById("benchmarkTable");
|
94 |
+
const rows = Array.from(table.tBodies[0].rows);
|
95 |
+
const columnIndex = Array.from(header.parentNode.children).indexOf(header);
|
96 |
+
const isAscending = sortDirection[columnIndex] === 'ascending';
|
|
|
97 |
sortDirection[columnIndex] = isAscending ? 'descending' : 'ascending';
|
98 |
|
99 |
+
Array.from(header.parentNode.children).forEach(th => {
|
100 |
+
const up = th.querySelector('.triangle-up');
|
101 |
+
const down = th.querySelector('.triangle-down');
|
102 |
+
if (up) up.classList.remove('active');
|
103 |
+
if (down) down.classList.remove('active');
|
104 |
});
|
105 |
|
106 |
+
if (sortDirection[columnIndex] === 'ascending') {
|
107 |
+
header.querySelector('.triangle-up').classList.add('active');
|
108 |
+
} else {
|
109 |
+
header.querySelector('.triangle-down').classList.add('active');
|
110 |
+
}
|
111 |
|
112 |
+
rows.sort((rowA, rowB) => {
|
113 |
+
const cellA = rowA.cells[columnIndex].innerText;
|
114 |
+
const cellB = rowB.cells[columnIndex].innerText;
|
115 |
if (isNaN(cellA)) {
|
116 |
return isAscending ? cellA.localeCompare(cellB) : cellB.localeCompare(cellA);
|
117 |
}
|
118 |
return isAscending ? parseFloat(cellA) - parseFloat(cellB) : parseFloat(cellB) - parseFloat(cellA);
|
119 |
});
|
120 |
|
121 |
+
rows.forEach(row => table.tBodies[0].appendChild(row));
|
|
|
|
|
122 |
}
|
123 |
</script>
|
124 |
"""
|
125 |
|
126 |
html += """
|
127 |
<style>
|
128 |
+
thead tr {
|
129 |
+
background-color: #f0f0f0;
|
130 |
+
}
|
131 |
+
.tri-container {
|
132 |
display: inline-block;
|
133 |
+
margin-left: 4px;
|
134 |
+
vertical-align: middle;
|
135 |
+
}
|
136 |
+
.triangle-up, .triangle-down {
|
137 |
+
display: block;
|
138 |
width: 0;
|
139 |
height: 0;
|
140 |
+
margin: 1px auto;
|
141 |
border-left: 5px solid transparent;
|
142 |
border-right: 5px solid transparent;
|
|
|
|
|
143 |
}
|
144 |
+
.triangle-up {
|
145 |
+
border-bottom: 5px solid #999;
|
146 |
+
}
|
147 |
+
.triangle-down {
|
148 |
+
border-top: 5px solid #999;
|
149 |
+
}
|
150 |
+
.triangle-up.active {
|
151 |
border-bottom: 5px solid #000;
|
152 |
}
|
153 |
+
.triangle-down.active {
|
154 |
border-top: 5px solid #000;
|
155 |
}
|
156 |
</style>
|
|
|
253 |
gr.HTML(
|
254 |
'''<h1 text-align="center">π Benchmark Contamination Monitoring System</h1>
|
255 |
|
256 |
+
<p style='font-size: 16px;'>This system monitors potential contamination in benchmark datasets used for evaluating language models across various open-source corpora π§.</p>
|
257 |
<p style='font-size: 16px;'>The system is released along with our paper Infini-gram mini: Exact n-gram Search at the Internet Scale with FM-Index, which documents the methodology and findings in detail.</p>
|
258 |
+
<p style='font-size: 16px;'>We welcome the community to submit new benchmarks for contamination analysis using the <b>"Add New Benchmarks"</b> tab.</p>
|
259 |
'''
|
260 |
)
|
261 |
|
|
|
266 |
gr.Markdown('''
|
267 |
The **Benchmark Contamination Bulletin** presents contamination statistics for evaluation benchmarks across different data sources.
|
268 |
|
269 |
+
- Benchmarks analyzed in our paper are under the **core** source. Community-submitted benchmarks appear under the **community** source.
|
270 |
+
- The contamination rate represents the percentage of *dirty* benchmark entries.
|
|
|
271 |
- The bulletin will be updated regularly to include contamination checks on newly released Common Crawl dumps.
|
|
|
272 |
''')
|
273 |
|
274 |
source_radio = gr.Radio(
|
data.json
CHANGED
@@ -22,7 +22,6 @@
|
|
22 |
{"Benchmark": "CSQA", "Category": "Commonsense Understanding", "Pile Dirty": 0.1, "DCLM Dirty": 1.0, "CC202505 Dirty": 0.1, "URL": "https://huggingface.co/datasets/tau/commonsense_qa"},
|
23 |
{"Benchmark": "HellaSwag", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.0, "CC202505 Dirty": 0.0, "URL": "https://huggingface.co/datasets/Rowan/hellaswag"},
|
24 |
{"Benchmark": "OpenbookQA", "Category": "Commonsense Understanding", "Pile Dirty": 10.8, "DCLM Dirty": 15.6, "CC202505 Dirty": 14.6, "URL": "https://huggingface.co/datasets/allenai/openbookqa"},
|
25 |
-
{"Benchmark": "PIQA", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.0, "CC202505 Dirty": 0.0, "URL": ""},
|
26 |
{"Benchmark": "Social IQa", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.5, "CC202505 Dirty": 0.2, "URL": "https://huggingface.co/datasets/allenai/social_i_qa"},
|
27 |
{"Benchmark": "WinoGrande", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.0, "CC202505 Dirty": 0.0, "URL": "https://huggingface.co/datasets/allenai/winogrande"},
|
28 |
|
|
|
22 |
{"Benchmark": "CSQA", "Category": "Commonsense Understanding", "Pile Dirty": 0.1, "DCLM Dirty": 1.0, "CC202505 Dirty": 0.1, "URL": "https://huggingface.co/datasets/tau/commonsense_qa"},
|
23 |
{"Benchmark": "HellaSwag", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.0, "CC202505 Dirty": 0.0, "URL": "https://huggingface.co/datasets/Rowan/hellaswag"},
|
24 |
{"Benchmark": "OpenbookQA", "Category": "Commonsense Understanding", "Pile Dirty": 10.8, "DCLM Dirty": 15.6, "CC202505 Dirty": 14.6, "URL": "https://huggingface.co/datasets/allenai/openbookqa"},
|
|
|
25 |
{"Benchmark": "Social IQa", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.5, "CC202505 Dirty": 0.2, "URL": "https://huggingface.co/datasets/allenai/social_i_qa"},
|
26 |
{"Benchmark": "WinoGrande", "Category": "Commonsense Understanding", "Pile Dirty": 0.0, "DCLM Dirty": 0.0, "CC202505 Dirty": 0.0, "URL": "https://huggingface.co/datasets/allenai/winogrande"},
|
27 |
|