File size: 6,462 Bytes
3d7abcf
 
 
4e70e1a
d071597
 
3d7abcf
4e70e1a
b838740
4e70e1a
 
 
d071597
4e70e1a
3d7abcf
 
8685a73
 
3d7abcf
e04bd55
3d7abcf
e04bd55
3d7abcf
4e70e1a
 
 
 
 
3d7abcf
 
4e70e1a
 
e04bd55
4e70e1a
e04bd55
4e70e1a
b3439f4
4e70e1a
 
 
e04bd55
 
4e70e1a
 
 
 
 
 
b3439f4
4e70e1a
3d7abcf
b3439f4
4e70e1a
3d7abcf
 
4e70e1a
 
 
 
 
 
 
 
3d7abcf
e04bd55
4e70e1a
a061418
b3439f4
4e70e1a
e04bd55
4e70e1a
3d7abcf
e04bd55
4e70e1a
 
 
 
 
3d7abcf
4e70e1a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3d7abcf
4e70e1a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3d7abcf
4e70e1a
d071597
4e70e1a
 
 
 
3d7abcf
4e70e1a
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
import gradio as gr
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
from huggingface_hub import ModelCard
import requests
import re
import pandas as pd
import os

# --------------------
# Helper functions
# --------------------
def pass_emoji(passed):
    return "โœ…" if passed else "โŒ"

api = HfApi()
USERNAMES_DATASET_ID = "huggingface-course/audio-course-u7-hands-on"
HF_TOKEN = os.environ.get("HF_TOKEN")

def get_user_models(hf_username, task):
    """
    List the user's models for a given task
    """
    try:
        models = api.list_models(author=hf_username, filter=[task])
    except Exception:
        return []

    user_model_ids = [x.modelId for x in models]

    # map task to dataset
    if task == "audio-classification":
        dataset = 'marsyas/gtzan'
    elif task == "automatic-speech-recognition":
        dataset = 'PolyAI/minds14'
    elif task == "text-to-speech":
        dataset = ""
    else:
        print(f"Unsupported task: {task}")
        return []

    dataset_specific_models = []
    for model in user_model_ids:
        try:
            meta = get_metadata(model)
            if meta is None:
                continue
            if dataset == "" or meta.get("datasets") == [dataset]:
                dataset_specific_models.append(model)
        except Exception:
            continue

    return dataset_specific_models

def get_metadata(model_id):
    """Load model metadata safely"""
    try:
        readme_path = hf_hub_download(model_id, filename="README.md", token=HF_TOKEN)
        return metadata_load(readme_path)
    except requests.exceptions.HTTPError:
        return None
    except Exception:
        return None

def extract_metric(model_card_content, task):
    """Extract metric from model card content"""
    accuracy_pattern = r"(?:Accuracy|eval_accuracy): (\d+\.\d+)"
    wer_pattern = r"Wer: (\d+\.\d+)"
    pattern = accuracy_pattern if task == "audio-classification" else wer_pattern
    match = re.search(pattern, model_card_content)
    return float(match.group(1)) if match else None

def parse_metrics(model, task):
    try:
        card = ModelCard.load(model)
        return extract_metric(card.content, task)
    except Exception:
        return None

def calculate_best_result(user_models, task):
    """Calculate best result for a task"""
    best_model = ""
    best_result = -100 if task == "audio-classification" else 100
    larger_is_better = task == "audio-classification"

    for model in user_models:
        metric = parse_metrics(model, task)
        if metric is None:
            continue
        if (larger_is_better and metric > best_result) or (not larger_is_better and metric < best_result):
            best_result = metric
            meta = get_metadata(model)
            if meta:
                best_model = meta.get('model-index', [{}])[0].get("name", model)
    return best_result, best_model

# --------------------
# Certification logic
# --------------------
def certification(hf_username):
    results_certification = [
        {"unit": "Unit 4: Audio Classification", "task": "audio-classification", "baseline_metric": 0.87, "best_result": 0, "best_model_id": "", "passed_": False},
        {"unit": "Unit 5: Automatic Speech Recognition", "task": "automatic-speech-recognition", "baseline_metric": 0.37, "best_result": 0, "best_model_id": "", "passed_": False},
        {"unit": "Unit 6: Text-to-Speech", "task": "text-to-speech", "baseline_metric": 0, "best_result": 0, "best_model_id": "", "passed_": False},
        {"unit": "Unit 7: Audio applications", "task": "demo", "baseline_metric": 0, "best_result": 0, "best_model_id": "", "passed_": False},
    ]

    for unit in results_certification:
        task = unit["task"]
        if task == "audio-classification":
            try:
                models = get_user_models(hf_username, task)
                best_result, best_model_id = calculate_best_result(models, task)
                unit["best_result"] = best_result
                unit["best_model_id"] = best_model_id
                unit["passed_"] = best_result >= unit["baseline_metric"]
            except Exception:
                pass
        elif task == "automatic-speech-recognition":
            try:
                models = get_user_models(hf_username, task)
                best_result, best_model_id = calculate_best_result(models, task)
                unit["best_result"] = best_result
                unit["best_model_id"] = best_model_id
                unit["passed_"] = best_result <= unit["baseline_metric"]
            except Exception:
                pass
        elif task == "text-to-speech":
            try:
                models = get_user_models(hf_username, task)
                if models:
                    unit["best_result"] = 0
                    unit["best_model_id"] = models[0]
                    unit["passed_"] = True
            except Exception:
                pass
        elif task == "demo":
            try:
                u7_file = hf_hub_download(USERNAMES_DATASET_ID, repo_type="dataset", filename="usernames.csv", token=HF_TOKEN)
                u7_users = pd.read_csv(u7_file)
                if hf_username in u7_users['username'].tolist():
                    unit["best_result"] = 0
                    unit["best_model_id"] = "Demo check passed"
                    unit["passed_"] = True
            except Exception:
                pass

        unit["passed"] = pass_emoji(unit["passed_"])

    df = pd.DataFrame(results_certification)
    df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
    return df

# --------------------
# Gradio UI
# --------------------
with gr.Blocks() as demo:
    gr.Markdown("""
    # ๐Ÿ† Check your progress in the Audio Course ๐Ÿ†
    - Pass 3 out of 4 assignments for a certificate.
    - Pass 4 out of 4 assignments for honors.
    - For Unit 7, first check your demo with the [Unit 7 assessment space](https://huggingface.co/spaces/huggingface-course/audio-course-u7-assessment).
    - Make sure your models are uploaded to Hub and public.
    """)
    
    hf_username_input = gr.Textbox(label="Your Hugging Face Username", placeholder="MariaK")
    check_button = gr.Button("Check my progress")
    output_table = gr.Dataframe()
    
    check_button.click(fn=certification, inputs=hf_username_input, outputs=output_table)

demo.launch()