add

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+COPY . /app
+
+RUN pip install --no-cache-dir flask flask-cors pandas openpyxl
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

README copy.md

@@ -0,0 +1,13 @@
+---
+title: Demo
+emoji: 🔥
+colorFrom: yellow
+colorTo: blue
+sdk: gradio
+sdk_version: 5.32.1
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,7 +1,362 @@
-import gradio as gr
+from flask import Flask, jsonify, send_file, request, send_from_directory
+from flask_cors import CORS
+import os, json, uuid, time
+import pandas as pd
+from datetime import datetime, timedelta
+from huggingface_hub import HfApi
+import sys
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from system.pledge_tracking import run_pipeline  
+from huggingface_hub import hf_hub_download
+import spacy
+import traceback
+import threading
 
-def greet(name):
-    return "Hello " + name + "!!"
+nlp = spacy.load("en_core_web_sm")
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+app = Flask(__name__, static_folder='.')
+CORS(app)
+
+HF_DATASET_REPO = "PledgeTracker/demo_feedback"
+HF_TOKEN = os.environ.get("HF_TOKEN")
+TMP_DIR = "tmp"
+FEEDBACK_DIR = "feedback_logs"
+os.makedirs(TMP_DIR, exist_ok=True)
+os.makedirs(FEEDBACK_DIR, exist_ok=True)
+
+REFERENCE_PLEDGES = []
+
+REFERENCE_PLEDGE_PATH = hf_hub_download(
+        repo_id="PledgeTracker/demo_feedback",    
+        filename="existing_pledges.txt",            
+        repo_type="dataset",                      
+        token=os.environ["HF_TOKEN"]            
+    )
+
+if os.path.exists(REFERENCE_PLEDGE_PATH):
+    with open(REFERENCE_PLEDGE_PATH, "r") as f:
+        REFERENCE_PLEDGES = [line.strip() for line in f if line.strip()]
+else:
+    print(f"Missing reference pledge file: {REFERENCE_PLEDGE_PATH}")
+
+
+def lemmatize(text):
+    doc = nlp(text)
+    return " ".join([token.lemma_ for token in doc if not token.is_punct and not token.is_space])
+
+
+@app.route("/api/similar-pledges", methods=["POST"])
+def similar_pledges():
+    data = request.get_json()
+    claim = data.get("claim", "").strip()
+    if not claim or not REFERENCE_PLEDGES:
+        return jsonify({"suggestions": []})
+
+    all_pledges = [claim] + REFERENCE_PLEDGES
+    lemmatized_pledges = [lemmatize(p) for p in all_pledges]
+
+    vectorizer = TfidfVectorizer().fit_transform(lemmatized_pledges)
+    similarities = cosine_similarity(vectorizer[0:1], vectorizer[1:]).flatten()
+    filtered = [(i, similarities[i]) for i in range(len(similarities)) if similarities[i] > 0.3]
+    top_filtered = sorted(filtered, key=lambda x: x[1], reverse=True)[:5]
+
+    suggestions = [
+        {"text": REFERENCE_PLEDGES[i], "index": int(i)}
+        for i, score in top_filtered
+    ]
+
+    return jsonify({"suggestions": suggestions})
+
+
+def calculate_time_range(option: str, pledge_date: str = None):
+    today = datetime.today()
+
+    if option == "week":
+        one_week_ago = today - timedelta(days=7)
+        start = max(one_week_ago, pledge_date)
+    elif option == "month":
+        one_month_ago = today - timedelta(days=30)
+        start = max(one_month_ago, pledge_date)
+    elif option == "year":
+        one_year_ago = today - timedelta(days=365)
+        start = max(one_year_ago, pledge_date)
+    elif option == "since_pledge_date":
+        if not pledge_date:
+            raise ValueError("Pledge date is required for 'since_pledge_date' option")
+        start = datetime.strptime(pledge_date, "%Y-%m-%d")
+    else:
+        raise ValueError("Invalid time range option")
+    print(start)
+    return start.strftime("%Y%m%d"), today.strftime("%Y%m%d")
+
+@app.route("/")
+def serve_html():
+    return send_from_directory('.', 'test.html')
+
+@app.route("/api/status")
+def check_status():
+    user_id = request.args.get("user_id")
+    timestamp = request.args.get("timestamp")
+    log_file_path = os.path.join(TMP_DIR, f"{timestamp}_{user_id}_status.log")
+    if not os.path.exists(log_file_path):
+        return jsonify({"status": {}}), 200
+    try:
+        with open(log_file_path, "r") as f:
+            status = json.load(f)
+    except Exception:
+        status = {}
+
+    return jsonify({"status": status})
+
+
+@app.route("/api/run-model", methods=["POST"])
+def run_model():
+    data = request.get_json()
+    claim = data.get("claim", "no input")
+    time_range_option = data.get("time_range", "month")
+    
+    suggestion_meta = data.get("suggestion_meta") 
+    pledge_date = data.get("pledge_date", "")
+    pledge_author = data.get("pledge_author", "")
+    timestamp = data.get("timestamp") or time.strftime("%Y-%m-%d_%H-%M-%S")
+    user_id = data.get("user_id") or str(uuid.uuid4())[:8]
+
+    log_file_path = os.path.join(TMP_DIR, f"{timestamp}_{user_id}_status.log")
+
+    status_lock = threading.Lock()
+
+    def update_status(step_id, msg):
+        print(f"[STATUS] Step {step_id}: {msg}")
+        with status_lock:
+            if os.path.exists(log_file_path):
+                try:
+                    with open(log_file_path, "r") as f:
+                        current = json.load(f)
+                except Exception:
+                    current = {}
+            else:
+                current = {}
+            current[str(step_id)] += f": {msg}"
+            with open(log_file_path, "w") as f:
+                json.dump(current, f, indent=2)
+
+    try:
+        time_start, time_end = calculate_time_range(time_range_option, pledge_date=pledge_date)
+        print(f"[DEMO] Received claim: {claim}")
+        print(f"[DEMO] Time range: {time_start} ~ {time_end}")
+        print(f"[DEMO] Time range: {pledge_date}")
+
+        # user_id = str(uuid.uuid4())[:8]  
+        # outputs = run_pipeline(claim, pledge_date, pledge_author, time_start, timestamp, user_id)
+
+
+        update_status(0, "📌 Starting the system ...")
+        print(suggestion_meta)
+
+        outputs = run_pipeline(
+                claim, pledge_date, pledge_author, time_start, timestamp, user_id,
+                update_fn=update_status, suggestion_meta=suggestion_meta
+            )
+
+        df = pd.read_excel(outputs["sorted_events"])
+        json_path = os.path.join(TMP_DIR, f"{timestamp}_{user_id}.json")
+        df.to_json(json_path, orient="records", indent=2)
+
+        events = df.to_dict(orient="records")
+        log_entry = {
+            "requested_time": timestamp,
+            "pledge": claim,
+            "suggestion_meta": suggestion_meta,
+            "user_id": user_id,
+            "pledge_author": pledge_author,
+            "pledge_date": pledge_date,
+            "events": events
+        }
+        default_log_path = f"{FEEDBACK_DIR}/feedback_{timestamp}_{user_id}.jsonl"
+
+        with open(default_log_path, "w") as f:
+            f.write(json.dumps(log_entry, indent=1))
+
+        try:
+            api = HfApi()
+            api.upload_file(
+                path_or_fileobj=default_log_path,
+                path_in_repo=f"logs/feedback_{timestamp}_{user_id}.jsonl",
+                repo_id=HF_DATASET_REPO,
+                repo_type="dataset",
+                token=HF_TOKEN
+            )
+            update_status(7, "✅ done")
+
+        except Exception as e:
+            traceback.print_exc()
+            print(f"[Default Feedback Upload Error] {e}")
+
+        return jsonify({
+            "status": "success",
+            "file": f"{timestamp}_{user_id}.json",
+            "user_id": user_id,
+            "timestamp": timestamp
+        })
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify({"status": "error", "detail": str(e)}), 500
+
+@app.route("/api/events")
+def get_events():
+    filename = request.args.get("file")
+    file_path = os.path.join(TMP_DIR, filename)
+
+    if not os.path.exists(file_path):
+        return jsonify({"error": "File not found"}), 404
+
+    with open(file_path, "r") as f:
+        events = json.load(f)
+
+    return jsonify(events)
+
+
+@app.route("/api/feedback", methods=["POST"])
+def receive_feedback():
+    data = request.get_json()
+    pledge = data.get("pledge", "no_pledge_text")
+    feedback_list = data.get("feedback", [])
+    filename = data.get("file")
+    file_path = os.path.join(TMP_DIR, filename)
+    pledge_date = data.get("pledge_date", "")
+    pledge_author = data.get("pledge_author", "")
+
+    if not os.path.exists(file_path):
+        return jsonify({"error": "Event file not found"}), 400
+
+    with open(file_path, "r") as f:
+        events = json.load(f)
+
+    # 直接存储反馈字符串值
+    feedback_dict = {int(item['eventIndex']): item['answer'] for item in feedback_list}
+
+    for idx, event in enumerate(events):
+        event["user_feedback"] = feedback_dict.get(idx)
+
+    log_entry = {
+        "requested_time": data.get("timestamp"),
+        "user_id": data.get("user_id"),
+        "pledge": pledge,
+        "pledge_author": pledge_author,
+        "pledge_date": pledge_date,
+        "events": events
+    }
+
+    timestamp = data.get("timestamp") 
+    user_id = data.get("user_id")
+    
+    if not user_id or not timestamp:
+        return jsonify({'status': 'error', 'detail': 'Missing user_id or timestamp'}), 400
+    
+    local_filename = f"{FEEDBACK_DIR}/feedback_{timestamp}_{user_id}.jsonl"
+
+    with open(local_filename, "w") as f:
+        f.write(json.dumps(log_entry, indent=1))
+
+    try:
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=local_filename,
+            path_in_repo=f"logs/feedback_{timestamp}_{user_id}.jsonl",
+            repo_id=HF_DATASET_REPO,
+            repo_type="dataset",
+            token=HF_TOKEN
+        )
+    except Exception as e:
+        return jsonify({'status': 'partial_success', 'error': str(e)}), 500
+
+    return jsonify({'status': 'success'})
+
+# @app.route("/api/feedback", methods=["POST"])
+# def receive_feedback():
+#     data = request.get_json()
+#     pledge = data.get("pledge", "no_pledge_text")
+#     feedback_list = data.get("feedback", [])
+#     filename = data.get("file")
+#     file_path = os.path.join(TMP_DIR, filename)
+#     pledge_date = data.get("pledge_date", "")
+#     pledge_author = data.get("pledge_author", "")
+
+#     if not os.path.exists(file_path):
+#         return jsonify({"error": "Event file not found"}), 400
+
+#     with open(file_path, "r") as f:
+#         events = json.load(f)
+
+#     feedback_dict = {int(item['eventIndex']): item['answer'] for item in feedback_list}
+
+#     for idx, event in enumerate(events):
+#         event["user_feedback"] = feedback_dict.get(idx)
+
+#     log_entry = {
+#         "requested_time": data.get("timestamp"),
+#         "user_id": data.get("user_id"),
+#         "pledge": pledge,
+#         "pledge_author": pledge_author,
+#         "pledge_date": pledge_date,
+#         "events": events
+#     }
+
+#     timestamp = data.get("timestamp") 
+#     user_id = data.get("user_id")
+#     timestamp = data.get("timestamp")
+
+#     if not user_id or not timestamp:
+#         return jsonify({'status': 'error', 'detail': 'Missing user_id or timestamp'}), 400
+    
+#     local_filename = f"{FEEDBACK_DIR}/feedback_{timestamp}_{user_id}.jsonl"
+
+#     with open(local_filename, "w") as f:
+#         f.write(json.dumps(log_entry, indent=1))
+
+#     try:
+#         api = HfApi()
+#         api.upload_file(
+#             path_or_fileobj=local_filename,
+#             path_in_repo=f"logs/feedback_{timestamp}_{user_id}.jsonl",
+#             repo_id=HF_DATASET_REPO,
+#             repo_type="dataset",
+#             token=HF_TOKEN
+#         )
+#     except Exception as e:
+#         return jsonify({'status': 'partial_success', 'error': str(e)}), 500
+
+#     return jsonify({'status': 'success'})
+
+@app.route("/download-feedback/<filename>")
+def download_feedback_file(filename):
+    return send_from_directory(FEEDBACK_DIR, filename, as_attachment=True)
+
+@app.route("/feedback-files")
+def list_feedback_files():
+    files = os.listdir(FEEDBACK_DIR)
+    return jsonify(sorted(files))
+
+@app.route("/download")
+def download_excel():
+    file = request.args.get("file")
+    if not file:
+        return "Missing file param", 400
+
+    json_path = os.path.join(TMP_DIR, file)
+    if not os.path.exists(json_path):
+        return "Event file not found", 404
+
+    with open(json_path, "r") as f:
+        data = json.load(f)
+
+    df = pd.DataFrame(data)
+    xlsx_path = os.path.join(TMP_DIR, file.replace(".json", ".xlsx"))
+    df.to_excel(xlsx_path, index=False)
+
+    return send_file(xlsx_path, as_attachment=True)
+
+
+if __name__ == '__main__':
+    app.run(host="0.0.0.0", port=7860)

requirements.txt

@@ -0,0 +1,25 @@
+flask
+flask_cors
+pandas
+openpyxl
+huggingface_hub
+PyMuPDF==1.23.25
+huggingface_hub==0.30.2
+lxml==5.3.1
+nltk==3.9.1
+numpy==2.2.6
+openai==1.84.0
+pandas==2.3.0
+rank_bm25==0.2.2
+Requests==2.32.3
+scikit_learn==1.7.0
+sentence_transformers==3.3.1
+spacy==3.8.2
+tiktoken==0.7.0
+torch==2.6.0
+tqdm
+trafilatura==2.0.0
+transformers==4.51.3
+vllm==0.8.4
+accelerate
+

system/augmented_searching.py

@@ -0,0 +1,101 @@
+import json
+import os
+import time
+import requests
+import pandas as pd
+from datetime import datetime
+from pathlib import Path
+import spacy
+
+def google_search(query, api_key, search_engine_id, start_date, end_date):
+    print(f"[SYSTEM] Calling Google Search API for: {query}")
+    sort = f"date:r:{start_date}:{end_date}"
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {
+        "q": query,
+        "key": api_key,
+        "cx": search_engine_id,
+        "num": 10,
+        "sort": sort,
+        "cr": "countryUK",
+        "gl": "uk"
+    }
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        return response.json().get("items", [])
+    except Exception as e:
+        print(f"[ERROR] Google Search Failed: {e}")
+        return []
+
+def save_tsv(file_name, id_value, string_value, value_list, query):
+
+    data = {
+        'ID': id_value,
+        'String': string_value,
+        'ListValue': value_list,
+        'query': query
+    }
+    df = pd.DataFrame(data)
+    df.to_csv(file_name, sep='\t', index=False, header=False)
+
+def ensure_directory_exists(path):
+    dir_path = Path(path).expanduser().resolve().parent
+    if not str(dir_path).startswith("/home") and not str(dir_path).startswith("/data") and not str(dir_path).startswith("outputs"):
+        raise ValueError(f"[ERROR] Unsafe path: {dir_path}")
+    dir_path.mkdir(parents=True, exist_ok=True)
+
+def run_augmented_searching(qa_file, pipeline_base_dir, suggestion_meta, pledge_author, start_date, end_date, user_id, claim_id):
+    if suggestion_meta==None:
+        qa_lines = open(f"{qa_file}","r").read()
+        qa_lines = json.loads(qa_lines)
+        claim_text = f"{pledge_author}: {qa_lines['claim']}"
+    else:
+        # claim_text = suggestion_meta["text"]
+        idx = suggestion_meta["index"]
+        qa_lines = open(f"{qa_file}","r").readlines()[idx]
+        qa_lines = json.loads(qa_lines)
+        claim_text = f"{qa_lines['claim']}"
+        print(qa_lines)
+        
+
+    api_key = os.environ.get("GOOGLE_API_KEY")
+    search_engine_id = os.environ.get("GOOGLE_SEARCH_CX")
+    if not api_key or not search_engine_id:
+        raise EnvironmentError("[ERROR] GOOGLE_API_KEY and GOOGLE_SEARCH_CX must be set in environment.")
+
+    # base_dir = pipeline_base_dir
+
+    tsv_file_path = os.path.join(pipeline_base_dir, "augmented_search_results.tsv")
+    ensure_directory_exists(tsv_file_path)
+
+
+    urls = []
+    string_values = []
+    queries = []
+    questions = []
+    questions = [evidence["question"] for evidence in qa_lines["evidence"] if evidence["question"] not in questions]
+    questions = questions[:10]
+
+
+    results = google_search(claim_text, api_key, search_engine_id, start_date, end_date)
+    print(results)
+    for result in results:
+        if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"]:
+            string_values.append("claim")
+            urls.append(result["link"])
+            queries.append(f"{pledge_author}: {claim_text}")
+    
+    for question in questions:
+        results = google_search(f"{question}", api_key, search_engine_id, start_date, end_date)
+        for result in results:
+            if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"]:
+                string_values.append("question")
+                urls.append(result["link"])
+                queries.append(f"{question}")
+
+    urls = list(dict.fromkeys(urls))
+
+    save_tsv(str(tsv_file_path), [0] * len(urls), string_values, urls, queries)
+    print(f"[SYSTEM] Saved {len(urls)} URLs for claim {claim_id} to {tsv_file_path}")
+    return str(tsv_file_path)

system/baseline/hyde_fc_generation_optimized.py

@@ -0,0 +1,163 @@
+from vllm import LLM, SamplingParams
+import json
+import torch
+import time
+from datetime import datetime, timedelta
+import argparse
+from tqdm import tqdm
+from typing import List, Dict, Any
+import concurrent.futures
+
+class VLLMGenerator:
+    def __init__(self, model_name: str, n: int = 8, max_tokens: int = 512, 
+                 temperature: float = 0.7, top_p: float = 1.0, 
+                 frequency_penalty: float = 0.0, presence_penalty: float = 0.0, 
+                 stop: List[str] = ['\n\n\n'], batch_size: int = 32):
+        self.device_count = torch.cuda.device_count()
+        print(f"Initializing with {self.device_count} GPUs")
+        self.llm = LLM(
+            model=model_name,
+            tensor_parallel_size=self.device_count,
+            max_model_len=4096,
+            gpu_memory_utilization=0.95,
+            enforce_eager=True,
+            trust_remote_code=True,
+            # quantization="bitsandbytes",
+            # dtype="half",
+            # load_format="bitsandbytes",
+            max_num_batched_tokens=4096,
+            max_num_seqs=batch_size
+        )
+        self.sampling_params = SamplingParams(
+            n=n,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            frequency_penalty=frequency_penalty,
+            presence_penalty=presence_penalty,
+            stop=stop,
+            logprobs=1
+        )
+        self.batch_size = batch_size
+        self.tokenizer = self.llm.get_tokenizer()
+        print(f"Initialization complete. Batch size: {batch_size}")
+    
+    def parse_response(self, responses):
+        all_outputs = []
+        for response in responses:
+            to_return = []
+            for output in response.outputs:
+                text = output.text.strip()
+                try:
+                    logprob = sum(logprob_obj.logprob for item in output.logprobs for logprob_obj in item.values())
+                except:
+                    logprob = 0  # Fallback if logprobs aren't available
+                to_return.append((text, logprob))
+            texts = [r[0] for r in sorted(to_return, key=lambda tup: tup[1], reverse=True)]
+            all_outputs.append(texts)
+        return all_outputs
+
+    def prepare_prompt(self, claim: str, model_name: str) -> str:
+        base_prompt = f"Please write a fact-checking article passage to support, refute, indicate not enough evidence, or present conflicting evidence regarding the claim.\nClaim: {claim}"
+        
+        if "OLMo" in model_name:
+            return base_prompt
+        else:
+            messages = [{"role": "user", "content": base_prompt}]
+            return self.tokenizer.apply_chat_template(messages, tokenize=False) + "<|start_header_id|>assistant<|end_header_id|>\n\nPassage: "
+
+    def process_batch(self, batch: List[Dict[str, Any]], model_name: str) -> tuple[List[Dict[str, Any]], float]:
+        start_time = time.time()
+        prompts = [self.prepare_prompt(example["claim"], model_name) for example in batch]
+        
+        try:
+            results = self.llm.generate(prompts, sampling_params=self.sampling_params)
+            outputs = self.parse_response(results)
+            
+            for example, output in zip(batch, outputs):
+                example['hypo_fc_docs'] = output
+            
+            batch_time = time.time() - start_time
+            return batch, batch_time
+        except Exception as e:
+            print(f"Error processing batch: {str(e)}")
+            return batch, time.time() - start_time
+
+# def format_time(seconds: float) -> str:
+#     return str(timedelta(seconds=int(seconds)))
+
+# def estimate_completion_time(start_time: float, processed_examples: int, total_examples: int) -> str:
+#     elapsed_time = time.time() - start_time
+#     examples_per_second = processed_examples / elapsed_time
+#     remaining_examples = total_examples - processed_examples
+#     estimated_remaining_seconds = remaining_examples / examples_per_second
+#     completion_time = datetime.now() + timedelta(seconds=int(estimated_remaining_seconds))
+#     return completion_time.strftime("%Y-%m-%d %H:%M:%S")
+
+def main(args):
+    total_start_time = time.time()
+    print(f"Script started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    
+    # Load data
+    print("Loading data...")
+    with open(args.target_data, 'r', encoding='utf-8') as json_file:
+        examples = json.load(json_file)
+    print(f"Loaded {len(examples)} examples")
+    
+    # Initialize generator
+    print("Initializing generator...")
+    generator = VLLMGenerator(
+        model_name=args.model,
+        batch_size=32
+    )
+    
+    # Process data in batches
+    processed_data = []
+    # batch_times = []
+    batches = [examples[i:i + generator.batch_size] for i in range(0, len(examples), generator.batch_size)]
+    
+    print(f"\nProcessing {len(batches)} batches...")
+    with tqdm(total=len(examples), desc="Processing examples") as pbar:
+        for batch_idx, batch in enumerate(batches, 1):
+            processed_batch, batch_time = generator.process_batch(batch, args.model)
+            processed_data.extend(processed_batch)
+            # batch_times.append(batch_time)
+            
+            # Update progress and timing information
+            # examples_processed = len(processed_data)
+            # avg_batch_time = sum(batch_times) / len(batch_times)
+            # estimated_completion = estimate_completion_time(total_start_time, examples_processed, len(examples))
+            
+            # pbar.set_postfix({
+            #     'Batch': f"{batch_idx}/{len(batches)}",
+            #     'Avg Batch Time': f"{avg_batch_time:.2f}s",
+            #     'ETA': estimated_completion
+            # })
+            # pbar.update(len(batch))
+    
+    # Calculate and display timing statistics
+    # total_time = time.time() - total_start_time
+    # avg_batch_time = sum(batch_times) / len(batch_times)
+    # avg_example_time = total_time / len(examples)
+    
+    # print("\nTiming Statistics:")
+    # print(f"Total Runtime: {format_time(total_time)}")
+    # print(f"Average Batch Time: {avg_batch_time:.2f} seconds")
+    # print(f"Average Time per Example: {avg_example_time:.2f} seconds")
+    # print(f"Throughput: {len(examples)/total_time:.2f} examples/second")
+    
+    # Save results
+    # print("\nSaving results...")
+    with open(args.json_output, "w", encoding="utf-8") as output_json:
+        json.dump(processed_data, output_json, ensure_ascii=False, indent=4)
+    
+    # print(f"Script completed at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    # print(f"Total runtime: {format_time(total_time)}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-i', '--target_data', default='data_store/averitec/dev.json')
+    parser.add_argument('-o', '--json_output', default='data_store/hyde_fc.json')
+    parser.add_argument('-m', '--model', default="meta-llama/Llama-3.1-8B-Instruct")
+    args = parser.parse_args()
+    main(args)

system/baseline/question_generation_optimized.py

@@ -0,0 +1,244 @@
+import os
+import argparse
+import time
+import json
+import nltk
+from rank_bm25 import BM25Okapi
+import numpy as np
+import torch
+from vllm import LLM, SamplingParams
+from datetime import datetime, timedelta
+from itertools import islice
+
+
+def download_nltk_data(package_name, download_dir='nltk_data'):
+    # Ensure the download directory exists
+    os.makedirs(download_dir, exist_ok=True)
+    
+    # Set NLTK data path
+    nltk.data.path.append(download_dir)
+    
+    try:
+        # Try to find the resource
+        nltk.data.find(f'tokenizers/{package_name}')
+        print(f"Package '{package_name}' is already downloaded")
+    except LookupError:
+        # If resource isn't found, download it
+        print(f"Downloading {package_name}...")
+        nltk.download(package_name, download_dir=download_dir)
+        print(f"Successfully downloaded {package_name}")
+
+# def format_time(seconds):
+#     """Format time duration nicely."""
+#     return str(timedelta(seconds=round(seconds)))
+
+def claim2prompts(example): 
+    claim = example["claim"]
+    claim_str = "Example [NUMBER]:||Claim: " + claim + "||Evidence: "
+
+    for question in example["questions"]:
+        q_text = question["question"].strip()
+        if len(q_text) == 0:
+            continue
+
+        if not q_text[-1] == "?":
+            q_text += "?"
+
+        answer_strings = []
+
+        for a in question["answers"]:
+            if a["answer_type"] in ["Extractive", "Abstractive"]:
+                answer_strings.append(a["answer"])
+            if a["answer_type"] == "Boolean":
+                answer_strings.append(a["answer"]  + ", because " + a["boolean_explanation"].lower().strip())
+
+        for a_text in answer_strings:
+            if not a_text[-1] in [".", "!", ":", "?"]:
+                a_text += "."
+
+            prompt_lookup_str = a_text
+            this_q_claim_str = claim_str + a_text.strip() + "||Question: " + q_text 
+            yield (prompt_lookup_str, this_q_claim_str.replace("\n", " ").replace("||", "\n")[:1500])
+
+def main(args):
+    # script_start = time.time()
+    # start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    # print(f"Script started at: {start_time}")
+    # print(f"Loading model: {args.model}")
+
+
+    download_nltk_data('punkt')
+    download_nltk_data('punkt_tab')
+
+    # Load and prepare reference corpus
+    # corpus_start = time.time()
+    with open(args.reference_corpus, "r", encoding="utf-8") as json_file:
+        train_examples = json.load(json_file)
+
+    prompt_corpus, tokenized_corpus = [], []
+    for example in train_examples:
+        for lookup_str, prompt in claim2prompts(example):
+            entry = nltk.word_tokenize(lookup_str)
+            tokenized_corpus.append(entry)
+            prompt_corpus.append(prompt)
+
+    prompt_bm25 = BM25Okapi(tokenized_corpus)
+    # print(f"Reference corpus processed in: {format_time(time.time() - corpus_start)}")
+    
+    # Initialize vLLM with optimized settings
+    gpu_count = torch.cuda.device_count()
+    print(f"Using {gpu_count} GPU{'s' if gpu_count > 1 else ''}")
+    
+    # model_start = time.time()
+    llm = LLM(
+        model=args.model,
+        tensor_parallel_size=gpu_count,
+        max_model_len=4096,
+        gpu_memory_utilization=0.95,
+        enforce_eager=True,
+        trust_remote_code=True,
+        # dtype="half",
+    )
+    llm.get_tokenizer().pad_token = "<|end_of_text|>"
+    # print(f"Model loaded in: {format_time(time.time() - model_start)}")
+    
+    sampling_params = SamplingParams(
+        temperature=0.6,
+        top_p=0.9,
+        top_k=1,
+        skip_special_tokens=False,
+        max_tokens=512,
+        stop=['<|end_of_text|>', '</s>', '<|im_end|>', '[INST]', '[/INST]','<|eot_id|>','<|end|>','<|endoftext|>']
+    )
+
+    # processing_start = time.time()
+
+    # Load target data
+    target_examples = []
+    with open(args.top_k_target_knowledge, "r", encoding="utf-8") as json_file:
+        for line in json_file:
+            target_examples.append(json.loads(line))
+
+    if args.end == -1:
+        args.end = len(target_examples)
+    print(f"Processing {args.end} examples")
+
+    # Process in batches
+    with torch.no_grad():
+        with open(args.output_questions, "w", encoding="utf-8") as output_file:
+            for idx in range(0, args.end, args.batch_size):
+                batch_end = min(idx + args.batch_size, args.end)
+                current_batch = target_examples[idx:batch_end]
+                print(f"\nProcessing batch {idx}-{batch_end}...")
+                
+                for example in current_batch:
+                    # batch_start = time.time()
+                    claim = example["claim"]
+                    claim_id = example["claim_id"]
+                    top_k_sentences_urls = example[f"top_{args.top_k}"]
+                    
+                    batch_prompts = []
+                    batch_metadata = []
+                    
+                    # Prepare all prompts for current example
+                    for sentences_urls in top_k_sentences_urls:
+                        prompt_lookup_str = sentences_urls["sentence"]
+                        url = sentences_urls["url"]
+                        
+                        prompt_s = prompt_bm25.get_scores(nltk.word_tokenize(prompt_lookup_str))
+                        prompt_n = 10
+                        prompt_top_n = np.argsort(prompt_s)[::-1][:prompt_n]
+                        prompt_docs = [prompt_corpus[i] for i in prompt_top_n]
+                        
+                        temp_prompt = "\n\n".join(prompt_docs)
+                        for k in range(1, temp_prompt.count("[NUMBER]")+1):
+                            temp_prompt = temp_prompt.replace("[NUMBER]", f"{k}", 1)
+                            
+                        claim_prompt = "Your task is to generate a question based on the given claim and evidence. The question should clarify the relationship between the evidence and the claim\n\n"
+                        evidence = prompt_lookup_str.replace("\n", " ")
+                        full_prompt = claim_prompt + temp_prompt + "\n\nNow, generate a question that links the following claim and evidence:" + f"\n\nClaim: {claim}" + f"\nEvidence: {evidence}"
+                        
+                        if "OLMo" in args.model:
+                            inputs = [full_prompt]
+                        else:
+                            messages = [{"role":"user", "content":full_prompt}]
+                            inputs = llm.get_tokenizer().apply_chat_template(messages, tokenize=False)
+                            inputs += "<|start_header_id|>assistant<|end_header_id|>\n\nQuestion: "
+                            
+                        batch_prompts.append(inputs)
+                        batch_metadata.append((url, prompt_lookup_str))
+                    
+                    # Process batch
+                    outputs = llm.generate(batch_prompts, sampling_params)
+                    
+                    # Process outputs
+                    evidence = []
+                    for output, (url, sent) in zip(outputs, batch_metadata):
+                        question = output.outputs[0].text.strip().split("?")[0].replace("\n", " ") + "?"
+                        evidence.append({
+                            "question": question,
+                            "answer": sent,
+                            "url": url
+                        })
+                    
+                    # Write results
+                    json_data = {
+                        "claim_id": claim_id,
+                        "claim": claim,
+                        "evidence": evidence
+                    }
+                    output_file.write(json.dumps(json_data, ensure_ascii=False) + "\n")
+                    output_file.flush()
+                    
+                    # batch_time = time.time() - batch_start
+                    # print(f"Processed example {claim_id}. Time elapsed: {batch_time:.2f}s")
+
+    # Calculate and display timing information
+    # total_time = time.time() - script_start
+    # processing_time = time.time() - processing_start
+    # end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    # print("\nTiming Summary:")
+    # print(f"Start time: {start_time}")
+    # print(f"End time: {end_time}")
+    # print(f"Total runtime: {format_time(total_time)}")
+    # print(f"Setup time: {format_time(processing_start - script_start)}")
+    # print(f"Processing time: {format_time(processing_time)}")
+    # print(f"Results written to: {args.output_questions}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Use a prompt to generate questions that could be answered by top-k retrieved evidence. Output generated questions.")
+    parser.add_argument("--model", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct")
+    parser.add_argument("--reference_corpus", default="baseline/train.json")
+    parser.add_argument(
+        "-i",
+        "--top_k_target_knowledge",
+        default="data_store/dev_reranking_top_k.json",
+        help="Directory where the sentences for the scraped data is saved.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output_questions",
+        default="data_store/dev_top_k_qa.json",
+        help="Directory where the sentences for the scraped data is saved.",
+    )
+    parser.add_argument(
+        "--top_k",
+        default=10,
+        type=int
+    )
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=4,
+        help="Number of examples to process in each batch"
+    )
+    parser.add_argument(
+        "-e",
+        "--end",
+        type=int,
+        default=-1
+    )
+    
+    args = parser.parse_args()
+    main(args)

system/baseline/reranking_optimized.py

@@ -0,0 +1,230 @@
+import os
+import torch
+import gc
+from transformers import AutoModel, AutoTokenizer
+from sentence_transformers import SentenceTransformer
+import numpy as np
+import json
+import argparse
+import time
+from datetime import datetime, timedelta
+import re
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+def encode_text(model, tokenizer, texts, batch_size=8, max_length=512):
+    """Encode texts to embeddings using AutoModel"""
+    all_embeddings = []
+    
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        
+        # Tokenize
+        encoded_input = tokenizer(
+            batch,
+            padding=True,
+            truncation=True,
+            max_length=max_length,
+            return_tensors='pt'
+        ).to(model.device)
+        
+        # Compute token embeddings
+        with torch.no_grad():
+            with torch.cuda.amp.autocast(dtype=torch.bfloat16):
+                model_output = model(**encoded_input)
+                # Use mean pooling
+                attention_mask = encoded_input['attention_mask']
+                token_embeddings = model_output[0]  # First element contains token embeddings
+                input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+                embeddings = torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+                all_embeddings.append(embeddings.cpu().numpy())
+        
+        # Clear some memory
+        if i % (batch_size * 4) == 0:
+            torch.cuda.empty_cache()
+            gc.collect()
+    
+    return np.vstack(all_embeddings)
+
+def compute_similarity(emb1, emb2):
+    """Compute cosine similarity between embeddings"""
+    return np.dot(emb1, emb2.T) / (
+        np.linalg.norm(emb1, axis=1).reshape(-1, 1) * 
+        np.linalg.norm(emb2, axis=1).reshape(1, -1)
+    )
+
+def get_detailed_instruct(task_description: str, query: str) -> str:
+    return f'Instruct: {task_description}\nQuery: {query}'
+
+def preprocess_sentences(sentence1, sentence2):
+    vectorizer = TfidfVectorizer().fit_transform([sentence1, sentence2])
+    vectors = vectorizer.toarray()
+    
+    cosine_sim = cosine_similarity(vectors)
+    similarity_score = cosine_sim[0][1]
+    return similarity_score
+
+def remove_trailing_special_chars(text):
+    return re.sub(r'[\W_]+$', '', text)
+
+def remove_special_chars_except_spaces(text):
+    return re.sub(r'[^\w\s]+', '', text)
+
+def select_top_k(claim, results, top_k):
+    '''
+    remove sentence of similarity claim
+    '''
+    dup_check = set()
+    top_k_sentences_urls = []
+    
+    i = 0
+    print(results)
+    claim = remove_special_chars_except_spaces(claim).lower()
+    while len(top_k_sentences_urls) < top_k and i < len(results):
+        print(i)
+        sentence = remove_special_chars_except_spaces(results[i]['sentence']).lower()
+        
+        if sentence not in dup_check:
+            if preprocess_sentences(claim, sentence) > 0.97:
+                dup_check.add(sentence)
+                continue
+            
+            if claim in sentence:
+                if len(claim) / len(sentence) > 0.92:
+                    dup_check.add(sentence)
+                    continue 
+            
+            top_k_sentences_urls.append({
+                'sentence': results[i]['sentence'],
+                'url': results[i]['url']}
+            )
+        i += 1
+        
+    return top_k_sentences_urls
+
+# def format_time(seconds):
+#     """Format time duration nicely."""
+#     return str(timedelta(seconds=round(seconds)))
+
+
+def compute_embeddings_batched(model, texts, batch_size=8):
+    """Compute embeddings in smaller batches to manage memory"""
+    all_embeddings = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        with torch.cuda.amp.autocast(dtype=torch.bfloat16):  # Use bfloat16
+            emb = model.encode(batch, batch_size=len(batch), show_progress_bar=False)
+            all_embeddings.append(emb)
+        
+        # Clear some memory
+        if i % (batch_size * 4) == 0:
+            torch.cuda.empty_cache()
+            gc.collect()
+    
+    return np.vstack(all_embeddings)
+
+def main(args):
+
+    
+    device = "cuda" if torch.cuda.is_available() else 'cpu'
+    print(f"Using device: {device}")
+    
+    # Load model and tokenizer
+    model = AutoModel.from_pretrained(
+        "Salesforce/SFR-Embedding-2_R",
+        torch_dtype=torch.bfloat16,
+        low_cpu_mem_usage=True,
+        device_map="auto"
+    )
+    tokenizer = AutoTokenizer.from_pretrained("Salesforce/SFR-Embedding-2_R")
+    
+    # Load target examples
+    target_examples = []
+    with open(args.target_data, "r", encoding="utf-8") as json_file:
+        for i, line in enumerate(json_file):
+            try:
+                example = json.loads(r"{}".format(line))
+                target_examples.append(example)
+            except:
+                print(f"CURRENT LINE broken {i}")
+    
+    if args.end == -1:
+        args.end = len(target_examples)
+    
+    files_to_process = list(range(args.start, args.end))
+    total = len(files_to_process)
+    
+    task = 'Given a web search query, retrieve relevant passages that answer the query'
+    
+    with open(args.json_output, "w", encoding="utf-8") as output_json:
+        done = 0
+        for idx, example in enumerate(target_examples):
+            if idx in files_to_process:
+                print(f"Processing claim {example['claim_id']}... Progress: {done + 1} / {total}")
+                
+                claim = example['claim']
+                query = [get_detailed_instruct(task, claim)] + [
+                    get_detailed_instruct(task, le) 
+                    for le in example['hypo_fc_docs'] 
+                    if len(le.strip()) > 0
+                ]
+                query_length = len(query)
+                sentences = [sent['sentence'] for sent in example[f'top_{5000}']][:args.retrieved_top_k]
+                
+                # st = time.time()
+                try:
+                    # Process query embeddings
+                    query_embeddings = encode_text(model, tokenizer, query, batch_size=4)
+                    avg_emb_q = np.mean(query_embeddings, axis=0)
+                    hyde_vector = avg_emb_q.reshape((1, -1))
+                    
+                    # Process sentence embeddings in smaller chunks
+                    sentence_embeddings = encode_text(
+                        model,
+                        tokenizer,
+                        sentences, 
+                        batch_size=args.batch_size
+                    )
+                    
+                    # Compute similarities in chunks to save memory
+                    chunk_size = 1000
+                    all_scores = []
+                    for i in range(0, len(sentence_embeddings), chunk_size):
+                        chunk = sentence_embeddings[i:i + chunk_size]
+                        chunk_scores = compute_similarity(hyde_vector, chunk)[0]
+                        all_scores.extend(chunk_scores)
+                    
+                    scores = np.array(all_scores)
+                    top_k_idx = np.argsort(scores)[::-1]
+                    results = [example['top_5000'][i] for i in top_k_idx]
+                    top_k_sentences_urls = select_top_k(claim, results, args.top_k)
+                    
+                    # print(f"Top {args.top_k} retrieved. Time elapsed: {time.time() - st:.2f}s")
+                    
+                    json_data = {
+                        "claim_id": example['claim_id'],
+                        "claim": claim,
+                        f"top_{args.top_k}": top_k_sentences_urls
+                    }
+                    output_json.write(json.dumps(json_data, ensure_ascii=False) + "\n")
+                    output_json.flush()
+                    
+                except RuntimeError as e:
+                    print(f"Error processing claim {example['claim_id']}: {e}")
+                    continue
+                
+                done += 1
+    
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target_data", default="data_store/dev_retrieval_top_k.json")
+    parser.add_argument("--retrieved_top_k", type=int, default=5000)
+    parser.add_argument("--top_k", type=int, default=10)
+    parser.add_argument("-o", "--json_output", type=str, default="data_store/dev_reranking_top_k.json")
+    parser.add_argument("--batch_size", type=int, default=32)
+    parser.add_argument("-s", "--start", type=int, default=0)
+    parser.add_argument("-e", "--end", type=int, default=-1)
+    args = parser.parse_args()
+    
+    main(args)

system/baseline/retrieval_optimized.py

@@ -0,0 +1,244 @@
+import argparse
+import json
+import os
+import time
+import numpy as np
+import pandas as pd
+import nltk
+from rank_bm25 import BM25Okapi
+from multiprocessing import Pool, cpu_count, Manager, Lock
+from functools import partial
+import heapq
+from threading import Thread, Event
+import queue
+from datetime import datetime, timedelta
+
+
+def download_nltk_data(package_name, download_dir='nltk_data'):
+    # Ensure the download directory exists
+    os.makedirs(download_dir, exist_ok=True)
+    
+    # Set NLTK data path
+    nltk.data.path.append(download_dir)
+    
+    try:
+        # Try to find the resource
+        nltk.data.find(f'tokenizers/{package_name}')
+        print(f"Package '{package_name}' is already downloaded")
+    except LookupError:
+        # If resource isn't found, download it
+        print(f"Downloading {package_name}...")
+        nltk.download(package_name, download_dir=download_dir)
+        print(f"Successfully downloaded {package_name}")
+
+
+def combine_all_sentences(knowledge_file):
+    sentences, urls = [], []
+    
+    with open(knowledge_file, "r", encoding="utf-8") as json_file:
+        for i, line in enumerate(json_file):
+            data = json.loads(line)
+            sentences.extend(data["url2text"])
+            urls.extend([data["url"] for _ in range(len(data["url2text"]))])
+    return sentences, urls, i + 1
+
+def remove_duplicates(sentences, urls):
+    df = pd.DataFrame({"document_in_sentences":sentences, "sentence_urls":urls})
+    df['sentences'] = df['document_in_sentences'].str.strip().str.lower()
+    df = df.drop_duplicates(subset="sentences").reset_index()
+    return df['document_in_sentences'].tolist(), df['sentence_urls'].tolist()
+                
+def retrieve_top_k_sentences(query, document, urls, top_k):
+    tokenized_docs = [nltk.word_tokenize(doc) for doc in document[:top_k]]
+    bm25 = BM25Okapi(tokenized_docs)
+    
+    scores = bm25.get_scores(nltk.word_tokenize(query))
+    top_k_idx = np.argsort(scores)[::-1][:top_k]
+
+    return [document[i] for i in top_k_idx], [urls[i] for i in top_k_idx]
+
+def process_single_example(idx, example, args, result_queue, counter, lock):
+    try:
+        with lock:
+            current_count = counter.value + 1
+            counter.value = current_count
+            print(f"\nProcessing claim {idx}... Progress: {current_count} / {args.total_examples}")
+        
+        # start_time = time.time()
+        
+        document_in_sentences, sentence_urls, num_urls_this_claim = combine_all_sentences(
+            os.path.join(args.knowledge_store_dir, f"{idx}.jsonl")
+        )
+        
+        print(f"Obtained {len(document_in_sentences)} sentences from {num_urls_this_claim} urls.")
+        
+        document_in_sentences, sentence_urls = remove_duplicates(document_in_sentences, sentence_urls)
+        
+        query = example["claim"] + " " + " ".join(example['hypo_fc_docs'])
+        top_k_sentences, top_k_urls = retrieve_top_k_sentences(
+            query, document_in_sentences, sentence_urls, args.top_k
+        )
+        
+
+        result = {
+            "claim_id": idx,
+            "claim": example["claim"],
+            f"top_{args.top_k}": [
+                {"sentence": sent, "url": url}
+                for sent, url in zip(top_k_sentences, top_k_urls)
+            ],
+            "hypo_fc_docs": example['hypo_fc_docs']
+        }
+        
+        result_queue.put((idx, result))
+        return True
+    except Exception as e:
+        print(f"Error processing example {idx}: {str(e)}")
+        result_queue.put((idx, None))
+        return False
+
+def writer_thread(output_file, result_queue, total_examples, stop_event):
+    next_index = 0
+    pending_results = []
+    
+    with open(output_file, "w", encoding="utf-8") as f:
+        while not (stop_event.is_set() and result_queue.empty()):
+            try:
+                idx, result = result_queue.get(timeout=1)
+                
+                if result is not None:
+                    heapq.heappush(pending_results, (idx, result))
+                
+                while pending_results and pending_results[0][0] == next_index:
+                    _, result_to_write = heapq.heappop(pending_results)
+                    f.write(json.dumps(result_to_write, ensure_ascii=False) + "\n")
+                    f.flush()
+                    next_index += 1
+                    
+            except queue.Empty:
+                continue
+
+# def format_time(seconds):
+#     """Format time duration nicely."""
+#     return str(timedelta(seconds=round(seconds)))
+
+def main(args):
+
+
+
+    download_nltk_data('punkt')
+    download_nltk_data('punkt_tab')
+    
+    with open(args.target_data, "r", encoding="utf-8") as json_file:
+        target_examples = json.load(json_file)
+
+    if args.end == -1:
+        args.end = len(target_examples)
+    
+    print(f"Total examples to process: {args.end - args.start}")
+
+    files_to_process = list(range(args.start, args.end))
+    examples_to_process = [(idx, target_examples[idx]) for idx in files_to_process]
+    
+    num_workers = min(args.workers if args.workers > 0 else cpu_count(), len(files_to_process))
+    print(f"Using {num_workers} workers to process {len(files_to_process)} examples")
+
+    with Manager() as manager:
+        counter = manager.Value('i', 0)
+        lock = manager.Lock()
+        args.total_examples = len(files_to_process)
+        
+        result_queue = manager.Queue()
+        
+        stop_event = Event()
+        writer = Thread(
+            target=writer_thread,
+            args=(args.json_output, result_queue, len(files_to_process), stop_event)
+        )
+        writer.start()
+
+        process_func = partial(
+            process_single_example,
+            args=args,
+            result_queue=result_queue,
+            counter=counter,
+            lock=lock
+        )
+        
+        with Pool(num_workers) as pool:
+            results = pool.starmap(process_func, examples_to_process)
+        
+        stop_event.set()
+        writer.join()
+        
+        # successful = sum(1 for r in results if r)
+        # print(f"\nSuccessfully processed {successful} out of {len(files_to_process)} examples")
+        # print(f"Results written to {args.json_output}")
+        
+        # # Calculate and display timing information
+        # total_time = time.time() - script_start
+        # avg_time = total_time / len(files_to_process)
+        # end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        # print("\nTiming Summary:")
+        # print(f"Start time: {start_time}")
+        # print(f"End time: {end_time}")
+        # print(f"Total runtime: {format_time(total_time)} (HH:MM:SS)")
+        # print(f"Average time per example: {avg_time:.2f} seconds")
+        # if successful > 0:
+        #     print(f"Processing speed: {successful / total_time:.2f} examples per second")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Get top 10000 sentences with BM25 in the knowledge store using parallel processing."
+    )
+    parser.add_argument(
+        "-k",
+        "--knowledge_store_dir",
+        type=str,
+        default="data_store/knowledge_store",
+        help="The path of the knowledge_store_dir containing json files with all the retrieved sentences.",
+    )
+    parser.add_argument(
+        "--target_data",
+        type=str,
+        default="data_store/hyde_fc.json",
+        help="The path of the file that stores the claim.",
+    )
+    parser.add_argument(
+        "-o",
+        "--json_output",
+        type=str,
+        default="data_store/dev_retrieval_top_k.json",
+        help="The output dir for JSON files to save the top 100 sentences for each claim.",
+    )
+    parser.add_argument(
+        "--top_k",
+        default=5000,
+        type=int,
+        help="How many documents should we pick out with BM25.",
+    )
+    parser.add_argument(
+        "-s",
+        "--start",
+        type=int,
+        default=0,
+        help="Starting index of the files to process.",
+    )
+    parser.add_argument(
+        "-e",
+        "--end",
+        type=int,
+        default=-1,
+        help="End index of the files to process.",
+    )
+    parser.add_argument(
+        "-w",
+        "--workers",
+        type=int,
+        default=0,
+        help="Number of worker processes (default: number of CPU cores)",
+    )
+
+    args = parser.parse_args()
+    main(args)

system/ee.py

@@ -0,0 +1,98 @@
+import json
+import os
+import argparse
+from tqdm import tqdm
+import tiktoken
+from openai import OpenAI
+
+def gpt_4o(input_text):
+    client=OpenAI(api_key=os.environ.get("OAI"))
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "user", "content": [{"type": "text", "text": input_text}]}
+        ],
+        response_format={"type": "json_object"},
+        temperature=0,
+        max_tokens=4096,
+        top_p=0,
+        frequency_penalty=0,
+        presence_penalty=0
+    )
+    return response.choices[0].message.content
+
+def run_gpt4_event_extraction(data_dir, icl_path, max_tokens=100000):
+
+    all_info_path = os.path.join(data_dir, "all_info_with_txt.json")
+    output_dir = os.path.join(data_dir, "gpt4_event_extraction")
+    os.makedirs(output_dir, exist_ok=True)
+
+    ICL = open(icl_path, "r").read()
+    all_info = open(all_info_path, "r").readlines()
+
+    enc = tiktoken.encoding_for_model("gpt-4o")
+
+    for i, line in enumerate(all_info):
+        ID = i
+        urls = []
+        results = []
+
+        data = json.loads(line)
+        docs = data["evidence"]
+        claim = data["claim"]
+
+        output_path = os.path.join(output_dir, f"gpt4o_results_{ID}_claim.json")
+        if os.path.exists(output_path):
+            print(f"输出已存在 {output_path}")
+
+        else:
+
+            for doc in tqdm(docs):
+                if doc["url"] in urls:
+                    continue
+
+                text = " ".join(doc["text"])
+                input_text = (
+                    f"{ICL}\nInput:\n\nTitle: {doc['metadata']['title']}\n"
+                    f"Date: {doc['metadata']['date']}\nArticle: {text}\n\n"
+                    f"Please only summarize events that are useful for verifying the claim '{claim}', and their dates in the JSON format.\n\nOutput:\n"
+                )
+
+                urls.append(doc["url"])
+                text_tokens = enc.encode(input_text)
+                if len(text_tokens) > max_tokens:
+                    input_text = enc.decode(text_tokens[:max_tokens])
+
+                try:
+                    output = gpt_4o(input_text)
+                    print(f"GPT-4o Response: {output}")
+                    results.append({
+                        "url": doc["url"],
+                        "title": doc["metadata"]["title"],
+                        "date": doc["metadata"]["date"],
+                        "article": text,
+                        "output": json.loads(output)
+                    })
+                except Exception as e:
+                    print(f"Error processing doc: {e}")
+                    continue
+
+            
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(results, f, ensure_ascii=False, indent=4)
+
+    return output_path
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run GPT-4o event extraction")
+    parser.add_argument("--data_dir", type=str, required=True, help="Root data directory")
+    parser.add_argument("--icl_path", type=str, required=True, help="Path to ICL prompt file")
+    parser.add_argument("--max_tokens", type=int, default=100000, help="Maximum token limit for input")
+
+    args = parser.parse_args()
+
+    run_gpt4_event_extraction(
+        base_dir=args.base_dir,
+        icl_path=args.icl_path,
+        max_tokens=args.max_tokens
+    )

system/existing_pledges.txt

@@ -0,0 +1,54 @@
+Take back our streets by halving serious violent crime
+We will support families with children by introducing free breakfast clubs in every primary school
+We will finally deliver a full trans-inclusive ban on conversion practices
+We will introduce a ‘Hillsborough Law’ which will place a legal duty of candour on public servants and authorities, and provide legal aid for victims of disasters or state-related deaths
+As a first step, in England, we will deliver an extra two million NHS operations, scans, and appointments every year; that is 40,000 more appointments every week
+We will end the use of offshore trusts to avoid inheritance tax so that everyone who makes their home here in the UK pays their taxes here
+We will abolish non-dom status once and for all, replacing it with a modern scheme for people genuinely in the country for a short period
+We will end the VAT exemption and business rates relief for private schools
+We will get Britain building again … with 1.5 million new homes over the next parliament
+We will ensure the next generation can never legally buy cigarettes
+We will not increase taxes on working people
+We will not increase taxes on working people, which is why we will not increase National Insurance
+We will not increase taxes on working people, which is why we will not increase [...] the basic, higher, or additional rates of Income Tax
+We will not increase taxes on working people, which is why we will not increase [...] VAT
+We will intervene earlier to stop young people being drawn into crime, creating a new Young Futures programme with a network of hubs reaching every community
+Raising confidence in the … criminal justice system to its highest levels
+We will recruit an additional 8,500 new staff to treat children and adults through our first term
+Kickstart economic growth to secure the highest sustained growth in the G7
+Raising confidence in the police … to its highest levels
+We will [introduce] new Respect Orders - powers to ban persistent adult offenders from town centres, which will stamp out issues such as public drinking and drug use
+We will create a new Border Security Command, with hundreds of new investigators, intelligence officers, and cross-border police officers
+Capitalised with £7.3 billion over the course of the next Parliament, the National Wealth Fund will have a remit to support We’s growth and clean energy missions
+We will introduce a new participation requirement [for House of Lords members]
+The next We government will therefore bring about an immediate modernisation, by introducing legislation to remove the right of hereditary peers to sit and vote in the House of Lords
+A new Energy Independence Act will establish the framework for We’s energy and climate policies
+We will introduce a Football Governance Bill, which will establish an independent regulator to ensure financial sustainability of football clubs in England
+Every fiscal event making significant changes to taxation or spending will be subject to an independent OBR forecast
+We will establish a National Wealth Fund
+We will conduct a Strategic Defence Review within our first year in government
+Ending the wasteful Migration and Economic Development partnership with Rwanda
+We will cap corporation tax at the current level of 25%, the lowest in the G7, for the entire parliament
+We will introduce a new ‘Fit For the Future’ fund to double the number of CT and MRI scanners, allowing the NHS to catch cancer and other conditions earlier, saving lives
+We will … [give] 16- and 17-year-olds the right to vote in all elections
+We will set up a new returns and enforcement unit, with an additional 1,000 staff, to fast-track removals to safe countries for people who do not have the right to stay here
+We will capitalise Great British Energy with £8.3 billion, over the next parliament
+We will immediately update the National Policy Planning Framework [sic] to undo damaging Conservative changes, including restoring mandatory housing targets
+Recruit 6,500 new expert teachers in key subjects
+We will carry out a review of sentencing to ensure it is brought up to date
+We will train thousands more GPs
+We will return to meeting NHS performance standards. That means patients should expect to wait no longer than 18 weeks from referral for consultant-led treatment of non-urgent health conditions
+Productivity growth in every part of the country
+The government will deliver a milestone of higher living standards in every part of the United Kingdom by the end of the Parliament
+Giving children the best start in life, with a record 75% of 5-year-olds in England ready to learn when they start school
+We will fix an additional one million potholes across England in each year of the next parliament
+We will not grant new coal licences
+We will set out the path to spending 2.5 per cent of GDP on defence
+We will [...] address the inconsistencies in voter ID rules that prevent legitimate voters from voting. For example, in the case of HM Armed Forces Veteran Cards
+We will also introduce a mandatory retirement age. At the end of the Parliament in which a member reaches 80 years of age, they will be required to retire from the House of Lords
+We will create a new publicly-owned company, Great British Energy
+We will negotiate additional returns arrangements to speed up returns
+We will not issue new licences to explore new [oil and gas] fields
+We will deliver our commitment to spend 2.5% of GDP on defence, but we will bring it forward so that we reach that level in 2027 and we will maintain that for the rest of this Parliament
+We will tackle the immediate crisis with a rescue plan to provide 700,000 more urgent dental appointments
+We will … end asylum hotels, saving the taxpayer billions of pounds

system/generate_output.py

@@ -0,0 +1,75 @@
+import json
+import os
+import argparse
+from system.html2lines import html2metadata
+from lxml.etree import tostring
+import lxml.etree
+
+def process_manifesto_data_with_metadata(input_base_dir: str):
+
+    input_file_path = os.path.join(input_base_dir, "hero/manifesto_icl_reranking_top_k_QA.json")
+    output_file_path = os.path.join(input_base_dir, "all_info_with_txt.json")
+
+    url2text_dir = os.path.join(input_base_dir, "augmented_data_store")
+
+    with open(input_file_path, "r", encoding="utf-8") as f:
+        input_file = f.readlines()
+
+    out_file = open(output_file_path, "w", encoding="utf-8")
+
+
+    i = 0
+
+    for id, line in enumerate(input_file):
+        line = json.loads(line)
+        claim = line["claim"]
+        QAs = line["top_50"]
+        new_line = {"claim": claim, "evidence": []}
+
+        json_path = os.path.join(url2text_dir, f"{id}.jsonl")
+        if not os.path.exists(json_path):
+            print(f"Warning: {json_path} not found")
+            continue
+
+        with open(json_path, "r", encoding="utf-8") as f:
+            try:
+                data_store = json.load(f)
+            except json.JSONDecodeError:
+                f.seek(0)
+                data_store = [json.loads(line) for line in f]
+
+        url_txt = {data["url"]: data["url2text"] for data in data_store}
+
+        URLs = []
+        for j, QA in enumerate(QAs):
+            newQA = QA.copy()
+            URL = QA["url"]
+            newQA["text"] = url_txt.get(URL, "")
+
+            if URL not in URLs:
+                try:
+                    meta = html2metadata(URL)
+                    if isinstance(meta, lxml.etree._Element):
+                        meta = tostring(meta, encoding="unicode", pretty_print=True)
+                    meta_save = {
+                        "title": meta["title"],
+                        "date": meta["date"]
+                    }
+                except Exception as e:
+                    print(f"Metadata extraction failed for URL: {URL}, error: {e}")
+                    meta_save = {
+                        "title": "",
+                        "date": ""
+                    }
+
+
+                newQA["metadata"] = meta_save
+                new_line["evidence"].append(newQA)
+
+        out_file.write(json.dumps(new_line) + "\n")
+
+    out_file.close()
+    return output_file_path
+
+
+

system/hero_QA.py

@@ -0,0 +1,60 @@
+import os
+from datetime import datetime
+import subprocess
+
+
+def run_hero_reranking(user_id, end_date):
+    base_dir = f"outputs/{user_id}_{end_date}"
+    hero_dir = os.path.join(base_dir, "hero")
+    os.makedirs(hero_dir, exist_ok=True)
+
+    hyde_output = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+
+    def safe_run(cmd, timeout=600):
+        try:
+            print(f"👉 Running: {' '.join(cmd)}")
+            subprocess.run(cmd, check=True, timeout=timeout)
+        except subprocess.CalledProcessError as e:
+            print(f"[❌ ERROR] Subprocess failed: {e}")
+            if e.stderr:
+                print("[stderr]:", e.stderr.decode())
+            raise
+        except subprocess.TimeoutExpired:
+            print(f"[❌ TIMEOUT] Command timed out: {' '.join(cmd)}")
+            raise
+
+    # Step 3.2: retrieval
+    print("🔍 Step 3.2: Retrieval from knowledge store ...")
+    knowledge_store_dir = os.path.join(base_dir, "augmented_data_store")
+    retrieval_output = os.path.join(hero_dir, "manifesto_icl_retrieval_top_k_QA.json")
+
+    if not os.path.exists(retrieval_output):
+        safe_run([
+            "python3.12", "baseline/retrieval_optimized.py",
+            "--knowledge_store_dir", knowledge_store_dir,
+            "--target_data", hyde_output,
+            "--json_output", retrieval_output
+        ])
+
+    # Step 3.3: reranking
+    print("🏷️ Step 3.3: Reranking retrieved evidence ...")
+    rerank_output = os.path.join(hero_dir, "manifesto_icl_reranking_top_k_QA.json")
+
+    if not os.path.exists(rerank_output):
+        safe_run([
+            "python3.12", "baseline/reranking_optimized.py",
+            "--target_data", retrieval_output,
+            "--json_output", rerank_output
+        ])
+
+    return {
+        "hyde": hyde_output,
+        "retrieved": retrieval_output,
+        "reranked": rerank_output,
+    }
+
+
+if __name__ == "__main__":
+    output_files = run_step3_hero_pipeline(user_id="xxx", end_date="20250604")
+    for key, path in output_files.items():
+        print(f"✅ {key}: {path}")

system/hero_pipeline.py

@@ -0,0 +1,157 @@
+import os
+from datetime import datetime
+import subprocess
+from huggingface_hub import hf_hub_download
+import json
+
+def run_hero_reranking(pipeline_base_dir, suggestion_meta):
+    base_dir = f"{pipeline_base_dir}"
+    hero_dir = os.path.join(base_dir, "hero")
+    os.makedirs(hero_dir, exist_ok=True)
+
+    if suggestion_meta:
+        hyde_path = hf_hub_download(
+                repo_id="PledgeTracker/demo_feedback",   
+                filename="manifesto_icl_hyde_fc.json",            
+                repo_type="dataset",                     
+                token=os.environ["HF_TOKEN"]            
+            )
+        with open(hyde_path, "r", encoding="utf-8") as f:
+            all_hyde_data = json.load(f)
+
+        idx = suggestion_meta["index"]
+        single_hyde = [all_hyde_data[idx]]
+        save_path = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+        with open(save_path, "w", encoding="utf-8") as f:
+            json.dump(single_hyde, f, indent=2)
+    
+    hyde_output = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+
+    def safe_run(cmd, timeout=600):
+        try:
+            print(f"👉 Running: {' '.join(str(x) for x in cmd)}")
+            subprocess.run(cmd, check=True, timeout=timeout)
+        except subprocess.CalledProcessError as e:
+            print(f"[❌ ERROR] Subprocess failed: {e}")
+            if e.stderr:
+                print("[stderr]:", e.stderr.decode())
+            raise
+        except subprocess.TimeoutExpired:
+            print(f"[❌ TIMEOUT] Command timed out: {' '.join(cmd)}")
+            raise
+
+    # Step 3.2: retrieval
+    print("🔍 Step 3.2: Retrieval from knowledge store ...")
+    knowledge_store_dir = os.path.join(base_dir, "augmented_data_store")
+    retrieval_output = os.path.join(hero_dir, "manifesto_icl_retrieval_top_k_QA.json")
+
+    if not os.path.exists(retrieval_output):
+        safe_run([
+            "python", "system/baseline/retrieval_optimized.py",
+            "--knowledge_store_dir", knowledge_store_dir,
+            "--target_data", hyde_output,
+            "--json_output", retrieval_output,
+        ])
+
+    # Step 3.3: reranking
+    print("🏷️ Step 3.3: Reranking retrieved evidence ...")
+    rerank_output = os.path.join(hero_dir, "manifesto_icl_reranking_top_k_QA.json")
+
+    if not os.path.exists(rerank_output):
+        safe_run([
+            "python", "system/baseline/reranking_optimized.py",
+            "--target_data", retrieval_output,
+            "--json_output", rerank_output,
+            "--top_k", str(50),
+        ])
+
+    return {
+        "hyde": hyde_output,
+        "retrieved": retrieval_output,
+        "reranked": rerank_output,
+    }
+
+
+def run_hero_pipeline(pipeline_base_dir):
+    base_dir = f"{pipeline_base_dir}"
+    hero_dir = os.path.join(base_dir, "hero")
+    os.makedirs(hero_dir, exist_ok=True)
+
+    target_data = os.path.join(base_dir, "claim.json")
+    hyde_output = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+
+    def safe_run(cmd, timeout=600):
+        try:
+            print(f"👉 Running: {' '.join(cmd)}")
+            subprocess.run(cmd, check=True, timeout=timeout)
+        except subprocess.CalledProcessError as e:
+            print(f"[❌ ERROR] Subprocess failed: {e}")
+            if e.stderr:
+                print("[stderr]:", e.stderr.decode())
+            raise
+        except subprocess.TimeoutExpired:
+            print(f"[❌ TIMEOUT] Command timed out: {' '.join(cmd)}")
+            raise
+
+    # Step 3.1: hyde_fc_generation
+    if not os.path.exists(hyde_output):
+        print("🧠 Step 3.1: HyDE ICL generation ...")
+        safe_run([
+            "python", "system/baseline/hyde_fc_generation_optimized.py",
+            "--target_data", target_data,
+            "--json_output", hyde_output
+        ])
+
+    # Step 3.2: retrieval
+    print("🔍 Step 3.2: Retrieval from knowledge store ...")
+    knowledge_store_dir = os.path.join(base_dir, "initial_data_store")
+    retrieval_output = os.path.join(hero_dir, "manifesto_icl_retrieval_top_k.json")
+
+    if not os.path.exists(retrieval_output):
+        safe_run([
+            "python", "system/baseline/retrieval_optimized.py",
+            "--knowledge_store_dir", knowledge_store_dir,
+            "--target_data", hyde_output,
+            "--json_output", retrieval_output
+        ])
+
+    # Step 3.3: reranking
+    print("🏷️ Step 3.3: Reranking retrieved evidence ...")
+    rerank_output = os.path.join(hero_dir, "manifesto_icl_reranking_top_k.json")
+
+    if not os.path.exists(rerank_output):
+        safe_run([
+            "python", "system/baseline/reranking_optimized.py",
+            "--target_data", retrieval_output,
+            "--json_output", rerank_output
+        ])
+
+    # Step 3.4: question generation
+    print("❓ Step 3.4: Generating QA pairs ...")
+    reference_corpus = "system/baseline/train.json"
+    qa_output = os.path.join(hero_dir, "manifesto_icl_top_k_qa.json")
+
+    if not os.path.exists(qa_output):
+        safe_run([
+            "python", "system/baseline/question_generation_optimized.py",
+            "--reference_corpus", reference_corpus,
+            "--top_k_target_knowledge", rerank_output,
+            "--output_questions", qa_output,
+            "--model", "meta-llama/Meta-Llama-3.1-8B-Instruct"
+        ])
+
+    return {
+        "hyde": hyde_output,
+        "retrieved": retrieval_output,
+        "reranked": rerank_output,
+        "qa_pairs": qa_output
+    }
+
+
+if __name__ == "__main__":
+    user_id="xxx"
+    end_date="20250604"
+    pipeline_base_dir = f"{user_id}_{end_date}"
+    output_files = run_step3_hero_pipeline(pipeline_base_dir)
+    for key, path in output_files.items():
+        print(f"✅ {key}: {path}")

system/html2lines.py

@@ -0,0 +1,105 @@
+import sys
+from time import sleep
+import trafilatura
+from trafilatura.meta import reset_caches
+from trafilatura.settings import DEFAULT_CONFIG
+import spacy
+from lxml.etree import tostring
+import lxml.etree
+
+
+import spacy
+import subprocess
+
+try:
+    nlp = spacy.load("en_core_web_lg")
+except OSError:
+    print("🔁 Downloading spaCy model 'en_core_web_lg' ...")
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_lg"], check=True)
+    nlp = spacy.load("en_core_web_lg")
+
+
+DEFAULT_CONFIG.MAX_FILE_SIZE = 50000
+MIN_CHAR = 50
+MAX_CHAR = 5000
+
+
+def get_page(url):
+    page = None
+    for _ in range(3):
+        try:
+            page = trafilatura.fetch_url(url, config=DEFAULT_CONFIG)
+            assert page is not None
+            print("Fetched " + url, file=sys.stderr)
+            break
+        except:
+            sleep(3)
+    return page
+
+
+def url2lines(url):
+    page = get_page(url)
+
+    if page is None:
+        return []
+
+    lines = html2lines(page)
+    return lines
+
+
+def line_correction(lines, max_size=100):
+    out_lines = []
+    for line in lines:
+        if len(line) < MIN_CHAR:
+            continue
+
+        if len(line) > max_size:
+            doc = nlp(
+                line[:MAX_CHAR]
+            )  # We split lines into sentences, but for performance we take only the first 5k characters per line
+            stack = ""
+            for sent in doc.sents:
+                if len(stack) > 0:
+                    stack += " "
+                stack += str(sent).strip()
+                if len(stack) > max_size:
+                    out_lines.append(stack)
+                    stack = ""
+
+            if (
+                len(stack) > MIN_CHAR
+            ):  # Ensure every lines in the out_lines suffice the MIN_CHAR restriction
+                out_lines.append(stack)
+        else:
+            out_lines.append(line)
+
+    return out_lines
+
+
+def html2lines(page):
+    out_lines = []
+
+    if len(page.strip()) == 0 or page is None:
+        return out_lines
+
+    text = trafilatura.extract(page, config=DEFAULT_CONFIG)
+    reset_caches()
+
+    if text is None:
+        return out_lines
+
+    return text.split(
+        "\n"
+    )  # We just spit out the entire page, so need to reformat later.
+
+
+def html2metadata(url):
+    page = get_page(url)
+    metadata = trafilatura.extract_metadata(page)
+    return metadata.as_dict()
+
+if __name__ == "__main__":
+    url = "https://www.bbc.co.uk/news/61407508"
+    metadata = html2metadata(url)
+    text = " ".join(html2lines(page))
+    print(metadata)

system/icl.txt

@@ -0,0 +1,69 @@
+Input:
+
+Title: New investment for Border Security Command
+Date: 2024-09-17
+Article: Up to £75 million in new investment for the Border Security Command paves way for an autumn immigration crime crackdown. The UK’s Border Security Command will deliver cutting edge new technology, extra officers and further covert capabilities across the system following a significant, immediate cash injection, Home Secretary Yvette Cooper announced today. As part of the new Border Security Command uplift, the National Crime Agency (NCA), the police and other law enforcement agency partners will receive a significant cash injection to bolster the UK’s border security and disrupt the criminal people smuggling gangs. The investment comes ahead of an expected effort by the smuggling gangs to cram ever more vulnerable people into unseaworthy boats launched from the French coast while the weather remains fair. Their industrial scale smuggling business is under sustained pressure from co-ordinated UK and European partner law enforcement action. The Home Secretary announced the package of up to £75 million, which redirects funds originally allocated to the previous government’s Illegal Migration Act. It will unlock sophisticated new technology and extra capabilities for the NCA to bolster UK border security and disrupt the criminal people smuggling gangs. The investment is designed to build on a pattern of successful upstream disruptions announced at an operational summit, attended by the Prime Minister, at the NCA headquarters last week. - covert cameras and state of the art monitoring technology, enhancing evidence collection, speeding up investigations and increasing the likelihood of successful prosecutions - establishing a new unit to improve intelligence collection across UK police forces and information flows to partners, alongside an uplift in prosecutors working in the Crown Prosecution Service to act on investigations to swiftly bring those responsible to justice - recruitment of additional personnel for the new Border Security Command, led by Commander Martin Hewitt, which will oversee the co-operation of all of the organisations involved in smashing the gangs - increased work to tackle organised crime groups facilitating irregular migration upstream by intensifying efforts in transit countries to prevent small boat equipment reaching the French coast The announcement follows yesterday’s meeting between the Prime Minister and his Italian counterpart, Giorgia Meloni, in Rome to discuss systematic bilateral co-operation on border security. Italy has seen a significant drop in irregular migration thanks to tougher enforcement and enhanced cooperation with international partners. Newly appointed Border Security Commander – a director general senior civil servant appointment – Martin Hewitt joined the UK delegation to Rome. The enhanced technical and staffing resources announced today will be an important platform for the work he will co-ordinate across UK law enforcement and intelligence agencies when he formally starts his role in the coming weeks. The funding also covers an additional 100 specialist investigators for the NCA, which was announced by the government last month, representing a 25% increase in the agency’s dedicated personnel tackling organised immigration crime. The government has also announced a 50% increase in the number of British officers stationed at Europol, supporting European operations to dismantle organised crime groups facilitating people smuggling. Criminal gangs are getting away with undermining our border security and putting lives at risk. The Border Security Command will deliver a major overhaul and upgrade in law enforcement against smugglers and trafficking gangs to boost our border security. State of the art technology and enhanced intelligence capabilities will ensure we are using every tool at our disposal to dismantle this vile trade. I welcome this funding, which will allow us to improve and extend our technology, data exploitation, and capacity-building both internationally and in the UK. Tackling organised immigration crime remains a top priority for the NCA, we are currently leading around 70 investigations into the gangs or individuals involved in the highest echelons of criminality, and we are devoting more resources to it than ever before. We are determined to do all we can to disrupt and dismantle these networks, wherever they operate. CPS Director of Public Prosecutions Stephen Parkinson said: CPS prosecutors will bring significant expertise to the new unit to help stop human trafficking gangs in their tracks, and pursue any assets gained through criminality. Working with partners, we will continue to discourage, disrupt and dismantle this exploitative trade through prosecutions and cross-border collaboration. The announcement coincides with a concerted push by UK ministers to tackle shared border security challenges. Immigration Minister Dame Angela Eagle is attending the annual Berlin Process Interior Ministers’ meeting in Germany today (Tuesday 17 September), to discuss strengthening border security, tackling organised crime groups and combatting violence against women and girls across the Western Balkans region. The meeting brings together European partners with a focus on working with partners across the Western Balkans, a key region in the journey of irregular migrants through Europe and, in many cases, onwards to the UK.
+
+Summary events and their dates in the JSON format.
+
+Output:
+
+{
+    "events":[
+    {
+        "event": "Announcement of up to £75 million in new investment for the UK's Border Security Command by Home Secretary Yvette Cooper.", 
+        "date": "2024-09-17"
+    },
+    {
+        "event": "Immigration Minister Dame Angela Eagle attending the annual Berlin Process Interior Ministers’ meeting in Germany to discuss strengthening border security.", 
+        "date": "2024-09-17"
+    },
+    {
+        "event": "An operational summit attended by the Prime Minister at the NCA headquarters where successful upstream disruptions were announced.", 
+        "date": "Last week (relative to 2024-09-17)"
+    },
+    {
+        "event": "A meeting took place in Rome between the UK Prime Minister and Italian Prime Minister Giorgia Meloni to discuss systematic bilateral cooperation on border security.", 
+        "date": "Yesterday (relative to 2024-09-17)"
+    },
+    {
+        "event": "The government announced an additional 100 specialist investigators for the NCA, representing a 25% increase in dedicated personnel tackling organised immigration crime.", 
+        "date": "Last month (relative to 2024-09-17)"
+    }
+    ]
+}
+
+Input:
+
+Title: Home Secretary announces new measures to boost Britain’s border security
+Date: 2024-08-21
+Article: Home Secretary announces new measures to boost Britain’s border security Home Secretary announces new measures to strengthen border security, enforce immigration rules and increase returns. New measures to boost Britain’s border security are being set out today (21 August) by the Home Secretary, including the immediate recruitment of up to 100 new specialist intelligence and investigation officers at the National Crime Agency (NCA) to target, dismantle and disrupt organised immigration crime networks. Yvette Cooper has also today announced a major surge in immigration enforcement and returns activity, to make sure that immigration and asylum rules are respected and enforced - saying that the government has new plans for the next 6 months to achieve the highest rate of removals of those with no right to be here, including failed asylum seekers, for 5 years (since 2018). In addition, a new intelligence-driven illegal working programme will be rolled out to target, investigate and take down unscrupulous employers who illegally employ those with no right to work here. The new measures are fulfilling on the government’s commitment to provide long-term security to our borders. - up to 100 new specialist intelligence and investigations officers deployed to the National Crime Agency (NCA) to disrupt and smash criminal smuggling gangs and prevent dangerous boat crossings - a large surge in enforcement and returns flights, with the aim of putting removals at their highest level since 2018, reversing the damaging drop in enforcement over recent years - increased detention capacity including 290 added beds at Campsfield and Haslar Immigration Removal Centres - redeployment of staff to drive this increase in returns - sanctions to be taken against unscrupulous employers who hire workers illegally This comes on top of the 50% uplift in the number of NCA officers stationed in Europol. These officers have been immediately deployed to support European operations to disrupt the activity of criminal smuggling gangs making millions out of small boat crossings. The NCA currently has around 70 investigations targeting the highest harm criminal networks involved in people smuggling and trafficking, and worked with international partners to support the seizure of around 400 boats and engines intended for use in channel crossings. A range of sanctions, including financial penalty notices, business closure orders and potential prosecution, will be taken against those employing illegal workers. Those caught working illegally and eligible for removal will be detained, pending their swift removal. Alongside this, the government is increasing detention spaces to support the higher pace of removals including reopening and adding 290 beds across Immigration Removal Centres (IRCs) at Campsfield and Haslar. This increase will ensure there is additional capacity to facilitate higher levels of enforcement and returns so that rules are properly respected. Building on 9 successful returns flights in the last six weeks, including the largest-ever chartered return flight, the government is redeploying personnel and resources to support further activity. Staff are being redeployed to increase removal of failed asylum seekers, which had dropped by 40% since 2010. Three hundred caseworkers have already been reassigned to progress thousands of failed asylum and returns cases, including enforced and voluntary returns. Enhanced digital capabilities will be deployed to ensure consistent contact throughout, preventing those with no right to be here from disappearing into exploitative illegal working and ensure they can be returned. This enforcement surge, overseen by Bas Javid, the Home Office’s Director General for Immigration Enforcement, is part of the government’s plans to transform the asylum system and secure UK borders. This will ensure that all Immigration Enforcement processes are implemented firmly, fairly, and accurately throughout, whilst also taking account of the important lessons learnt from Windrush. We are taking strong and clear steps boost our border security and ensure the rules are respected and enforced. Our new Border Security Command is already gearing up, with new staff being urgently recruited and additional staff already stationed across Europe, working with European enforcement agencies to find every route in to smashing the criminal smuggling gangs organising dangerous boat crossings which undermine our border security and putting lives at risk. And by increasing enforcement capabilities and returns we will establish a system that is better controlled and managed, in place of the chaos that has blighted the system for far too long. NCA Director General of Operations Rob Jones said: Tackling organised immigration crime remains a key priority for the NCA and we are dedicating more effort and resource than ever before. These extra officers will play a key role in that, with the NCA currently leading around 70 investigations into the highest harm people smuggling and trafficking groups. Taking on these dangerous and exploitative gangs requires international co-operation and we continue to further enhance our already strong relationship with Europol and other law enforcement partners. We are determined to do all we can to disrupt and dismantle these networks, whether they are operating in the UK or overseas. This work builds on the Prime Minister’s meeting at the European Political Community last month, where he held discussions with the migration working group alongside Italy, Albania, Germany, Malta, Denmark, Hungary, The Netherlands, and Slovakia. The European leaders discussed border security, their joint efforts to tackle people-smuggling, and the ambition to work collectively with other countries to deliver solutions. Since taking office, the Home Secretary has also held calls with a range of partners to discuss increasing cooperation to tackle organised immigration crime.
+
+Summary events and their dates in the JSON format.
+
+Output:
+
+{
+  "events": [
+    {
+      "event": "Home Secretary Yvette Cooper announces new measures to boost Britain's border security, including the recruitment of up to 100 new specialist intelligence and investigation officers at the National Crime Agency (NCA).",
+      "date": "2024-08-21"
+    },
+    {
+      "event": "Announcement of a major surge in immigration enforcement and returns activity to achieve the highest rate of removals of those with no right to be in the UK since 2018.",
+      "date": "2024-08-21"
+    },
+    {
+      "event": "Introduction of a new intelligence-driven illegal working programme to target and take down employers who illegally employ individuals with no right to work in the UK.",
+      "date": "2024-08-21"
+    },
+    {
+      "event": "The government announces increased detention capacity, including 290 added beds at Campsfield and Haslar Immigration Removal Centres.",
+      "date": "2024-08-21"
+    },
+    {
+      "event": "The Prime Minister's meeting at the European Political Community last month, where discussions were held with European leaders on border security and tackling people-smuggling.",
+      "date": "Last month (relative to 2024-08-21)"
+    }
+  ]
+}

system/initial_searching.py

@@ -0,0 +1,101 @@
+import json
+import os
+import time
+import requests
+import pandas as pd
+from datetime import datetime
+from pathlib import Path
+import spacy
+import subprocess
+
+try:
+    nlp = spacy.load("en_core_web_sm")
+except OSError:
+    print("🔁 Downloading en_core_web_sm model ...")
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"], check=True)
+    nlp = spacy.load("en_core_web_sm")
+
+def clean_keywords(text):
+    doc = nlp(text)
+    keywords = []
+    for chunk in doc.noun_chunks:
+        words = [token.text for token in chunk if not token.is_stop and token.is_alpha]
+        if words:
+            cleaned_phrase = " ".join(words)
+            if len(cleaned_phrase) > 2:
+                keywords.append(cleaned_phrase)
+    return list(set(keywords))
+
+def google_search(query, api_key, search_engine_id, start_date, end_date):
+    print(f"[SYSTEM] Calling Google Search API for: {query}")
+    sort = f"date:r:{start_date}:{end_date}"
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {
+        "q": query,
+        "key": api_key,
+        "cx": search_engine_id,
+        "num": 10,
+        "sort": sort,
+        "cr": "countryUK",
+        "gl": "uk"
+    }
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        return response.json().get("items", [])
+    except Exception as e:
+        print(f"[ERROR] Google Search Failed: {e}")
+        return []
+
+def save_tsv(file_path, claim_id, claim_text, url_list):
+    df = pd.DataFrame({
+        'ID': [claim_id] * len(url_list),
+        'String': ["claim"] * len(url_list),
+        'ListValue': url_list,
+        'query': [claim_text] * len(url_list)
+    })
+    df.to_csv(file_path, sep='\t', index=False, header=False)
+
+def ensure_directory_exists(path):
+    dir_path = Path(path).expanduser().resolve().parent
+    if not str(dir_path).startswith("/home") and not str(dir_path).startswith("/data") and not str(dir_path).startswith("outputs"):
+        raise ValueError(f"[ERROR] Unsafe path: {dir_path}")
+    dir_path.mkdir(parents=True, exist_ok=True)
+
+def run_initial_searching(claim_text, pipeline_base_dir, start_date, end_date, user_id, claim_id):
+    api_key = os.environ.get("GOOGLE_API_KEY")
+    search_engine_id = os.environ.get("GOOGLE_SEARCH_CX")
+    if not api_key or not search_engine_id:
+        raise EnvironmentError("[ERROR] GOOGLE_API_KEY and GOOGLE_SEARCH_CX must be set in environment.")
+
+    base_dir = pipeline_base_dir
+    manifesto_json_file = os.path.join(base_dir,"claim.json")
+    tsv_file_path = os.path.join(base_dir,"initial_search_results.tsv")
+
+    ensure_directory_exists(tsv_file_path)
+
+    claim_record = {"claim_id": claim_id, "claim": claim_text}
+    # if manifesto_json_file.exists():
+    #     with open(manifesto_json_file, "r") as f:
+    #         records = json.load(f)
+    # else:
+    records = []
+    records.append(claim_record)
+    with open(manifesto_json_file, "w") as f:
+        json.dump(records, f, indent=1)
+
+    urls = []
+    results = google_search(f"{claim_text}", api_key, search_engine_id, start_date, end_date)
+    urls += [r["link"] for r in results if "link" in r]
+    keywords = clean_keywords(claim_text)
+    keyword_text = " ".join(keywords)
+    # for kw in keywords:
+    #     results = google_search(kw, api_key, search_engine_id, start_date, end_date)
+    #     urls += [r["link"] for r in results if "link" in r]
+    results = google_search(keyword_text, api_key, search_engine_id, start_date, end_date)
+    urls += [r["link"] for r in results if "link" in r]
+    urls = list(dict.fromkeys(urls))
+
+    save_tsv(str(tsv_file_path), claim_id, claim_text, urls)
+    print(f"[SYSTEM] Saved {len(urls)} URLs for claim {claim_id} to {tsv_file_path}")
+    return str(tsv_file_path), str(manifesto_json_file)

system/instruction.txt

@@ -0,0 +1,7 @@
+You are given a pledge, the pledge speaker, and the date of when the pledge is made, and a key event summarized from an online article along with the date of when the event happens. Your task is to determine whether this event summary is useful to track the fulfilment of this pledge. 
+
+Yes:
+The summary presents developments or actions that demonstrate progress (or lack thereof) towards fulfilling the pledge. It helps evaluate whether the pledge is on track or not.
+
+No:
+The summary only provides background or contextual information, but no progress information for evaluating the fulfilment of the pledge. Or the summary is less than or not related to the pledge.

system/pledge_tracking.py

@@ -0,0 +1,219 @@
+from huggingface_hub import login
+from datetime import datetime
+import os, time
+import pandas as pd
+
+from system.initial_searching import run_initial_searching
+from system.scraper import run_scraper
+from system.hero_pipeline import run_hero_pipeline, run_hero_reranking
+from system.augmented_searching import run_augmented_searching
+from system.generate_output import process_manifesto_data_with_metadata
+from system.ee import run_gpt4_event_extraction
+from system.process_time import extract_and_sort_events
+import spacy
+import subprocess
+from huggingface_hub import hf_hub_download
+import json
+
+try:
+    spacy.load("en_core_web_sm")
+except OSError:
+    print("🔁 Downloading en_core_web_sm model ...")
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
+    nlp = spacy.load("en_core_web_sm")
+
+
+def count_total_events(output_path):
+    with open(output_path, "r", encoding="utf-8") as f:
+        results = json.load(f)
+
+    total_events = 0
+    for item in results:
+        try:
+            events = item["output"]
+            if isinstance(events, list):
+                total_events += len(events)
+            else:
+                print(f"invalid: {events}")
+        except KeyError:
+            print(f"lack item: {item}")
+
+    print(f"{total_events} events in total")
+    return total_events
+
+def run_pipeline(claim, pledge_date, pledge_author, start_date, timestamp, user_id, update_fn=None, suggestion_meta=None):
+    pipeline_base_dir = f"outputs/{timestamp}_{user_id}"
+    os.makedirs(pipeline_base_dir, exist_ok=True)
+
+    step_id=1
+
+    # Step 1: Google 搜索
+    if suggestion_meta==None:
+        
+
+        print("🔍 Step 1: Initial searching ...")
+        initial_tsv_file, claim_json_path = run_initial_searching(
+            claim_text=f"{pledge_author} : {claim}",
+            # pledge_author=pledge_author,
+            pipeline_base_dir=pipeline_base_dir,
+            start_date=start_date,
+            end_date="",
+            user_id=user_id,
+            claim_id=0,
+        )
+        with open(initial_tsv_file, "r", encoding="utf-8") as f:
+            line_count = sum(1 for line in f)
+        if update_fn:
+            update_fn(step_id, f"We have found {line_count} URLs")
+            step_id+=1
+        
+
+        print("🌐 Step 2: Scraping URLs ...")
+        initial_data_store_dir = os.path.join(pipeline_base_dir, "initial_data_store")
+        os.makedirs(initial_data_store_dir, exist_ok=True)
+        initial_scraped_output_path = os.path.join(initial_data_store_dir, "0.jsonl")
+        run_scraper(initial_tsv_file, initial_scraped_output_path)
+
+        with open(initial_scraped_output_path, "r", encoding="utf-8") as f:
+            line_count = sum(1 for line in f if json.loads(line)["url2text"] != [])
+        if update_fn: 
+            update_fn(step_id, f"We have scraped {line_count} URLs")
+            step_id+=1
+
+
+        print("🧠 Step 3: HerO processing ...")
+        hero_output_dir = os.path.join(pipeline_base_dir, "hero")
+        os.makedirs(hero_output_dir, exist_ok=True)
+        run_hero_pipeline(pipeline_base_dir)    
+
+        qa_file_path = os.path.join(hero_output_dir, "manifesto_icl_top_k_qa.json")
+
+        with open(qa_file_path, "r", encoding="utf-8") as f:
+            questions = {line["question"] for line in json.load(f)["evidence"]}
+            line_count = len(questions)
+        if update_fn: 
+            update_fn(step_id, f"We have generated {line_count} search queries")
+            step_id+=1
+
+    else: 
+        claim_json_path = None
+        initial_scraped_output_path = None
+        initial_tsv_file = None
+        hero_output_dir = None
+        qa_file_path = hf_hub_download(
+                repo_id="PledgeTracker/demo_feedback",   
+                filename="manifesto_with_QA_icl_top_k_qa.json",            
+                repo_type="dataset",                     
+                token=os.environ["HF_TOKEN"]            
+            )
+        print(qa_file_path)
+
+
+    augmented_tsv_file = run_augmented_searching(
+        qa_file=qa_file_path,
+        pledge_author=pledge_author,
+        pipeline_base_dir=pipeline_base_dir,
+        start_date=start_date,
+        suggestion_meta=suggestion_meta,
+        end_date="",
+        user_id=user_id,
+        claim_id=0,
+    )
+    with open(augmented_tsv_file, "r", encoding="utf-8") as f:
+        line_count = sum(1 for line in f)
+    if update_fn:
+        update_fn(step_id, f"We have found {line_count} URLs")
+        step_id+=1
+
+    augmented_data_store_dir = os.path.join(pipeline_base_dir, "augmented_data_store")
+    os.makedirs(augmented_data_store_dir, exist_ok=True)
+    augmented_scraped_output_path = os.path.join(augmented_data_store_dir, "0.jsonl")
+    run_scraper(augmented_tsv_file, augmented_scraped_output_path)
+
+    with open(augmented_scraped_output_path, "r", encoding="utf-8") as f:
+        line_count = sum(1 for line in f if json.loads(line)["url2text"] != [])
+    if update_fn: 
+        update_fn(step_id, f"We have scraped {line_count} URLs")
+        step_id+=1
+    
+
+    run_hero_reranking(pipeline_base_dir, suggestion_meta)
+
+    # Step 7: Preparing for GPT-4
+    # print("🧠 Step 7: Processing format ...")
+
+    meta_data_dir = process_manifesto_data_with_metadata(input_base_dir=pipeline_base_dir)
+
+    # Step 8: Event extraction using GPT-4
+    print("🧠 Extracting events ...")
+
+    all_info_path = os.path.join(pipeline_base_dir, "all_info_with_txt.json")
+    unique_urls = set()
+    with open(all_info_path, "r", encoding="utf-8") as f:
+        for line in f:
+            data = json.loads(line)
+            docs = data.get("evidence", [])
+            for doc in docs:
+                if "url" in doc:
+                    unique_urls.add(doc["url"])
+    if update_fn: 
+        update_fn(step_id, f"We have found {len(unique_urls)} most relevant documents")
+        step_id+=1
+    
+    extracted_event_path = run_gpt4_event_extraction(data_dir=pipeline_base_dir, icl_path="system/icl.txt", max_tokens=100000)
+
+    events_num = count_total_events(extracted_event_path)
+
+    if update_fn: 
+        update_fn(step_id, f"We have extracted {events_num}")
+        step_id+=1
+
+
+    # Step 9: Sorting events and label usefulness
+    print("📅 Sorting events temporally ...")
+
+    
+    sorted_events = extract_and_sort_events(
+        data_dir=pipeline_base_dir, 
+        pledge_date=pledge_date, 
+        pledge_author=pledge_author, 
+        claim=claim,
+        suggestion_meta=suggestion_meta
+        )
+    print(sorted_events)
+    df = pd.DataFrame(sorted_events)
+    sorted_event_path = f"{pipeline_base_dir}/sorted_events.xlsx"
+    df.to_excel(sorted_event_path, index=False)
+
+    if update_fn: 
+        update_fn(step_id, "All events are sorted!")
+
+    return {
+        "claim_json": claim_json_path,
+        "initial_scraped_jsonl": initial_scraped_output_path,
+        "initial_tsv_file": initial_tsv_file,
+        "hero_dir": hero_output_dir,
+        "augmented_scraped_jsonl": augmented_scraped_output_path,
+        "augmented_tsv_file": augmented_tsv_file,
+        "meta_data_dir": meta_data_dir,
+        "unsorted_events": extracted_event_path,
+        "sorted_events": sorted_event_path,
+    }
+
+
+if __name__ == "__main__":
+    start = time.time()
+
+    if os.environ.get("HF_TOKEN"):
+        login(token=os.environ["HF_TOKEN"])
+    else:
+        print("No Hugging Face token found in environment variable HF_TOKEN.")
+
+    claim = "“We will support families with children by introducing free breakfast clubs in every primary school”"
+    start_date = "20250504"
+    timestamp = "xxxxx"
+    user_id = "xxx"
+
+    outputs = run_pipeline(claim, time_start, timestamp, user_id)
+    print("🎯 Pipeline finished. Outputs:", outputs)
+    print(f"⏱️ Total time: {time.time() - start:.2f} seconds")

system/process_time.py

@@ -0,0 +1,238 @@
+import json
+import datetime
+import re
+import pandas as pd
+import os, argparse
+import random
+import csv
+from openai import OpenAI
+from huggingface_hub import hf_hub_download
+import json
+import os
+
+
+
+def gpt_4o_useful(input):
+    client=OpenAI(api_key=os.environ.get("OAI"))
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": input
+                    }
+                ]
+            }
+        ],
+        response_format={"type": "text"},
+        temperature=0.0000000001,
+        max_tokens=4096,
+        top_p=0,
+        frequency_penalty=0,
+        presence_penalty=0,
+        logprobs=True
+    )
+
+    text = response.choices[0].message.content
+
+    if response.choices[0].logprobs and response.choices[0].logprobs.content:
+        first_token_logprob = response.choices[0].logprobs.content[0]
+        token = first_token_logprob.token
+        logprob = first_token_logprob.logprob
+    else:
+        token = None
+        logprob = None
+
+    return text, token, logprob
+
+
+
+def get_ICL(data, top_k=None):
+
+    ICL =""
+    if top_k == None:
+        data = data
+    else:
+        # print(data)
+        data = data[:top_k]
+    for line in data:
+        # line = json.loads(line)
+        pledge = line["pledge"]
+        event = line["event_description"]
+        time = line["event_date"]
+        input=f"Pledge: {pledge}\nEvent Summary: {event} (Event Date: {time})\nIs this event summary useful?"
+        input = input.strip()
+        output = line["label"].strip()
+        ICL = f"{ICL}Input: {input}\nOutput: {output}\n\n"
+    return ICL
+
+def load_json(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    return data
+
+
+def gpt_eval(test_instance, train_data, instruction, suggestion_meta, ICL_id=None):
+
+    if suggestion_meta:
+        # print(ICL_id)
+        
+        train_data = [line for line in train_data if str(line.get("pledge_id")) == str(ICL_id)]
+
+    else:
+        random.seed(42)
+        random.shuffle(train_data)
+
+    ICL = get_ICL(train_data, top_k=50)
+    # print(ICL)
+    input = f"{instruction}\nBelow are examples:\n\n{ICL}Now, please assign a label for the below instance.\nInput: {test_instance}\nOutput:"
+
+    try:
+        text, tokens, logprobs = gpt_4o_useful(input)
+    except Exception as e:
+        print(e)
+        tokens = None
+        logprobs = None
+
+    return tokens, logprobs
+
+def extract_columns_to_dict(file_path, delimiter='\t'):
+
+    data_dict = {}
+
+    with open(file_path, mode='r', encoding='utf-8') as file:
+        reader = csv.reader(file, delimiter=delimiter)
+        for row in reader:
+            if len(row) >= 4:  
+                key = row[2]  
+                value = row[3]  
+                data_dict[key] = value 
+
+    return data_dict
+
+
+def parse_date(date_str):
+    try:
+        return datetime.datetime.strptime(date_str, "%Y-%m-%d"), date_str
+    except ValueError:
+        match = re.search(r'(.*) \(relative to (\d{4}-\d{2}-\d{2})\)', date_str)
+        if match:
+            reference = datetime.datetime.strptime(match.group(2), "%Y-%m-%d")
+            if "Last month" in match.group(1):
+                return reference - datetime.timedelta(days=30), date_str
+            elif "Yesterday" in match.group(1):
+                return reference - datetime.timedelta(days=1), date_str
+            elif "Last week" in match.group(1):
+                return reference - datetime.timedelta(days=7), date_str
+            elif "This week" in match.group(1):
+                return reference, date_str
+        
+        # 处理不同格式的日期
+        match = re.fullmatch(r'\d{4}', date_str)  # 处理年份格式: '2014'
+        if match:
+            return datetime.datetime(int(date_str), 1, 1), date_str
+        
+        match = re.fullmatch(r'(\w+) (\d{4})', date_str)  # 处理月份+年份格式: 'November 2023'
+        if match:
+            try:
+                return datetime.datetime.strptime(date_str, "%B %Y"), date_str
+            except ValueError:
+                return None, date_str
+        
+        match = re.fullmatch(r'(\d{4})-Q(\d)', date_str)  # 处理季度格式: '2024-Q1'
+        if match:
+            year, quarter = int(match.group(1)), int(match.group(2))
+            month = (quarter - 1) * 3 + 1
+            return datetime.datetime(year, month, 1), date_str
+        
+        match = re.fullmatch(r'(\d{4}) (Spring|Summer|Autumn|Fall|Winter)', date_str, re.IGNORECASE)  # 处理季度名称格式: '2023 Autumn' 或 '2023 Fall'
+        if match:
+            year = int(match.group(1))
+            season_map = {"Spring": 3, "Summer": 6, "Autumn": 9, "Fall": 9, "Winter": 12}
+            month = season_map[match.group(2).capitalize()]
+            return datetime.datetime(year, month, 1), date_str
+        
+        return None, date_str
+
+def extract_and_sort_events(data_dir, pledge_date, pledge_author, claim, suggestion_meta):
+
+    events = []
+
+    # url_path = os.path.join(data_dir, "augmented_search_results.tsv")
+    # url_query_dict = extract_columns_to_dict(file_path=url_path, delimiter='\t')
+    
+    pledge = claim.strip()
+
+    file_path = os.path.join(data_dir, "gpt4_event_extraction", "gpt4o_results_0_claim.json")
+    gpt4_results_json = load_json(file_path)
+
+    print(gpt4_results_json)
+    train_file_path = hf_hub_download(
+        repo_id="PledgeTracker/demo_feedback",     
+        filename="train_useful.json",              
+        repo_type="dataset",                      
+        token=os.environ["HF_TOKEN"]              
+    )
+
+    with open(train_file_path, "r", encoding="utf-8") as f:
+        train_data = json.load(f)
+        print(train_data[0])
+
+    instruction = open(f"system/instruction.txt", "r").read()
+    
+    map_file_path = hf_hub_download(
+        repo_id="PledgeTracker/demo_feedback",     
+        filename="mapping.txt",              
+        repo_type="dataset",                      
+        token=os.environ["HF_TOKEN"]              
+    )
+    mapping_f = open(map_file_path, "r").readlines()
+    mapping = {}
+
+    for map_id, line in enumerate(mapping_f):
+        mapping[map_id] = int(line.strip())
+
+    ICL_id = None
+    if suggestion_meta:
+        try:
+            idx = int(suggestion_meta["index"])
+            ICL_id = mapping.get(idx)
+            print(f"[Suggestion] index: {idx} → pledge_id: {ICL_id}")
+        except Exception as e:
+            print(f"[Mapping error]: {e}")
+
+    for doc in gpt4_results_json:
+        mete_date = doc["date"]
+        for event in doc.get("output", {}).get("events", []):
+            parsed_date, original_date = parse_date(event["date"])
+            if parsed_date:
+                if mete_date!= parsed_date:
+                    event_date_and_pub_date = original_date+f" ({mete_date})"
+                else:
+                    event_date_and_pub_date = original_date
+                
+                test_instance = f"Pledge: {pledge} (Speaker: {pledge_author}; Pledge Date: {pledge_date})\nEvent Summary: {event['event']} (Event Date: {original_date})\nIs this event summary useful?"
+
+                print(test_instance)
+
+                label, score = gpt_eval(test_instance, train_data, instruction, suggestion_meta, ICL_id=ICL_id)
+
+                URL = doc["url"]
+                events.append({
+                    "date": original_date, 
+                    "event date (publication date if different)": event_date_and_pub_date,
+                    "event": event["event"],
+                    "url": URL,
+                    "label": label,
+                    "confident": score
+                })
+    
+    # 按时间排序
+    events.sort(key=lambda x: parse_date(x["date"])[0], reverse=True)
+    return events
+
+
+

system/scraper.py

@@ -0,0 +1,95 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import os
+import csv
+import json
+import fitz
+import time
+import requests
+import pandas as pd
+from time import sleep
+from pathlib import Path
+from system.html2lines import url2lines, line_correction, html2metadata
+
+MAX_RETRIES = 3
+TIMEOUT = 5  # seconds
+
+
+def scrape_text_from_url(url, temp_name):
+    response = None
+    for attempt in range(MAX_RETRIES):
+        try:
+            response = requests.get(url, timeout=TIMEOUT)
+            break
+        except requests.RequestException:
+            if attempt < MAX_RETRIES - 1:
+                sleep(3)
+
+    if response is None or response.status_code == 503:
+        return []
+
+    if url.endswith(".pdf"):
+        pdf_dir = Path("/tmp/pdf_dir")
+        pdf_dir.mkdir(parents=True, exist_ok=True)
+        pdf_path = pdf_dir / f"{temp_name}.pdf"
+        with open(pdf_path, "wb") as f:
+            f.write(response.content)
+
+        extracted_text = ""
+        doc = fitz.open(str(pdf_path))
+        for page in doc:
+            extracted_text += page.get_text() or ""
+
+        return line_correction(extracted_text.split("\n"))
+
+    return line_correction(url2lines(url))
+    
+def process_row(row, claim_id):
+    try:
+        url = row[2]
+        json_data = {
+            "claim_id": claim_id,
+            "type": row[1],
+            "query": row[3],
+            "url": url,
+            "url2text": scrape_text_from_url(url, claim_id),
+            "metadata": {}
+        }
+        meta = html2metadata(url)
+        json_data["metadata"] = {
+            "title": meta.get("title"),
+            "date": meta.get("date")
+        }
+        return json_data
+    except Exception as e:
+        print(f"[WARN] Failed to scrape {row[2]}: {e}")
+        return None
+
+def run_scraper(tsv_file_path: str, output_jsonl_path: str, max_workers: int = 10):
+    claim_id = Path(tsv_file_path).stem
+    output_jsonl_path = Path(output_jsonl_path)
+    output_jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if output_jsonl_path.exists():
+        print(f"[INFO] Skipping processing as output file already exists: {output_jsonl_path}")
+        return str(output_jsonl_path)
+
+    try:
+        df = pd.read_csv(tsv_file_path, sep="\t", header=None)
+        print("[INFO] Data loaded successfully with Pandas.")
+    except Exception as e:
+        raise RuntimeError(f"[ERROR] Failed to load TSV: {e}")
+
+    results = []
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(process_row, row, claim_id) for _, row in df.iterrows()]
+        for future in as_completed(futures):
+            result = future.result()
+            if result:
+                results.append(result)
+
+    with open(output_jsonl_path, "w", encoding="utf-8") as json_file:
+        for item in results:
+            json_file.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+    print(f"[SYSTEM] Output saved to {output_jsonl_path}")
+    return str(output_jsonl_path)

system/test.py

@@ -0,0 +1,15 @@
+from huggingface_hub import hf_hub_download
+import json
+import os
+
+file_path = hf_hub_download(
+    repo_id="PledgeTracker/demo_feedback",     # 你的私密 dataset 名
+    filename="train_useful.json",              # 你上传的文件名
+    repo_type="dataset",                       # 必须设置为 dataset 类型
+    token=os.environ["HF_TOKEN"]               # 需要 HF token 才能访问私密文件
+)
+
+with open(file_path, "r", encoding="utf-8") as f:
+    train_data = json.load(f)
+
+print(train_data[0])

test.html

@@ -0,0 +1,513 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Pledge Tracker – Demo</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+  </head>
+  <body class="bg-gray-50 text-gray-800">
+    <header class="bg-white shadow py-4 sticky top-0 z-10">
+      <div class="container mx-auto flex items-center justify-between px-4">
+        <div class="flex items-center gap-2">
+          <span class="text-2xl font-bold text-purple-600">🤗</span>
+          <span class="font-semibold text-lg">Pledge Tracking</span>
+        </div>
+        <nav class="hidden md:flex gap-6 font-medium">
+          <a class="hover:text-purple-600" href="#eval-response">Track Your Pledge</a>
+          <a class="hover:text-purple-600" href="#about">About</a>
+        </nav>
+      </div>
+    </header>
+
+    <section class="py-16 bg-gradient-to-r from-purple-50 to-purple-50 text-center">
+      <div class="container mx-auto px-4 max-w-2xl">
+        <h1 class="text-3xl md:text-4xl font-extrabold mb-4">
+          Fact-Checking Election Promises
+        </h1>
+        <p class="text-lg text-gray-600">
+          Extract progress towards fulfilling the promise.
+        </p>
+      </div>
+    </section>
+
+    <section id="eval-response" class="py-12">
+      <div class="container mx-auto px-4 max-w-4xl">
+        <!-- <h2 class="text-2xl font-bold mb-6">Track Manifesto Pledge</h2> -->
+        <label for="claim" class="block text-sm font-medium mb-2">
+          Please enter the pledge:
+        </label>
+        <textarea
+          id="claim"
+          class="w-full border rounded-lg p-3 h-40 focus:outline-none focus:ring-2 focus:ring-purple-500"
+          placeholder="For example: 'We will support families with children by introducing free breakfast clubs in every primary school...'"
+        ></textarea>
+
+        <div id="similar-suggestions" class="mt-3 text-sm text-gray-600 hidden"></div>
+
+        <div class="mt-4">
+          <label for="pledge-date" class="block text-sm font-medium mb-2">
+            When was this pledge made?
+          </label>
+          <div class="grid grid-cols-[1fr_auto] items-center gap-2">
+            <input
+              type="date"
+              id="pledge-date"
+              class="w-full border rounded-lg p-2"
+            />
+            <button
+              onclick="setDefaultDate()"
+              type="button"
+              class="px-2 py-1 text-sm bg-purple-600 text-white rounded hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500"
+            >
+              Use default: 4th Jul 2024
+            </button>
+          </div>
+          <div id="date-warning" class="text-sm text-red-600 mt-1 hidden">
+            Please select a date or click the button to use the default.
+          </div>
+        </div>
+        
+        <div class="mt-4">
+          <label for="pledge-author" class="block text-sm font-medium mb-2">
+            Who made this pledge?
+          </label>
+          <div class="grid grid-cols-[1fr_auto] items-center gap-2">
+            <input
+              type="text"
+              id="pledge-author"
+              class="w-full border rounded-lg p-2"
+              placeholder="Enter the name of the party or person"
+            />
+            <button
+              onclick="setDefaultAuthor()"
+              type="button"
+              class="px-2 py-1 text-sm bg-purple-600 text-white rounded hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500"
+            >
+              Use default: Labour
+            </button>
+          </div>
+          <div id="author-warning" class="text-sm text-red-600 mt-1 hidden">
+            Please enter a speaker or click the button to use the default.
+          </div>
+        </div>
+        
+        
+
+        <label for="time-range" class="block text-sm font-medium mt-4 mb-2">
+          Please select a time range:
+        </label>
+        <select id="time-range" class="w-full border rounded-lg p-2">
+          <option value="week">Past one week</option>
+          <option value="month">Past one month</option>
+          <!-- <option value="year">From when the pledge was made</option> -->
+          <option value="since_pledge_date">From when the pledge was made</option>
+        </select>
+
+        <button
+          id="check"
+          class="mt-4 px-6 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500"
+        >
+          Let's fact check!
+        </button>
+
+        <div id="progress" class="mt-6 hidden border p-4 rounded-lg bg-white shadow">
+          <h3 class="font-semibold mb-2">System Progress</h3>
+          <div id="status" class="text-sm text-gray-800 font-normal leading-relaxed"></div>
+        </div>
+
+
+        <div id="result" class="mt-6 hidden border p-4 rounded-lg bg-white shadow">
+          <h3 class="font-semibold mb-2">Result</h3>
+          <p class="text-gray-700"></p>
+        </div>
+      </div>
+    </section>
+
+    <section id="about" class="py-12">
+      <div class="container mx-auto px-4 max-w-4xl">
+        <h2 class="text-2xl font-bold mb-6">About</h2>
+        <p class="text-gray-700 leading-relaxed">
+          This demo connects a static front-end with a Python back-end using Flask.
+          The back-end generates event data and returns structured events related
+          to a manifesto pledge.
+        </p>
+      </div>
+    </section>
+
+
+    
+
+    <script>
+      let suggestedPledge = null; 
+      let currentAbortController = null;
+      const feedbackData = {};
+      let lastUsedFile = null;
+      let lastUserId = null;
+      let lastTimestamp = null;
+      const checkBtn = document.getElementById("check");
+
+      const stepListStandard = {
+        1: "Retrieving evidence related to the pledge",
+        2: "Scraping documents from URLs",
+        3: "Generating more queries based on the retrieved evidence",
+        4: "Searching more articles",
+        5: "Scraping documents from URLs",
+        6: "Finding the most relevant documents",
+        7: "Extracting events from top documents",
+        8: "Sorting events temporally"
+      };
+
+      const stepListSuggestion = {
+        1: "Retrieving evidence based on genertaed queries",
+        2: "Scraping documents from URLs",
+        3: "Finding the most relevant documents",
+        4: "Extracting events from top documents",
+        5: "Sorting events temporally"
+      };
+
+      let stepList = stepListStandard;
+
+      function renderStatus(statusDict) {
+        let html = "<ul class='list-disc ml-6 space-y-1 text-sm'>";
+        for (let step in stepList) {
+          const content = statusDict?.[step] || stepList[step];
+          const prefix = statusDict?.[step] ? "✅" : "⏳";
+          html += `<li>${prefix} Step ${step}: ${content}</li>`;
+        }
+        html += "</ul>";
+        return html;
+      }
+
+      function setDefaultDate() {
+        const input = document.getElementById("pledge-date");
+        input.value = "2024-07-04";
+        document.getElementById("date-warning").classList.add("hidden");
+      }
+
+      function setDefaultAuthor() {
+        const input = document.getElementById("pledge-author");
+        input.value = "Labour";
+        document.getElementById("author-warning").classList.add("hidden");
+      }
+
+      // function setFeedback(index, answer) {
+      //   feedbackData[index] = answer;
+      //   const message = document.getElementById(`msg-${index}`);
+      //   message.textContent = `✓ Selected: ${answer ? 'Yes' : 'No'}`;
+      //   message.className = answer
+      //     ? "text-sm text-green-600 mt-1"
+      //     : "text-sm text-red-600 mt-1";
+      // }
+      function setFeedback(index, answer) {
+        feedbackData[index] = answer;
+        const message = document.getElementById(`msg-${index}`);
+        
+        let displayText = "";
+        let colorClass = "";
+        
+        switch(answer) {
+          case "not_relevant":
+            displayText = "Not relevant";
+            colorClass = "text-red-300";
+            break;
+          case "relevant_seen":
+            displayText = "Relevant but already seen";
+            colorClass = "text-grey-400";
+            break;
+          case "relevant_updated":
+            displayText = "Relevant and up-to-date";
+            colorClass = "text-blue-400";
+            break;
+        }
+        
+        message.textContent = `✓ Selected: ${displayText}`;
+        message.className = `text-sm ${colorClass} mt-1`;
+      }
+
+      function pollStatus(userId, timestamp, statusElement) {
+        if (window.pollIntervalId) {
+          clearInterval(window.pollIntervalId);
+        }
+
+        window.pollIntervalId = setInterval(async () => {
+          try {
+            const res = await fetch(`/api/status?user_id=${userId}&timestamp=${timestamp}&_=${Date.now()}`);
+            const data = await res.json();
+
+            // 动态渲染结构化状态
+            if (data.status) {
+              statusElement.innerHTML = renderStatus(data.status);
+            }
+
+            // 检查是否完成
+            const values = Object.values(data.status || {});
+            const finalText = values.join(" ").toLowerCase();
+
+            if (finalText.includes("done") || finalText.includes("finished")) {
+              clearInterval(window.pollIntervalId);
+              window.pollIntervalId = null;
+              statusElement.innerHTML += `<div class="mt-2 text-green-600 font-semibold">✅ All done.</div>`;
+              checkBtn.disabled = false;
+              checkBtn.classList.remove("opacity-50", "cursor-not-allowed");
+              if (lastUsedFile) loadEvents(lastUsedFile);
+            } else if (finalText.includes("error") || finalText.includes("fail")) {
+              clearInterval(window.pollIntervalId);
+              window.pollIntervalId = null;
+              statusElement.innerHTML += `<div class="mt-2 text-red-600 font-semibold">❌ The process failed.</div>`;
+              checkBtn.disabled = false;
+              checkBtn.classList.remove("opacity-50", "cursor-not-allowed");
+            }
+          } catch (err) {
+            clearInterval(window.pollIntervalId);
+            window.pollIntervalId = null;
+            statusElement.innerHTML = `<div class="text-red-600">❌ Failed to check status: ${err.message}</div>`;
+          }
+        }, 2000);
+      }
+
+
+
+      async function submitAllFeedback() {
+        const entries = Object.entries(feedbackData);
+        if (entries.length === 0) {
+          alert("No feedback to submit!");
+          return;
+        }
+        const confirmed = confirm("Submit all feedback?");
+        if (!confirmed) return;
+
+        const pledgeText = document.getElementById("claim").value.trim();
+
+        const res = await fetch('/api/feedback', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            pledge: pledgeText,
+            file: lastUsedFile,
+            user_id: lastUserId,       
+            timestamp: lastTimestamp,
+            feedback: entries.map(([index, answer]) => ({ 
+              eventIndex: index, 
+              answer: answer 
+            }))
+          })
+        });
+
+        alert(res.ok ? "✅ Feedback submitted successfully!" : "❌ Submission failed.");
+      }
+
+      async function loadEvents(file) {
+        const resultBox = document.getElementById("result");
+        const p = resultBox.querySelector("p");
+        resultBox.classList.remove("hidden");
+
+        try {
+          const fileParam = encodeURIComponent(file);
+          const eventsRes = await fetch(`/api/events?file=${fileParam}`);
+          if (!eventsRes.ok) throw new Error("Event file not found or malformed");
+          const data = await eventsRes.json();
+          if (!Array.isArray(data)) throw new Error("Unexpected data format");
+
+          p.innerHTML = `<strong>We have found ${data.length} events for this pledge.</strong><br><br>` +
+            data.map((e, index) => `
+              <div class="mb-6 border-b pb-4">
+                🗓️ <b>${e.date}</b>: ${e.event}<br>
+                🔗 <a href="${e.url}" target="_blank" class="text-purple-400 underline">Source</a>
+                
+                <div class="mt-3">
+                  <label class="block text-sm font-medium mb-2">How relevant is this event?</label>
+                  <div class="flex flex-wrap gap-2">
+                    <button onclick="setFeedback(${index}, 'not_relevant')" 
+                            class="px-3 py-1.5 bg-gray-100 hover:bg-gray-200 border border-gray-300 rounded-lg text-gray-700">
+                      Not relevant
+                    </button>
+                    <button onclick="setFeedback(${index}, 'relevant_seen')" 
+                            class="px-3 py-1.5 bg-blue-100 hover:bg-blue-200 border border-blue-300 rounded-lg text-blue-700">
+                      Relevant but seen
+                    </button>
+                    <button onclick="setFeedback(${index}, 'relevant_updated')" 
+                            class="px-3 py-1.5 bg-green-100 hover:bg-green-200 border border-green-300 rounded-lg text-green-700">
+                      Relevant & up-to-date
+                    </button>
+                  </div>
+                  <div id="msg-${index}" class="text-sm mt-1"></div>
+                </div>
+              </div>
+            `).join('') +
+            `<button onclick="submitAllFeedback()" class="mt-6 px-4 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700">
+              📤 Submit All Feedback
+            </button>
+            <button onclick="window.location.href='/download?file=${fileParam}'" class="mt-4 ml-4 px-4 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700">
+              📅 Download Excel
+            </button>`;
+        } catch (err) {
+          p.textContent = `❌ Failed to load timeline: ${err.message}`;
+        }
+      }
+
+      let suggestTimer = null;
+        document.getElementById("claim").addEventListener("input", () => {
+          clearTimeout(suggestTimer);
+          suggestTimer = setTimeout(fetchSuggestions, 300); // 300ms delay to avoid flooding
+        });
+
+        async function fetchSuggestions() {
+          const claimText = document.getElementById("claim").value.trim();
+          const suggestionBox = document.getElementById("similar-suggestions");
+
+          if (!claimText) {
+            suggestionBox.classList.add("hidden");
+            return;
+          }
+
+          const res = await fetch("/api/similar-pledges", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ claim: claimText })
+          });
+          const data = await res.json();
+          const suggestions = data.suggestions || [];
+
+          if (suggestions.length === 0) {
+            suggestionBox.classList.add("hidden");
+          } else {
+            const author = "Labour";
+            const date = "2024-07-04";
+            suggestionBox.innerHTML =
+              "<div class='font-semibold mb-1'>💡 Are you fact-checking ... </div>" +
+              "<ul class='list-disc ml-6 mt-1'>" +
+              suggestions.map(s => `
+                <li class="mb-2">
+                  ${author}: ${s.text} (${date})
+                  <button
+                    onclick="useSuggestedPledge('${s.text.replace(/'/g, "\\'")}', ${s.index})"
+                    class="ml-2 px-2 py-1 text-xs bg-purple-600 text-white rounded hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500">
+                    Fact-check this pledge
+                  </button>
+                </li>
+              `).join("") +
+              "</ul>";
+            suggestionBox.classList.remove("hidden");
+          }
+        }
+
+        
+      checkBtn.addEventListener("click", async () => {
+        const claim = document.getElementById("claim").value.trim();
+        const pledgeDate = document.getElementById("pledge-date").value.trim();
+        const pledgeAuthor = document.getElementById("pledge-author").value.trim();
+        const statusElement = document.getElementById("status");
+        const resultBox = document.getElementById("result");
+        // resultBox.classList.remove("hidden");
+        const p = resultBox.querySelector("p");
+
+
+
+        let valid = true;
+        if (!claim) {
+          alert("Please enter the pledge text.");
+          valid = false;
+        }
+        if (!pledgeDate) {
+          document.getElementById("date-warning").classList.remove("hidden");
+          valid = false;
+        }
+        if (!pledgeAuthor) {
+          document.getElementById("author-warning").classList.remove("hidden");
+          valid = false;
+        }
+
+        if (!valid) return;
+
+        checkBtn.disabled = true;
+        checkBtn.classList.add("opacity-50", "cursor-not-allowed");
+
+        // document.getElementById("status").classList.remove("hidden");
+        statusElement.innerHTML = renderStatus({}); 
+        document.getElementById("result").classList.remove("hidden");
+        document.getElementById("progress").classList.remove("hidden");
+
+
+
+      try {
+          const timeRange = document.getElementById("time-range").value;
+          const pledgeDate = document.getElementById("pledge-date").value;
+          const pledgeAuthor = document.getElementById("pledge-author").value;
+          if (currentAbortController) currentAbortController.abort();
+          currentAbortController = new AbortController();
+          const signal = currentAbortController.signal;
+          let valid = true;
+
+          stepList = (suggestedPledge !== null) ? stepListSuggestion : stepListStandard;
+
+          if (!pledgeDate) {
+            document.getElementById("date-warning").classList.remove("hidden");
+            valid = false;
+          }
+          if (!pledgeAuthor) {
+            document.getElementById("author-warning").classList.remove("hidden");
+            valid = false;
+          }
+          if (!valid) return; 
+
+          const userId = Math.random().toString(36).substring(2, 10);  
+          const now = new Date();
+          const timestamp = now.toISOString().replace(/[:.]/g, "-").slice(0, 19); 
+          statusElement.textContent = ""; 
+          // pollStatus(userId, timestamp, p); 
+          pollStatus(userId, timestamp, document.getElementById("status"));
+
+
+          const runRes = await fetch("/api/run-model", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+              claim,
+              time_range: timeRange,
+              pledge_date: pledgeDate,
+              pledge_author: pledgeAuthor,
+              user_id: userId,
+              timestamp: timestamp,
+              signal: signal,
+              suggestion_meta: suggestedPledge
+            })
+          });
+
+          const runData = await runRes.json();
+
+          lastUsedFile = runData.file;
+          lastUserId = runData.user_id;    
+          lastTimestamp = runData.timestamp;
+        } catch (err) {
+          if (err.name === "AbortError") {
+            console.log("Previous request aborted.");
+            checkBtn.disabled = false;
+            checkBtn.classList.remove("opacity-50", "cursor-not-allowed");
+            return;
+          }
+          p.textContent = `❌ Failed to load timeline: ${err.message}`;
+        }
+
+      });
+
+    
+      async function useSuggestedPledge(text, index) {
+          document.getElementById("claim").value = text;
+          document.getElementById("pledge-author").value = "Labour";
+          document.getElementById("pledge-date").value = "2024-07-04";
+          suggestedPledge = { text, index };
+          alert("✅ This pledge has been filled in. You can now click 'Let's fact check!'");
+          await fetch("/api/log-similar-selection", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+              selected_text: text,
+              index: index
+            })
+          });
+        }
+
+    </script>
+  </body>
+</html>