File size: 4,387 Bytes
1f05644 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
# This file is used to compute the embedding of the technologies, easily executable on google colab
#!pip install sentence-transformers
#!pip install nltk
import numpy as np
from sentence_transformers import SentenceTransformer
import pickle
import pandas as pd
import nltk
from nltk.stem import *
nltk.download("punkt_tab")
print("Loading SentenceTransformer model...")
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
print("Model loaded.")
def load_technologies():
df = pd.read_excel('technologies_database.xlsx')
return df
def tech_to_dict(technologies):
tech_dict = []
for index, tech in enumerate(technologies):
if not tech.find("<title>") > 1:
tab = tech.split("\n")
tab.pop(0)
tab.pop(len(tab)-1)
tech_dict.append({"title": tab[0][tab[0].find(": ")+2:],
"purpose": tab[1][tab[1].find(": ")+2:],
"key_components": tab[2][tab[2].find(": ")+2:],
"advantages": tab[3][tab[3].find(": ")+2:],
"limitations": tab[4][tab[4].find(": ")+2:],
"id": index})
return tech_dict
def stem(data,data_type):
stemmer = SnowballStemmer("english")
processed_data = []
if data_type == "technologies":
for t_item in data:
processed_data.append({
"title": stemmer.stem(t_item["title"]),
"purpose": stemmer.stem(t_item["purpose"]),
"key_components": stemmer.stem(t_item["key_components"]),
"advantages": stemmer.stem(t_item["advantages"]),
"limitations": stemmer.stem(t_item["limitations"]),
"id": t_item["id"]
})
else:
for t_item in data:
print(t_item)
processed_data.append({
"title": stemmer.stem(t_item),
"description": stemmer.stem(data[t_item])
})
return processed_data
def preprocess_tech_data(_df):
if _df is None or "description" not in _df.columns:
return [], []
technologies_list = _df["description"].to_list()
tech_dict_raw = tech_to_dict(technologies_list)
tech_dict_filtered = [
t for t in tech_dict_raw if (
len(t.get("title", "")) >= 5 and
len(t.get("advantages", "")) >= 5 and
len(t.get("key_components", "")) >= 5
)
]
if not tech_dict_filtered:
return [], []
processed_tech_wt = stem(tech_dict_filtered,"technologies")
for t_item_wt in processed_tech_wt:
kc = t_item_wt.get("key_components")
if isinstance(kc, str):
t_item_wt["key_components"] = ''.join(nltk.sent_tokenize(kc))
else:
t_item_wt["key_components"] = ""
original_tech_for_display = tech_dict_filtered[:len(processed_tech_wt)]
_keys = list(processed_tech_wt[0].keys()) if processed_tech_wt else []
return processed_tech_wt, _keys, original_tech_for_display
df = load_technologies()
global_tech,keys,original_tech = preprocess_tech_data(df)
global_tech_purposes = [t["purpose"] for t in global_tech]
# Encode all global_tech purposes into embeddings
print("Encoding global_tech purposes into embeddings... This might take a while for 1000 elements.")
global_tech_embeddings = model.encode(global_tech_purposes, show_progress_bar=True)
print("Global tech embeddings created.")
# Define the filename for the pickle file
output_filename = 'global_tech_embeddings.pkl'
# Save the embeddings and the global_tech data (optional, but good for context)
# Saving global_tech alongside embeddings ensures you have the original data if needed
data_to_save = {
'global_tech': global_tech, # The original list of dictionaries
'global_tech_embeddings': global_tech_embeddings # The numpy array of embeddings
}
print(f"Saving embeddings and global_tech data to {output_filename}...")
with open(output_filename, 'wb') as f:
pickle.dump(data_to_save, f)
print(f"Data saved successfully to {output_filename}.")
print(f"\nTo load this file later in your API, use: \n"
f"with open('{output_filename}', 'rb') as f:\n"
f" loaded_data = pickle.load(f)\n"
f"global_tech = loaded_data['global_tech']\n"
f"global_tech_embeddings = loaded_data['global_tech_embeddings']\n")
|