dataset model csv ingles

app.py

@@ -477,27 +477,22 @@ class ModeloDataset:
             print('idioma:',idioma)            
             self.tokenizer = AutoTokenizer.from_pretrained("dayannex/distilbert-tuned-4labels")
             self.model = AutoModelForTokenClassification.from_pretrained("dayannex/distilbert-tuned-4labels")     
-            print('_sentences',_sentences)
-            print('type(_sentences)',type(_sentences))
-            inputs = self.tokenizer(_sentences, padding=True, truncation=True, return_tensors="pt", max_length=512)
-            print('inputs')
-            #model.eval()
-
+            sentences_list = _sentences.apply(lambda x: x[0].tolist() if isinstance(x, np.ndarray) else x.tolist())
+            inputs = self.tokenizer(list(sentences_list), padding=True, truncation=True, return_tensors="pt", max_length=512)
             with torch.no_grad():
                 outputs = model(**inputs)
-            print('out')
-        
+
+
             logits = outputs.logits
-            print('logits',logits)
             predictions = torch.argmax(logits, dim=2)
-           
+
             id2label = model.config.id2label
 
 
             all_tokens = []
             all_label_ids = []
             all_labels = []
-            for i, sentence in enumerate(_sentences):
+            for i, sentence in enumerate(sentences_list):
             
                 tokens = self.tokenizer.convert_ids_to_tokens(inputs.input_ids[i])
                 label_ids = predictions[i].tolist()
@@ -537,11 +532,12 @@ class ModeloDataset:
                             i=i+1
             print('new_tokens') 
             print(new_tokens[1])
-            print(all_tokens[1])
+            #print(all_tokens[1])
 
             print(len(new_tokens[1]))
             print(len(new_identificadores[1]))
-            
+            print(new_identificadores[1])
+                        
             
             return new_identificadores, new_tokens