Add authorizing through environment var

app.py

@@ -6,14 +6,6 @@ from functools import partial
 from datasets import load_dataset
 from pathlib import Path
 
-# get secret environment variable
-token = os.environ["HF_TOKEN"]
-# write to disk
-path = Path("./huggingface")
-path.mkdir(parents=True, exist_ok=True)
-with open(path/"token", "w") as f:
-    f.write(token)
-
 dataset_names = [
     "AI4Code",
     "AMPS",
@@ -44,7 +36,7 @@ for name in dataset_names:
     ds = load_dataset(
         "CarperAI/pilev2_smol_metadata",
         data_files=path,
-        use_auth_token=True,
+        use_auth_token=os.environ["HF_TOKEN"],
         split="train",
         # download_mode="force_redownload",
     )