File size: 95,356 Bytes
33f7caa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V3bxTMy4MTcO"
      },
      "source": [
        "#Create Corpus\n",
        "\n",
        "[CSV file with Azure & Microsoft links created](https://docs.google.com/spreadsheets/d/1b_QcHNPBg34Q05FPsmRqzPW5XTUGmedaP_SLonyiMt4/edit?usp=sharing)\n",
        "\n",
        "Columns:\n",
        "- Name\n",
        "- URL\n",
        "- Content type"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y0DKd30YMxAw",
        "outputId": "92cf65ad-bd80-4230-e6cf-0087000816ea"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/56.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m77.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m454.8/454.8 kB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m45.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m80.2/80.2 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.2/71.2 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m77.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m266.8/266.8 kB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.1/211.1 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.1/45.1 MB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.4/9.4 MB\u001b[0m \u001b[31m80.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.8/45.8 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.3/81.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m304.2/304.2 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.4/107.4 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m499.2/499.2 kB\u001b[0m \u001b[31m32.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.9/50.9 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.3/129.3 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for html2text (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for tinysegmenter (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for spider-client (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for feedfinder2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for jieba3k (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sgmllib3k (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torch 2.6.0+cu124 requires nvidia-cublas-cu12==12.4.5.8; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cublas-cu12 12.5.3.2 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cuda-cupti-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cuda-cupti-cu12 12.5.82 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cuda-nvrtc-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cuda-nvrtc-cu12 12.5.82 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cuda-runtime-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cuda-runtime-cu12 12.5.82 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cudnn-cu12==9.1.0.70; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cudnn-cu12 9.3.0.75 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cufft-cu12==11.2.1.3; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cufft-cu12 11.2.3.61 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-curand-cu12==10.3.5.147; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-curand-cu12 10.3.6.82 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cusolver-cu12==11.6.1.9; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cusolver-cu12 11.6.3.83 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-cusparse-cu12==12.3.1.170; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-cusparse-cu12 12.5.1.3 which is incompatible.\n",
            "torch 2.6.0+cu124 requires nvidia-nvjitlink-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\", but you have nvidia-nvjitlink-cu12 12.5.82 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "# Install requirements\n",
        "\n",
        "!pip install -q llama-index==0.12.12 openai==1.59.6 tiktoken==0.8.0 llama-index-readers-web==0.3.4 firecrawl-py==2.7.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MdEa-LKlNef8"
      },
      "outputs": [],
      "source": [
        "# set variables\n",
        "import os\n",
        "\n",
        "from google.colab import userdata\n",
        "\n",
        "os.environ[\"OPENAI_API_KEY\"] = userdata.get('openai_api_key')\n",
        "os.environ[\"FIRECRAWL_API_KEY\"] = userdata.get('FIRECRAWL_API_KEY')\n",
        "os.environ[\"HF_TOKEN\"] = userdata.get('HF_TOKEN2')\n",
        "\n",
        "FIRECRAWL_API_KEY = userdata.get('FIRECRAWL_API_KEY')\n",
        "HF_TOKEN = userdata.get('HF_TOKEN2')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5r73CntzOAc4"
      },
      "outputs": [],
      "source": [
        "# Download dataset list\n",
        "\n",
        "import requests\n",
        "import csv\n",
        "\n",
        "# Google Sheets file URL (CSV export link)\n",
        "url = 'https://docs.google.com/spreadsheets/d/1b_QcHNPBg34Q05FPsmRqzPW5XTUGmedaP_SLonyiMt4/export?format=csv'\n",
        "\n",
        "# Send a GET request to fetch the CSV file\n",
        "response = requests.get(url)\n",
        "\n",
        "response_list = []\n",
        "\n",
        "# Check if the request was successful\n",
        "if response.status_code == 200:\n",
        "    # Decode the content to a string\n",
        "    content = response.content.decode('utf-8')\n",
        "\n",
        "    # Use the csv.DictReader to read the content as a dictionary\n",
        "    csv_reader = csv.DictReader(content.splitlines(), delimiter=',')\n",
        "    response_list = [row for row in csv_reader]\n",
        "else:\n",
        "    print(f\"Failed to retrieve the file: {response.status_code}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wi6LzQXkgPdM",
        "outputId": "099faa6e-baeb-4846-f61e-fb06da3702bc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "CSV data\n",
            "[{'Content type': 'Technical blogs and expert solutions',\n",
            "  'Exclude': '',\n",
            "  'Name': 'Tech Community',\n",
            "  'URL': 'https://techcommunity.microsoft.com/'}]\n"
          ]
        }
      ],
      "source": [
        "import pprint\n",
        "print(\"CSV data\")\n",
        "pprint.pprint(response_list[2:3])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GVVsd8O5gbmZ"
      },
      "outputs": [],
      "source": [
        "# Initialise Firecrawl\n",
        "import os\n",
        "from firecrawl import FirecrawlApp\n",
        "\n",
        "app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "Z6FHM0s_g4cu",
        "outputId": "56ee85c4-5341-473d-fb33-b303de645e86"
      },
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'response_list' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-2630878180>\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mscraped_pages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwebsite_dict\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse_list\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m     \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwebsite_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'URL'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mexcludePaths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwebsite_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Exclude'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'response_list' is not defined"
          ]
        }
      ],
      "source": [
        "import time\n",
        "from firecrawl import ScrapeOptions\n",
        "\n",
        "# Crawl websites and handle responses\n",
        "url_response = {}\n",
        "crawl_per_min = 15  # Max crawl per minute\n",
        "\n",
        "# Track crawls\n",
        "crawled_websites = 0\n",
        "scraped_pages = 0\n",
        "\n",
        "for i, website_dict in enumerate(response_list[2:3]):\n",
        "    url = website_dict.get('URL')\n",
        "    excludePaths = website_dict.get('Exclude')\n",
        "    print(f\"Crawling: {url}\")\n",
        "\n",
        "    try:\n",
        "        response = app.crawl_url(\n",
        "            url,\n",
        "            limit=15000,\n",
        "            scrape_options=ScrapeOptions(formats=['markdown']),\n",
        "            exclude_paths=[excludePaths]\n",
        "        )\n",
        "\n",
        "        #'includePaths': [r\"^https:\\/\\/[^\\/]+(?:\\/en-US(?:\\/.*)?|(\\/(?![a-z]{2}-[A-Z]{2})([^\\/]+)?(?:\\/[^\\/]*)*)?)$\"],\n",
        "        crawled_websites += 1\n",
        "\n",
        "\n",
        "    except Exception as exc:\n",
        "        print(f\"Failed to fetch {url} -> {exc}\")\n",
        "        continue\n",
        "\n",
        "    # Store the scraped data and associated info in the response dict\n",
        "    url_response[url] = {\n",
        "        \"scraped_data\": response.data,\n",
        "        \"csv_data\": website_dict\n",
        "    }\n",
        "\n",
        "    # Pause to comply with crawl per minute limit for free version its 1 crawl per minute\n",
        "    if i!=len(response_list) and (i + 1) % crawl_per_min == 0:\n",
        "        print(\"Pausing for 1 minute to comply with crawl limit...\")\n",
        "        time.sleep(60)  # Pause for 1 minute after every crawl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gq4EiKzVx2hg"
      },
      "outputs": [],
      "source": [
        "# Initialise HG\n",
        "\n",
        "from huggingface_hub import HfApi\n",
        "\n",
        "api = HfApi(token=HF_TOKEN)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0qAM_aeni0A3"
      },
      "outputs": [],
      "source": [
        "from llama_index.core import Document\n",
        "documents = []\n",
        "\n",
        "for _, scraped_content in url_response.items():\n",
        "    csv_data = scraped_content.get(\"csv_data\")\n",
        "    scraped_results = scraped_content.get(\"scraped_data\")\n",
        "\n",
        "\n",
        "    for result in scraped_results:\n",
        "        markdown_content = result.markdown\n",
        "        title = result.metadata.get(\"title\")\n",
        "        url = result.metadata.get(\"sourceURL\")\n",
        "        documents.append(\n",
        "            Document(\n",
        "                text=markdown_content,\n",
        "                metadata={\n",
        "                    \"title\": title,\n",
        "                    \"url\": url,\n",
        "                    \"name\": csv_data.get(\"Name\"),\n",
        "                    \"contentType\": csv_data.get(\"Content type\")\n",
        "                }\n",
        "            )\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AqdCTtzqqxnX",
        "outputId": "1312691a-abe4-498d-ea37-64e25951f1fb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Documents\n",
            "1\n",
            "[Document(id_='59766750-ed2d-4359-98d0-a8024f5b7c3b', embedding=None, metadata={'title': 'Azure updates | Microsoft Azure', 'url': 'https://azure.microsoft.com/en-us/updates/', 'name': 'Azure Blog / Updates', 'contentType': 'Platform changes, new features'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, metadata_template='{key}: {value}', metadata_separator='\\n', text_resource=MediaResource(embeddings=None, data=None, text='Trace Id is missing\\n\\n[Skip to main content](javascript:void(0))\\n\\n![](https://cdn-dynmedia-1.microsoft.com/is/image/microsoftcorp/ACOM-Roadmap-hero?resMode=sharp2&op_usm=1.5,0.65,15,0&wid=1600&hei=635&qlt=85&fit=constrain)\\n\\n# Azure Updates\\n\\nGet the latest updates on Azure products and features to meet your cloud investment needs. Subscribe to notifications to stay informed.\\n\\n[Get the user guide](https://go.microsoft.com/fwlink/?linkid=2289891&clcid=0x409)[Subscribe via RSS](https://www.microsoft.com/releasecommunications/api/v2/azure/rss)\\n\\nFilter(8768 updates)\\n\\nSave filters\\n\\nClear filters\\n\\nApply filters\\n\\nFilter(8768 updates)\\n\\nFilter\\n\\nSort\\n\\n- RECENTLY MODIFIED\\n\\n- Newest to oldest\\n\\n- Oldest to newest\\n\\n\\nLoading...\\n\\nFilter\\n\\nProducts\\n\\nSEARCH PRODUCTS\\n\\nNo matches found.\\n\\n[Expand all](https://azure.microsoft.com/en-us/updates/#) [Collapse all](https://azure.microsoft.com/en-us/updates/#)\\n\\n0 Selected\\n\\nClear All\\n\\n- AI + machine learning [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAI Anomaly Detector\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Bot Service\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Content Safety\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Custom Vision\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Foundry\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Language\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Personalizer\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Search\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Services\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Translator\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Video Indexer\\n\\n\\n\\n\\n\\n\\n\\nAzure Databricks\\n\\n\\n\\n\\n\\n\\n\\nAzure Machine Learning\\n\\n\\n\\n\\n\\n\\n\\nAzure Open Datasets\\n\\n\\n\\n\\n\\n\\n\\nAzure OpenAI Service\\n\\n\\n\\n\\n\\n\\n\\nData Science Virtual Machines\\n\\n\\n\\n\\n\\n\\n\\nHealth Bot\\n\\n\\n\\n\\n\\n\\n\\nLanguage Understanding (LUIS)\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Copilot for Security\\n\\n\\n\\n\\n\\n\\n\\nQnA Maker\\n\\n\\n\\n\\n\\n\\n\\nSpeaker recognition\\n\\n\\n\\n\\n\\n\\n\\nSpeech to text\\n\\n\\n\\n\\n\\n\\n\\nSpeech translation\\n\\n\\n\\n\\n\\n\\n\\nText to speech\\n\\n- Analytics [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Analysis Services\\n\\n\\n\\n\\n\\n\\n\\nAzure Chaos Studio\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Explorer\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Factory\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Lake Storage\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Share\\n\\n\\n\\n\\n\\n\\n\\nAzure Databricks\\n\\n\\n\\n\\n\\n\\n\\nAzure HDInsight on Azure Kubernetes Service (AKS)\\n\\n\\n\\n\\n\\n\\n\\nAzure Stream Analytics\\n\\n\\n\\n\\n\\n\\n\\nAzure Synapse Analytics\\n\\n\\n\\n\\n\\n\\n\\nData Catalog\\n\\n\\n\\n\\n\\n\\n\\nData Lake Analytics\\n\\n\\n\\n\\n\\n\\n\\nEvent Hubs\\n\\n\\n\\n\\n\\n\\n\\nHDInsight\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Fabric\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Purview\\n\\n\\n\\n\\n\\n\\n\\nPower BI Embedded\\n\\n- Compute [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nApp Service\\n\\n\\n\\n\\n\\n\\n\\nAzure Compute Fleet\\n\\n\\n\\n\\n\\n\\n\\nAzure Container Instances\\n\\n\\n\\n\\n\\n\\n\\nAzure CycleCloud\\n\\n\\n\\n\\n\\n\\n\\nAzure Dedicated Host\\n\\n\\n\\n\\n\\n\\n\\nAzure Functions\\n\\n\\n\\n\\n\\n\\n\\nAzure Kubernetes Service (AKS)\\n\\n\\n\\n\\n\\n\\n\\nAzure Modeling and Simulation Workbench\\n\\n\\n\\n\\n\\n\\n\\nAzure Quantum\\n\\n\\n\\n\\n\\n\\n\\nAzure Service Fabric\\n\\n\\n\\n\\n\\n\\n\\nAzure Spot Virtual Machines\\n\\n\\n\\n\\n\\n\\n\\nAzure Spring Apps\\n\\n\\n\\n\\n\\n\\n\\nAzure Virtual Desktop\\n\\n\\n\\n\\n\\n\\n\\nAzure VM Image Builder\\n\\n\\n\\n\\n\\n\\n\\nAzure VMware Solution\\n\\n\\n\\n\\n\\n\\n\\nBatch\\n\\n\\n\\n\\n\\n\\n\\nCloud Services\\n\\n\\n\\n\\n\\n\\n\\nLinux Virtual Machines\\n\\n\\n\\n\\n\\n\\n\\nSQL Server on Azure Virtual Machines\\n\\n\\n\\n\\n\\n\\n\\nStatic Web Apps\\n\\n\\n\\n\\n\\n\\n\\nVirtual Machine Scale Sets\\n\\n\\n\\n\\n\\n\\n\\nVirtual Machines\\n\\n\\n\\n\\n\\n\\n\\nWindows Virtual Machines\\n\\n- Containers [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nApp Configuration\\n\\n\\n\\n\\n\\n\\n\\nAzure Container Apps\\n\\n\\n\\n\\n\\n\\n\\nAzure Container Instances\\n\\n\\n\\n\\n\\n\\n\\nAzure Container Registry\\n\\n\\n\\n\\n\\n\\n\\nAzure Container Storage\\n\\n\\n\\n\\n\\n\\n\\nAzure Functions\\n\\n\\n\\n\\n\\n\\n\\nAzure Kubernetes Fleet Manager\\n\\n\\n\\n\\n\\n\\n\\nAzure Kubernetes Service (AKS)\\n\\n\\n\\n\\n\\n\\n\\nAzure Red Hat OpenShift\\n\\n\\n\\n\\n\\n\\n\\nAzure Service Fabric\\n\\n\\n\\n\\n\\n\\n\\nWeb App for Containers\\n\\n\\n\\n\\n\\n\\n\\nWeb Apps for Containers\\n\\n- Databases [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Cache for Redis\\n\\n\\n\\n\\n\\n\\n\\nAzure confidential ledger\\n\\n\\n\\n\\n\\n\\n\\nAzure Cosmos DB\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Factory\\n\\n\\n\\n\\n\\n\\n\\nAzure Database for MariaDB\\n\\n\\n\\n\\n\\n\\n\\nAzure Database for MySQL\\n\\n\\n\\n\\n\\n\\n\\nAzure Database for PostgreSQL\\n\\n\\n\\n\\n\\n\\n\\nAzure Database Migration Service\\n\\n\\n\\n\\n\\n\\n\\nAzure Managed Instance for Apache Cassandra\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL Database\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL Edge\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL Managed Instance\\n\\n\\n\\n\\n\\n\\n\\nSQL Server on Azure Virtual Machines\\n\\n- Developer tools [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nApp Configuration\\n\\n\\n\\n\\n\\n\\n\\nAzure Deployment Environments\\n\\n\\n\\n\\n\\n\\n\\nAzure DevOps\\n\\n\\n\\n\\n\\n\\n\\nAzure DevTest Labs\\n\\n\\n\\n\\n\\n\\n\\nAzure Load Testing\\n\\n\\n\\n\\n\\n\\n\\nAzure Pipelines\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Dev Box\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Playwright Testing\\n\\n\\n\\n\\n\\n\\n\\nSDKs\\n\\n\\n\\n\\n\\n\\n\\nVisual Studio\\n\\n\\n\\n\\n\\n\\n\\nVisual Studio Code\\n\\n- DevOps [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Boards\\n\\n\\n\\n\\n\\n\\n\\nAzure Deployment Environments\\n\\n\\n\\n\\n\\n\\n\\nAzure DevOps\\n\\n\\n\\n\\n\\n\\n\\nAzure DevTest Labs\\n\\n\\n\\n\\n\\n\\n\\nAzure Load Testing\\n\\n\\n\\n\\n\\n\\n\\nAzure Managed Grafana\\n\\n\\n\\n\\n\\n\\n\\nAzure Monitor\\n\\n\\n\\n\\n\\n\\n\\nAzure Pipelines\\n\\n\\n\\n\\n\\n\\n\\nAzure Repos\\n\\n\\n\\n\\n\\n\\n\\nDevOps tool integrations\\n\\n\\n\\n\\n\\n\\n\\nGitHub Advanced Security for Azure DevOps\\n\\n\\n\\n\\n\\n\\n\\nGitHub Enterprise\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Dev Box\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Playwright Testing\\n\\n- Hybrid + multicloud [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Arc\\n\\n\\n\\n\\n\\n\\n\\nAzure Database for PostgreSQL\\n\\n\\n\\n\\n\\n\\n\\nAzure DevOps\\n\\n\\n\\n\\n\\n\\n\\nAzure ExpressRoute\\n\\n\\n\\n\\n\\n\\n\\nAzure IoT Edge\\n\\n\\n\\n\\n\\n\\n\\nAzure Modular Datacenter\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL Database\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL Edge\\n\\n\\n\\n\\n\\n\\n\\nAzure Stack\\n\\n\\n\\n\\n\\n\\n\\nAzure Stack Edge\\n\\n\\n\\n\\n\\n\\n\\nAzure Stack HCI\\n\\n\\n\\n\\n\\n\\n\\nAzure Stack Hub\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Defender for Cloud\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Sentinel\\n\\n- Identity [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Active Directory B2C\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Entra Domain Services\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Entra ID (formerly Azure AD)\\n\\n- Integration [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\nAzure Health Data Services\\n\\n\\n\\n\\n\\n\\n\\nAzure Web PubSub\\n\\n\\n\\n\\n\\n\\n\\nEvent Grid\\n\\n\\n\\n\\n\\n\\n\\nLogic Apps\\n\\n\\n\\n\\n\\n\\n\\nService Bus\\n\\n- Internet of Things [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\nAzure Cosmos DB\\n\\n\\n\\n\\n\\n\\n\\nAzure Digital Twins\\n\\n\\n\\n\\n\\n\\n\\nAzure Functions\\n\\n\\n\\n\\n\\n\\n\\nAzure IoT Central\\n\\n\\n\\n\\n\\n\\n\\nAzure IoT Edge\\n\\n\\n\\n\\n\\n\\n\\nAzure IoT Hub\\n\\n\\n\\n\\n\\n\\n\\nAzure Machine Learning\\n\\n\\n\\n\\n\\n\\n\\nAzure Maps\\n\\n\\n\\n\\n\\n\\n\\nAzure RTOS\\n\\n\\n\\n\\n\\n\\n\\nAzure Sphere\\n\\n\\n\\n\\n\\n\\n\\nAzure SQL Edge\\n\\n\\n\\n\\n\\n\\n\\nAzure Stream Analytics\\n\\n\\n\\n\\n\\n\\n\\nAzure Time Series Insights\\n\\n\\n\\n\\n\\n\\n\\nEvent Grid\\n\\n\\n\\n\\n\\n\\n\\nLogic Apps\\n\\n\\n\\n\\n\\n\\n\\nNotification Hubs\\n\\n\\n\\n\\n\\n\\n\\nWindows 10 IoT Core Services\\n\\n\\n\\n\\n\\n\\n\\nWindows for IoT\\n\\n- Management and governance [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAutomation\\n\\n\\n\\n\\n\\n\\n\\nAzure Advisor\\n\\n\\n\\n\\n\\n\\n\\nAzure Automanage\\n\\n\\n\\n\\n\\n\\n\\nAzure Backup\\n\\n\\n\\n\\n\\n\\n\\nAzure Blueprints\\n\\n\\n\\n\\n\\n\\n\\nAzure Chaos Studio\\n\\n\\n\\n\\n\\n\\n\\nAzure Lighthouse\\n\\n\\n\\n\\n\\n\\n\\nAzure Managed Applications\\n\\n\\n\\n\\n\\n\\n\\nAzure Managed Grafana\\n\\n\\n\\n\\n\\n\\n\\nAzure Migrate\\n\\n\\n\\n\\n\\n\\n\\nAzure mobile app\\n\\n\\n\\n\\n\\n\\n\\nAzure Monitor\\n\\n\\n\\n\\n\\n\\n\\nAzure Policy\\n\\n\\n\\n\\n\\n\\n\\nAzure Resource Manager\\n\\n\\n\\n\\n\\n\\n\\nAzure Resource Manager templates\\n\\n\\n\\n\\n\\n\\n\\nAzure Resource Mover\\n\\n\\n\\n\\n\\n\\n\\nAzure Service Health\\n\\n\\n\\n\\n\\n\\n\\nAzure Site Recovery\\n\\n\\n\\n\\n\\n\\n\\nCloud Shell\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Azure portal\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Copilot for Azure\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Cost Management\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Purview\\n\\n\\n\\n\\n\\n\\n\\nNetwork Watcher\\n\\n\\n\\n\\n\\n\\n\\nTraffic Manager\\n\\n\\n\\n\\n\\n\\n\\nUpdate management center\\n\\n- Media [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Media Player\\n\\n\\n\\n\\n\\n\\n\\nContent Delivery Network\\n\\n\\n\\n\\n\\n\\n\\nLive and On-Demand Streaming\\n\\n\\n\\n\\n\\n\\n\\nMedia Services\\n\\n- Migration [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Box\\n\\n\\n\\n\\n\\n\\n\\nAzure Database Migration Service\\n\\n\\n\\n\\n\\n\\n\\nAzure Migrate\\n\\n\\n\\n\\n\\n\\n\\nAzure Site Recovery\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Cost Management\\n\\n- Mixed reality [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Digital Twins\\n\\n\\n\\n\\n\\n\\n\\nRemote Rendering\\n\\n- Mobile [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\nApp Center\\n\\n\\n\\n\\n\\n\\n\\nApp Configuration\\n\\n\\n\\n\\n\\n\\n\\nApp Service\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Search\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Services\\n\\n\\n\\n\\n\\n\\n\\nAzure Communication Services\\n\\n\\n\\n\\n\\n\\n\\nAzure Maps\\n\\n\\n\\n\\n\\n\\n\\nNotification Hubs\\n\\n- Networking [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nApplication Gateway\\n\\n\\n\\n\\n\\n\\n\\nAzure Bastion\\n\\n\\n\\n\\n\\n\\n\\nAzure DDoS Protection\\n\\n\\n\\n\\n\\n\\n\\nAzure DNS\\n\\n\\n\\n\\n\\n\\n\\nAzure ExpressRoute\\n\\n\\n\\n\\n\\n\\n\\nAzure Firewall\\n\\n\\n\\n\\n\\n\\n\\nAzure Firewall Manager\\n\\n\\n\\n\\n\\n\\n\\nAzure Front Door\\n\\n\\n\\n\\n\\n\\n\\nAzure NAT Gateway\\n\\n\\n\\n\\n\\n\\n\\nAzure Private Link\\n\\n\\n\\n\\n\\n\\n\\nAzure Route Server\\n\\n\\n\\n\\n\\n\\n\\nAzure Virtual Network Manager\\n\\n\\n\\n\\n\\n\\n\\nContent Delivery Network\\n\\n\\n\\n\\n\\n\\n\\nLoad Balancer\\n\\n\\n\\n\\n\\n\\n\\nNetwork Watcher\\n\\n\\n\\n\\n\\n\\n\\nTraffic Manager\\n\\n\\n\\n\\n\\n\\n\\nVirtual Network\\n\\n\\n\\n\\n\\n\\n\\nVirtual WAN\\n\\n\\n\\n\\n\\n\\n\\nVPN Gateway\\n\\n\\n\\n\\n\\n\\n\\nWeb Application Firewall\\n\\n- Security [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nApp Configuration\\n\\n\\n\\n\\n\\n\\n\\nApplication Gateway\\n\\n\\n\\n\\n\\n\\n\\nAzure Bastion\\n\\n\\n\\n\\n\\n\\n\\nAzure confidential ledger\\n\\n\\n\\n\\n\\n\\n\\nAzure DDoS Protection\\n\\n\\n\\n\\n\\n\\n\\nAzure Dedicated HSM\\n\\n\\n\\n\\n\\n\\n\\nAzure Firewall\\n\\n\\n\\n\\n\\n\\n\\nAzure Firewall Manager\\n\\n\\n\\n\\n\\n\\n\\nAzure Front Door\\n\\n\\n\\n\\n\\n\\n\\nKey Vault\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Copilot for Security\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Defender for Cloud\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Entra Domain Services\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Sentinel\\n\\n\\n\\n\\n\\n\\n\\nVPN Gateway\\n\\n\\n\\n\\n\\n\\n\\nWeb Application Firewall\\n\\n- Storage [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nArchive Storage\\n\\n\\n\\n\\n\\n\\n\\nAvere vFXT for Azure\\n\\n\\n\\n\\n\\n\\n\\nAzure Backup\\n\\n\\n\\n\\n\\n\\n\\nAzure Blob Storage\\n\\n\\n\\n\\n\\n\\n\\nAzure confidential ledger\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Box\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Lake Storage\\n\\n\\n\\n\\n\\n\\n\\nAzure Data Share\\n\\n\\n\\n\\n\\n\\n\\nAzure Disk Storage\\n\\n\\n\\n\\n\\n\\n\\nAzure Elastic SAN\\n\\n\\n\\n\\n\\n\\n\\nAzure Files\\n\\n\\n\\n\\n\\n\\n\\nAzure HPC Cache\\n\\n\\n\\n\\n\\n\\n\\nAzure Managed Lustre\\n\\n\\n\\n\\n\\n\\n\\nAzure NetApp Files\\n\\n\\n\\n\\n\\n\\n\\nAzure Storage Actions\\n\\n\\n\\n\\n\\n\\n\\nQueue Storage\\n\\n\\n\\n\\n\\n\\n\\nStorage Accounts\\n\\n\\n\\n\\n\\n\\n\\nStorage Explorer\\n\\n- Virtual desktop infrastructure [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAzure Lab Services\\n\\n\\n\\n\\n\\n\\n\\nAzure Virtual Desktop\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Dev Box\\n\\n- Web [open Microsoft home page](javascript:void(0);)\\n\\n\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\nApp Configuration\\n\\n\\n\\n\\n\\n\\n\\nApp Service\\n\\n\\n\\n\\n\\n\\n\\nAzure AI Search\\n\\n\\n\\n\\n\\n\\n\\nAzure Communication Services\\n\\n\\n\\n\\n\\n\\n\\nAzure Maps\\n\\n\\n\\n\\n\\n\\n\\nAzure SignalR Service\\n\\n\\n\\n\\n\\n\\n\\nAzure Web PubSub\\n\\n\\n\\n\\n\\n\\n\\nContent Delivery Network\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Playwright Testing\\n\\n\\n\\n\\n\\n\\n\\nNotification Hubs\\n\\n\\n\\n\\n\\n\\n\\nStatic Web Apps\\n\\n\\n\\n\\n\\n\\n\\nWeb App for Containers\\n\\n\\nLoading...\\n\\nStatus\\n\\n0 Selected\\n\\nAll (3)\\n\\nIn development\\n\\nIn preview\\n\\nLaunched\\n\\nLoading...\\n\\nUpdate type\\n\\n0 Selected\\n\\nAll (17)\\n\\nCompliance\\n\\nFeatures\\n\\nGallery\\n\\nManagement\\n\\nMicrosoft Build\\n\\nMicrosoft Connect\\n\\nMicrosoft Ignite\\n\\nMicrosoft Inspire\\n\\nMicrosoftBuild\\n\\nOpen Source\\n\\nOperating System\\n\\nPricing & Offerings\\n\\nRegions & Datacenters\\n\\nRetirements\\n\\nSDK and Tools\\n\\nSecurity\\n\\nServices\\n\\nLoading...\\n\\nNew or updated\\n\\nLoading...\\n\\nSave filters\\n\\nClear filters\\n\\nApply filters\\n\\nIn development (124)\\n\\nLimited offering to select customers for non-production use and testing.\\n\\nIn preview (2743)\\n\\nAvailable to all Azure customers for non-production use and testing.\\n\\nLaunched (4371)\\n\\nFully released, production-ready product available to all Azure customers.\\n\\n- Clear All\\n\\n\\n\"test2\"\\n\\n\"test3\"\\n\\nSort by\\n\\nRecently Modified: Newest to oldest\\n\\n- RECENTLY MODIFIED\\n\\n- Newest to oldest\\n\\n- Oldest to newest\\n\\n\\nLoading...\\n\\n- ## Public Preview: Live Resize for Premium SSD v2 and Ultra NVMe Disks\\n\\n\\n\\n\\n\\nAzure Disk Storage\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAnnouncing the public preview of Live Resize for Premium SSD v2 (Pv2)\\nand Ultra [NVMe Disks](https://learn.microsoft.com/en-us/azure/virtual-machines/nvme-overview). This feature allows you to dynamically expand\\nthe storage capacity of your disks without any disruption to your applications.\\nTo optimize costs, you can start with smaller disks and gradually increase\\ntheir storage capacity as needed, all without experiencing any downtime.\\n\\n\\n\\n\\n\\n\\n\\nGet\\nstarted with Live Resizing your Ultra and/or Pv2 disks [here.](https://aka.ms/LiveResizePublicLink)\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n495106\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nStorage\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/29/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/29/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/495106)\\n\\n- ## Generally Available: Destination Network Address Translation (DNAT) on Azure Firewall Private IP address\\n\\n\\n\\n\\n\\nAzure Firewall\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure Firewall enhances the DNAT rule configuration to\\nsupport port translations on its Private IP address.\\n\\n\\n\\nDNAT on Azure Firewall Private IP address helps connect\\noverlapped IP networks, which is a common scenario for enterprises when\\nonboarding new partners to their network or merging with new\\nacquisitions.\\n\\n\\n\\nThis capability is also relevant for hybrid scenarios,\\nconnecting on-premises datacenters to Azure, where DNAT bridges the gap,\\nenabling communication between private resources over non-routable IP\\naddresses.\\n\\n\\n\\n[Learn\\\\\\\\\\nmore](https://techcommunity.microsoft.com/blog/azurenetworksecurityblog/private-ip-dnat-support-preview-and-scenarios-with-azure-firewall/4230073).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n493296\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nNetworking, Security\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/29/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/29/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/493296)\\n\\n- ## Public Preview: Azure Backup for Elastic SAN\\n\\n\\n\\n\\n\\nAzure Backup\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\nPRIVATE PREVIEW\\n\\n\\n\\nJuly 2024\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\nPRIVATE PREVIEW\\n\\n\\n\\nJuly 2024\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure Backup now supports Elastic SAN, offering a fully\\nmanaged solution for backing up and restoring Elastic SAN volumes. This\\nintegration helps protect data against accidental deletions, ransomware\\nattacks, and application updates by exporting Elastic SAN volumes to\\nindependent Managed Disk Incremental Snapshots. These snapshots are stored in\\nlocally redundant storage and are independent of the Elastic SAN volume\\nlifecycle.\\n\\n\\n\\nThe solution supports up to 450 restore points with 24 hours\\nbackup frequency. It is currently available in select Azure regions and\\nsupports volumes up to 4 TiB. Long-term vaulted backups and hourly backups are\\nnot supported in this preview.\\n\\n\\n\\nThis public preview does not incur Azure Backup Protected\\nInstance fees. However, standard charges apply for Managed Disk Incremental\\nSnapshots.\\n\\n\\n\\nTo get started, visit the Azure Business Continuity Center\\nand configure protection for your Elastic SAN volumes.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n494438\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nManagement and governance, Storage\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nCompliance, Management, Features\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/29/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/29/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494438)\\n\\n- ## Public Preview: Using Server-sent events with Application Gateway\\n\\n\\n\\n\\n\\nApplication Gateway\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure Application Gateway supports use of Server-sent\\nevents in preview, enabling real-time data streaming from server to client. Server-sent\\nevents utilize server push technology on a persistent HTTP connection for\\nseamless updates to the clients.\\n\\n\\n\\nTo implement this, specific configurations\\nare required on both the application gateway resource and the backend\\napplication. [Explore\\\\\\\\\\nthese configurations](https://learn.microsoft.com/azure/application-gateway/use-server-sent-events) to know how you can use Server-sent events with\\nApplication Gateway.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n494787\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nNetworking, Security\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nManagement, Features\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/28/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494787)\\n\\n- ## Generally Available: App Service Hybrid Connection Manager\\n\\n\\n\\n\\n\\nApp Service\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAnnouncing\\xa0 App Service Hybrid Connection has been updated, and the latest version is now generally available.\\n\\nThis new version has an updated look and feel and provides the following advantages over the previous version:\\n\\n\\n\\n- Support for both Windows and Linux clients\\n- Enhanced logging and visibility into operating status\\n- Refreshed GUI and a new CLI experience for cross-platform compatibility\\n\\n[Learn more](https://learn.microsoft.com/azure/app-service/app-service-hybrid-connections?tabs=windows#hybrid-connection-manager)\\n\\nAzure ID\\n\\n494993\\n\\nProduct Categories(s)\\n\\nCompute, Mobile, Web\\n\\nUpdate Types(s)\\n\\nFeatures\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\\\|\\n\\nLast modified: 05/28/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494993)\\n\\n- ## Generally Available: Customer-managed keys for Azure NetApp Files volume encryption with Azure Key Vault Managed HSM\\n\\n\\n\\n\\n\\nAzure NetApp Files\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure NetApp Files volume encryption choices have expanded to offer support customer-managed keys for Azure NetApp Files volume encryption with Azure Key Vault Managed HSM.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nThis capability offers increased security from FIPS 140-2 Level 2 to FIPS 140-2 Level 3 for critical deployments. Various applications that leverage HSM security include payment processing, application-level encryption, authentication. Industry verticals that use HSMs include financial services, public sector, IT/Telco (secure communications), energy (securing critical infrastructure).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nThis feature is generally available in [these regions](https://learn.microsoft.com/en-us/azure/azure-netapp-files/configure-customer-managed-keys-hardware#supported-regions).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLearn more:\\n\\n\\n\\n\\n\\n\\n\\n- [What\\'s new in Azure NetApp Files](https://learn.microsoft.com/en-us/azure/azure-netapp-files/whats-new?msclkid=0da6b3e7d15111ecb7b535244a027cc8)\\n\\n- [Configure customer-managed keys with managed Hardware Security Module for Azure NetApp Files volume encryption](https://learn.microsoft.com/en-us/azure/azure-netapp-files/configure-customer-managed-keys-hardware)\\n\\n\\nAzure ID\\n\\n493909\\n\\nProduct Categories(s)\\n\\nStorage\\n\\nUpdate Types(s)\\n\\nFeatures, SDK and Tools, Services\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\\\|\\n\\nLast modified: 05/28/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/493909)\\n\\n- ## Private Preview: Application Awareness\\n\\n\\n\\n\\n\\nAzure Migrate\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nA key step in any cloud transformation plan is a current state analysis of the entire IT estate covering workloads and applications, and relationships/dependencies among them. More often, customers are looking to set their migration goals in terms of the applications they want to move to cloud, and not in terms of the individual servers, databases or webapps in silos.\\n\\nWe are excited to announce the public preview of application aware experiences in Azure Migrate. The capability includes identification of ideal migration strategy among Rehost and Replatform from Gartner\\'s 6Rs to allow customers to gain insights into the total cost of ownership, identify suitable IaaS and PaaS targets, and receive tailored migration guidance.\\n\\nAs part of delivering application awareness, we will also be delivering a refreshed user experience for Azure Migrate. This experience is designed to guide users through the journey and deliver an integrated seamless experience from Azure Migrate to other bespoke tools, depending upon their goals and personas.\\n\\n**Learn more:**\\n\\n- Read [blog](https://aka.ms/AzureMigrateBuild2025Blog) post\\n- Read [Review an application assessment](https://learn.microsoft.com/en-us/azure/migrate/review-application-assessment?view=migrate#overview)\\n\\n- Watch\\xa0[Azure Migrate guided application aware user experience](https://www.youtube.com/watch?v=aquRVLvau7c)\\n\\nAzure ID\\n\\n494101\\n\\nProduct Categories(s)\\n\\nManagement and governance, Migration\\n\\nUpdate Types(s)\\n\\nMicrosoft Build, Features\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\\\|\\n\\nLast modified: 05/28/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494101)\\n\\n- ## Public Preview: Microsoft Planetary Computer Pro\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nMicrosoft Planetary Computer Pro is a\\ncomprehensive geospatial data platform designed to accelerate the generation of\\ngeospatial insights and their integration into enterprise Data & AI\\nworkflows. It offers robust capabilities to ingest, manage, and disseminate\\ngeospatial datasets, thus enabling organizations to harness the power of\\nlocation-based data effectively. By streamlining geospatial processes, Microsoft Planetary Computer Pro\\nfacilitates improved decision-making and operational efficiency.\\xa0[Learn more](https://aka.ms/planetarycomputerpro).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n494165\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures, Microsoft Build\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/28/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494165)\\n\\n- ## Generally Available: Azure Migrate enhances support with Premium SSD v2 Disks\\n\\n\\n\\n\\n\\nAzure Disk Storage\\n\\n\\n\\nAzure Migrate\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n[Azure Migrate](https://learn.microsoft.com/en-us/azure/migrate/?view=migrate-classic)\\xa0now supports migration to [Premium SSD v2 (Pv2)](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-deploy-premium-v2?tabs=azure-cli)\\xa0disks, offering customers a seamless experience to migrate their on-premises\\xa0workloads to Azure and benefit from the greater flexibility and enhanced performance of Pv2 disks in Azure. Pv2 disks offer sub-millisecond disk latencies for demanding IO-intensive workloads at a low-cost. Customers can use that to improve the price-performance of a broad range of enterprise production workloads such as SQL Server, Oracle, MariaDB, SAP, Cassandra, Mongo DB, big data, analytics, gaming, on virtual machines, or stateful containers. Azure Migrate now recommends Pv2 as the target disk type for eligible data disks in regions where Pv2 is available and offers Pv2 as a selectable option\\xa0for migrating applicable data disks.\\n\\n[Learn more](https://review.learn.microsoft.com/en-us/azure/migrate/vmware/tutorial-migrate-vmware?branch=main&branchFallbackFrom=pr-en-us-294110#replicate-vms).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n495302\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nStorage, Management and governance, Migration\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures, Management\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/28/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/495302)\\n\\n- ## Public Preview: Azure Migrate expands support for migrations with Ultra SSD\\n\\n\\n\\n\\n\\nAzure Disk Storage\\n\\n\\n\\nAzure Migrate\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n[Azure Migrate](https://learn.microsoft.com/en-us/azure/migrate/?view=migrate-classic) now supports migration to [Ultra Disk](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-enable-ultra-ssd?tabs=azure-portal), enabling customers to seamlessly migrate their\\non-premises workloads to Azure while taking advantage of Ultra Disk’s\\ncutting-edge performance and scalability. Designed for I/O-intensive workloads,\\nsuch as SAP HANA, high-performance databases like SQL and Oracle, and\\nlatency-sensitive applications, Ultra Disk delivers up to 400,000 IOPS and\\n10,000 MBps with low sub-millisecond latency, dynamic scalability, and\\nenterprise-grade reliability. In regions where Ultra Disk is available, Azure\\nMigrate has been enhanced to offer Ultra Disk as a selectable option for\\nmigrating applicable data disks.\\n\\n\\n\\n[Learn more](https://review.learn.microsoft.com/en-us/azure/migrate/vmware/tutorial-migrate-vmware?branch=main&branchFallbackFrom=pr-en-us-294110#replicate-vms).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n495312\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nStorage, Management and governance, Migration\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures, Management\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/28/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/28/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/495312)\\n\\n- ## Generally Available: Azure Cosmos DB JavaScript SDK 4.0\\n\\n\\n\\n\\n\\nAzure Cosmos DB\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nThe Azure Cosmos DB JavaScript SDK 4.0 is now generally available.\\xa0This major update brings a range of enhancements to help you build more efficient and scalable applications,\\xa0whether you\\'re\\xa0working with large datasets, securing sensitive information, or building smart search functionalities.\\n\\n\\n\\n\\n\\n\\n\\nKey improvements include enhanced diagnostic logging for better performance tracking, an improved bulk API for faster data operations, and a more flexible query design for efficient scaling. The SDK\\xa0also introduces client-side encryption to support data security, as well as AI-driven features, including vector\\xa0search and full-text search to enable advanced search capabilities.\\n\\n\\n\\n\\n\\n\\n\\nThese updates make the Azure Cosmos DB JavaScript SDK 4.0 a powerful tool for building high-performance applications.\\n\\n\\n\\n\\n\\n\\n\\n[Learn more](https://aka.ms/azure-cosmosdb-javascript-4.0).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n442638\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nDatabases, Internet of Things\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures, Microsoft Build\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/27/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/27/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/442638)\\n\\n- ## Public Preview: Azure Cosmos DB for MongoDB (vCore) trigger and bindings for Azure Functions\\n\\n\\n\\n\\n\\nAzure Cosmos DB\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIntegration between Azure Functions and Azure Cosmos DB for MongoDB (vCore) is now available in public preview.\\xa0You can now respond to changes in your collections using Azure Functions\\xa0triggers and bindings, enabling you to easily build real-time, event-driven applications. The public preview supports Azure Functions\\xa0triggers in C# for Azure Cosmos DB\\xa0for MongoDB (vCore).\\n\\n\\n\\n[Learn more](https://aka.ms/vcore-azure-functions).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n491267\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nDatabases, Internet of Things\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures, Microsoft Build\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/27/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/27/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/491267)\\n\\n- ## Public Preview: Microsoft DocumentDB Docker Image\\n\\n\\n\\n\\n\\nAzure Cosmos DB\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIntroducing the Microsoft DocumentDB\\xa0Docker Image,\\xa0a local development environment you can use to build and test apps with DocumentDB.\\xa0This image\\xa0simulates the DocumentDB\\xa0on your local machine, providing a fast, cost-effective, and isolated\\xa0environment for development. You can also integrate the\\xa0DocumentDB\\xa0Docker Image into continuous integration and continuous delivery (CI/CD)\\xa0pipelines to support automated testing and validation of app behavior against a local instance, ensuring smooth deployments\\xa0and consistent performance across environments.\\n\\n[Learn more](https://aka.ms/documentdb-docker).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n491272\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nDatabases, Internet of Things\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures, Microsoft Build\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/27/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/27/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/491272)\\n\\n- ## Public Preview: Azure Front Door now supports origin authentication via Managed Identities\\n\\n\\n\\n\\n\\nAzure Front Door\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFront Door Standard and Premium now\\nsupports sending authenticated requests to its origins using Managed Identities.\\nThis feature allows customers to secure their origins by allowing only approved\\nFront Door profiles to access the origins.\\n\\n\\n\\nUsing Managed Identities eliminates the\\nneed for customers to manually handle the credentials involved in the authentication\\nprocess, thereby reducing potential risks associated with credential leakage.\\n\\n\\n\\n**Learn more**:\\n\\n\\n\\n- About\\u202f[managed\\\\\\\\\\nidentities](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/overview).\\n- About\\xa0[how to enable\\\\\\\\\\nmanaged identities on Azure Front Door Standard and Premium.](https://learn.microsoft.com/azure/frontdoor/origin-authentication-with-managed-identities)\\n\\nAzure ID\\n\\n494352\\n\\nProduct Categories(s)\\n\\nNetworking, Security\\n\\nUpdate Types(s)\\n\\nFeatures, Services\\n\\nAdded to roadmap: 05/27/2025\\n\\n\\\\|\\n\\nLast modified: 05/27/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494352)\\n\\n- ## Generally Available: Container Apps and Functions as Private Link enabled origins for Front Door Premium\\n\\n\\n\\n\\n\\nAzure Front Door\\n\\n\\n\\nAzure Private Link\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nYou can now configure Azure Container Apps\\nand Azure Functions as Private Link enabled origins in your Front Door Premium\\nprofile. Private Link enabled origins in Front Door allow you to deliver\\ncontent to your end-users through public Front Door endpoints while ensuring\\nthat your origins remain inaccessible to the public internet.\\n\\n\\n\\n**Learn more:**\\n\\n\\n\\n- [Secure your Origin with Private Link in Azure Front Door\\\\\\\\\\nPremium.](https://learn.microsoft.com/azure/frontdoor/private-link)\\n\\n- How to [connect Azure Front Door Premium to an Azure Container App\\\\\\\\\\norigin with Private Link](https://learn.microsoft.com/azure/container-apps/how-to-integrate-with-azure-front-door?pivots=azure-portal).\\n- How to [connect\\\\\\\\\\nAzure Front Door Premium to an App Service (Web App or Functions) origin\\\\\\\\\\nwith Private Link.](https://learn.microsoft.com/en-us/azure/frontdoor/standard-premium/how-to-enable-private-link-web-app)\\n\\nAzure ID\\n\\n494357\\n\\nProduct Categories(s)\\n\\nNetworking, Security\\n\\nUpdate Types(s)\\n\\nFeatures, Services\\n\\nAdded to roadmap: 05/27/2025\\n\\n\\\\|\\n\\nLast modified: 05/27/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/494357)\\n\\n- ## Generally Available: Private subnet\\n\\n\\n\\n\\n\\nVirtual Network\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nWe are announcing the general availability of private subnet functionality in Azure.\\n\\n\\n\\nCurrently, when virtual machines are created\\nin a virtual network without any explicit outbound connectivity, they are\\nassigned a default outbound public IP address. These implicit IPs are subject\\nto change, not associated with a subscription, difficult to troubleshoot, and\\ndo not follow Azure\\'s model of \"secure by default\" which ensures\\ncustomers have strong security without additional steps needed.\\xa0 The\\nprivate subnet feature prevents this insecure implicit connectivity for any\\nnewly created subnets by setting the \"default outbound access\"\\nparameter to false. You can then pick your preferred method for explicit\\noutbound connectivity, such as a NAT Gateway or Public IP address.\\n\\n\\n\\nAdditionally, please note that after\\nSeptember 30th, 2025, new virtual networks will default to using private subnets,\\nmeaning that an explicit outbound method must be enabled in order to reach\\npublic endpoints on the Internet and within Microsoft.\\xa0 Older versions of\\nthe Azure API will not be affected, and there would also be no change to\\nexisting virtual networks. This means that there will be no change in the operation of existing or new virtual machines in these subnets.\\n\\n[Learn more](https://learn.microsoft.com/en-us/azure/virtual-network/ip-services/default-outbound-access).\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAzure ID\\n\\n\\n\\n492953\\n\\n\\n\\n\\n\\n\\n\\nProduct Categories(s)\\n\\n\\n\\nNetworking\\n\\n\\n\\n\\n\\n\\n\\nUpdate Types(s)\\n\\n\\n\\nFeatures\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdded to roadmap: 05/27/2025\\n\\n\\n\\n\\\\|\\n\\n\\n\\n\\nLast modified: 05/27/2025\\n\\n\\n\\n\\n\\nShare\\n\\n\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/492953)\\n\\n- ## Generally Available: Inbound Private Endpoint Support for Azure API Management Standard v2\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAnnouncing\\nthe general availability of the inbound private endpoint feature for the Azure\\nAPI Management Standard v2 tier. This update enables organizations to securely\\nexpose their API Management gateway exclusively over Azure Private Link,\\nensuring that API traffic remains\\xa0fully contained within the Microsoft\\nbackbone network.\\n\\n\\n\\nThis\\ncapability is critical for customers who need network-level security and\\ncompliance for API access particularly in regulated industries like finance,\\nhealthcare, and government. With inbound private endpoints, API Management\\nStandard v2 now supports:\\n\\n\\n\\n- End-to-end private\\nconnectivity\\n- Improved security posture\\n- Reduced attack surface\\n- Better control over data flow\\nand API exposure\\n\\n[Learn\\\\\\\\\\nmore](https://learn.microsoft.com/en-us/azure/api-management/v2-service-tiers-overview).\\n\\nAzure ID\\n\\n492607\\n\\nProduct Categories(s)\\n\\nIntegration, Internet of Things, Mobile, Web\\n\\nUpdate Types(s)\\n\\nMicrosoft Build, Features\\n\\nAdded to roadmap: 05/23/2025\\n\\n\\\\|\\n\\nLast modified: 05/23/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/492607)\\n\\n- ## Generally Available: Import from Azure AI Foundry to Azure API Management’s AI Gateway\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAnnouncing\\nthe general availability of importing model endpoints from Azure AI Foundry\\ndirectly into Azure API Management’s AI Gateway. This capability simplifies\\nonboarding of large language model (LLM) APIs by enabling seamless integration\\nthrough the Azure portal.\\n\\n\\n\\n\\n\\n\\n\\nKey\\nbenefits:\\n\\n\\n\\n- Rapid onboarding of LLM\\nendpoints from Azure AI Foundry\\n- Configure token limiting,\\ntoken tracking, semantic caching, and content safety\\n- Centralized API governance and\\nobservability for generative AI workloads\\n\\n[Learn more](https://aka.ms/apim/genai/ai-foundry-import).\\n\\nAzure ID\\n\\n491980\\n\\nProduct Categories(s)\\n\\nIntegration, Internet of Things, Mobile, Web\\n\\nUpdate Types(s)\\n\\nFeatures, Microsoft Build\\n\\nAdded to roadmap: 05/23/2025\\n\\n\\\\|\\n\\nLast modified: 05/23/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/491980)\\n\\n- ## Generally Available: Support for AWS Bedrock API in AI Gateway Capabilities in Azure API Management\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLAUNCHED\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGENERAL AVAILABILITY\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAnnouncing\\nexpanded support for AWS Bedrock model endpoints across all Generative AI\\npolicies in Azure API Management’s AI Gateway. This release enables you to\\napply advanced management and optimization features such as Token Limit Policy,\\nToken Metric Policy, and Semantic Caching Policy to AWS Bedrock models,\\nempowering you to seamlessly manage and optimize\\xa0your multi-cloud AI\\nworkloads.\\n\\n\\n\\n\\n\\n\\n\\nKey\\nbenefits:\\n\\n\\n\\n- Apply token limiting,\\ntracking, and logging to AWS Bedrock APIs for better control\\n- Enable semantic caching to\\nenhance performance and response times for Bedrock models\\n- Achieve unified observability\\nand governance across multi-cloud AI endpoints\\n\\n[Learn more](https://aka.ms/apim/genai/bedrock).\\n\\nAzure ID\\n\\n491985\\n\\nUpdate Types(s)\\n\\nMicrosoft Build, Features\\n\\nAdded to roadmap: 05/23/2025\\n\\n\\\\|\\n\\nLast modified: 05/23/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/491985)\\n\\n- ## Public Preview: Model Context Protocol support in Azure API Management and Azure API Center\\n\\n\\n\\n\\n\\nAPI Management\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIN PREVIEW\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPREVIEW\\n\\n\\n\\nMay 2025\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAnnouncing\\nthe public preview of Model Context Protocol (MCP) support in Azure API\\nManagement and Azure API Center. With this new capability, enterprises can\\ntransform their existing APIs into dynamic, agent-ready tools, while improving\\nsecurity and simplifying management.\\n\\n\\n\\n\\n\\n\\n\\nEnhanced\\nSecurity for MCP Servers:\\n\\n\\n\\n- Apply gateway policies to\\nprotect MCP servers by enforcing authentication, authorization, rate\\nlimiting, and other security measures.\\n\\nTransform\\nExisting APIs into MCP Servers:\\n\\n- Easily expose any APIM-managed\\nAPI as an MCP server, transforming your existing APIs into dynamic,\\nagent-ready tools with minimal effort.\\n\\nPrivate\\nMCP Registry for Enterprise Organizations:\\n\\n- Use Azure API Center as a\\nprivate remote MCP registry for your organization, giving you full control\\nover what services are exposed.\\n\\nStreamlined\\nMCP Discovery and Consumption:\\n\\n- Expose APIs as MCP servers in\\nAzure API Center (APIC) for a better developer experience and smoother API\\nconsumption.\\n\\nThese\\nfeatures make it easier to secure, manage, and consume APIs in a way that\\nsupports advanced use cases like AI integrations, all while improving security\\nand governance.\\n\\n[Learn more](https://aka.ms/apim-mcp-support).\\n\\nAzure ID\\n\\n491990\\n\\nProduct Categories(s)\\n\\nIntegration, Internet of Things, Mobile, Web\\n\\nUpdate Types(s)\\n\\nMicrosoft Build, Features\\n\\nAdded to roadmap: 05/23/2025\\n\\n\\\\|\\n\\nLast modified: 05/23/2025\\n\\nShare\\n\\n[![Accordion detail - Share RSS](https://azure.microsoft.com/content/dam/microsoft/final/en-us/microsoft-product-and-services/microsoft-365/roadmap/icon-rss-blue.svg)](https://www.microsoft.com/releasecommunications/api/v2/azure/rss/491990)\\n\\n- <\\n- 1\\n- 2\\n- 3\\n- ...\\n- 439\\n- >\\n\\n0 Updates found\\n\\nPlease update your selections\\n\\n## Additional Resources\\n\\n![](https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/icon-blog-darkblue?resMode=sharp2&op_usm=1.5,0.65,15,0&qlt=85)\\n\\n### Microsoft Azure Blog\\n\\nLearn about the latest Microsoft Security solutions.\\n\\n[Learn more](https://azure.microsoft.com/en-us/blog)\\n\\n![](https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/icon-security-darkblue?resMode=sharp2&op_usm=1.5,0.65,15,0&qlt=85)\\n\\n### Community support\\n\\nGet answers to your questions from Microsoft and Community experts.\\n\\n[Learn more](https://azure.microsoft.com/en-us/support/community/)\\n\\n![](https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/icon-documentation-darkblue?resMode=sharp2&op_usm=1.5,0.65,15,0&qlt=85)\\n\\n### Provide feedback\\n\\nTell us what you think of Azure and what you want to see in the future.\\n\\n[Learn more](https://go.microsoft.com/fwlink/?linkid=2295127&clcid=0x409)\\n\\n![](https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/icon-roadmap-darkblue?resMode=sharp2&op_usm=1.5,0.65,15,0&qlt=85)\\n\\n### Product availability by region\\n\\nAzure is available in more regions than any other cloud provider.\\n\\n[Learn more](https://azure.microsoft.com/en-us/explore/global-infrastructure/products-by-region)\\n\\n- [![](https://cdn-dynmedia-1.microsoft.com/is/content/microsoftcorp/acom_social_icon_azure) Get the Azure mobile app](https://azure.microsoft.com/en-us/get-started/azure-portal/mobile-app)\\n\\n- [![Twitter](https://cdn-dynmedia-1.microsoft.com/is/image/microsoftcorp/Twitter-2x-New-3?scl=1)](https://go.microsoft.com/fwlink/?linkid=2275431)\\n- [![Instagram](https://cdn-dynmedia-1.microsoft.com/is/image/microsoftcorp/Instagram%202x-3?scl=1)](https://go.microsoft.com/fwlink/?linkid=2275352)\\n- [![LinkedIn](https://cdn-dynmedia-1.microsoft.com/is/image/microsoftcorp/LinkedIn-29-1?scl=1)](https://go.microsoft.com/fwlink/?linkid=2268687)\\n- [![YouTube](https://cdn-dynmedia-1.microsoft.com/is/image/microsoftcorp/YouTube%202x-3?scl=1)](https://go.microsoft.com/fwlink/?linkid=2239941)\\n\\n[iframe](https://mscom.demdex.net/dest5.html?d_nsid=0#https%3A%2F%2Fazure.microsoft.com)\\n\\n![Microsoft Logo](https://www.microsoft.com/library/svy/ms-logo-short.png)![Close Survey Invite](https://www.microsoft.com/library/svy/icon_close.png)\\n\\nMicrosoft is conducting an online survey to understand your opinions about the Microsoft Azure website. If you choose to participate, the online survey will be presented to you when you leave the website.\\n\\nWould you like to participate?\\n\\n[Privacy Statement](https://go.microsoft.com/fwlink/?LinkId=521839 \"Privacy Statement\")\\n\\n[iframe](https://s.company-target.com/s/sync?exc=lr)![](https://id.rlcdn.com/464526.gif)\\n\\n![](https://bat.bing.com/action/0?ti=4000034&Ver=2&mid=8b2921e8-0d11-4cc5-9260-a0ad534e2f30&bo=2&sid=eb9ec9f03cb711f0a123cb3cfb18dafa&vid=eb9ee0d03cb711f08082df71aaf24cb3&vids=1&msclkid=N&pi=918639831&lg=en-US&sw=1280&sh=1024&sc=24&tl=Azure%20updates%20%7C%20Microsoft%20Azure&p=https%3A%2F%2Fazure.microsoft.com%2Fen-us%2Fupdates%2F&r=&lt=710&evt=pageLoad&sv=1&asc=G&cdb=AQAQ&rn=569396)', path=None, url=None, mimetype=None), image_resource=None, audio_resource=None, video_resource=None, text_template='{metadata_str}\\n\\n{content}')]\n"
          ]
        }
      ],
      "source": [
        "print(\"Documents\")\n",
        "print( len(documents))\n",
        "pprint.pprint(documents[0:10])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153,
          "referenced_widgets": [
            "bde8f75e71d84f90b6fbcc40a15ccd9e",
            "e80f53dc9a784c29a3bd17c1569bd56a",
            "4ff6592efda145cfb32c644f5b7bb7e1",
            "e8db992d46f3404cb6c94fed540e89a4",
            "e496685e94a34540908ad47423978d37",
            "44aa911946a6412faad36abfcf24a5d4",
            "38e3f8c2e78348a191c6de7ac2d02b62",
            "28986ca2cbc2439e97f1f02eecc9706e",
            "1658b21204a04c6da554ad380c3d26f1",
            "6d808760aa88409d9066f2a363275672",
            "13f5df276e014dbda0f0d8b8b2d80d94"
          ]
        },
        "id": "gMLXOYuZ15jE",
        "outputId": "601d53e9-6498-41a2-816a-70c17ccb1e65"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Directorio 'dataset' creado.\n",
            "The documents array has been saved to 'dataset/documents.csv'\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bde8f75e71d84f90b6fbcc40a15ccd9e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "documents.csv:   0%|          | 0.00/44.7k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "CommitInfo(commit_url='https://huggingface.co/datasets/vicpada/AzureResources/commit/6e04a0cabff20f42d9f440972f7fec91caa5cb45', commit_message='Upload documents.csv with huggingface_hub', commit_description='', oid='6e04a0cabff20f42d9f440972f7fec91caa5cb45', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/vicpada/AzureResources', endpoint='https://huggingface.co', repo_type='dataset', repo_id='vicpada/AzureResources'), pr_revision=None, pr_num=None)"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# prompt: Write documents array to csv file\n",
        "\n",
        "# Define the CSV filename for the documents\n",
        "filename = 'dataset/documents.csv'\n",
        "directory_name = \"dataset\"\n",
        "\n",
        "# Comprueba si el directorio no existe\n",
        "if not os.path.exists(directory_name):\n",
        "  # Crea el directorio\n",
        "  os.makedirs(directory_name)\n",
        "  print(f\"Directorio '{directory_name}' creado.\")\n",
        "else:\n",
        "  print(f\"El directorio '{directory_name}' ya existe.\")\n",
        "\n",
        "# Ensure documents is a list before proceeding\n",
        "if isinstance(documents, list):\n",
        "  # Open the file in write mode ('w')\n",
        "  # newline='' avoids issues with blank lines\n",
        "  with open(filename, 'w', newline='') as file:\n",
        "    writer = csv.writer(file)\n",
        "    # Write the header row\n",
        "    writer.writerow(['Text', 'Title', 'URL', 'Name', 'ContentType'])\n",
        "    # Write each document as a row\n",
        "    for doc in documents:\n",
        "      writer.writerow([\n",
        "          doc.text,\n",
        "          doc.metadata.get('title'),\n",
        "          doc.metadata.get('url'),\n",
        "          doc.metadata.get('name'),\n",
        "          doc.metadata.get('contentType')\n",
        "      ])\n",
        "\n",
        "  print(f\"The documents array has been saved to '{filename}'\")\n",
        "else:\n",
        "  print(\"documents is not a list. Cannot write to CSV.\")\n",
        "\n",
        "# Upload the new documents CSV to Hugging Face\n",
        "api.upload_file(\n",
        "    path_or_fileobj=filename,\n",
        "    path_in_repo='documents.csv',\n",
        "    repo_id=\"vicpada/AzureResources\",\n",
        "    repo_type=\"dataset\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KR4W9wpM1QPm",
        "outputId": "2adcaebb-1890-4ceb-c5dc-e3993e964541"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "El directorio 'dataset' ya existe.\n",
            "El array ha sido guardado en 'dataset/website-content.csv'\n"
          ]
        }
      ],
      "source": [
        "# prompt: write a dictionary to a csv file\n",
        "\n",
        "import csv\n",
        "\n",
        "directory_name = \"dataset\"\n",
        "\n",
        "# Comprueba si el directorio no existe\n",
        "if not os.path.exists(directory_name):\n",
        "  # Crea el directorio\n",
        "  os.makedirs(directory_name)\n",
        "  print(f\"Directorio '{directory_name}' creado.\")\n",
        "else:\n",
        "  print(f\"El directorio '{directory_name}' ya existe.\")\n",
        "\n",
        "# save to CSV\n",
        "filename = 'dataset/website-content.csv'\n",
        "\n",
        "# Ensure url_response is a dictionary before proceeding\n",
        "if isinstance(url_response, dict):\n",
        "  # Open the file in write mode ('w')\n",
        "  # newline='' avoids issues with blank lines\n",
        "  with open(filename, 'w', newline='') as file:\n",
        "    writer = csv.writer(file)\n",
        "    # Write the header row\n",
        "    writer.writerow(['URL', 'Scraped Data', 'CSV Data'])\n",
        "    # Write each row from the dictionary\n",
        "    for url, data in url_response.items():\n",
        "      writer.writerow([url, data.get('scraped_data'), data.get('csv_data')])\n",
        "\n",
        "  print(f\"El array ha sido guardado en '{filename}'\")\n",
        "else:\n",
        "  print(\"url_response is not a dictionary. Cannot write to CSV.\")\n",
        "\n",
        "  # Upload to HG\n",
        "\n",
        "api.upload_folder(\n",
        "    folder_path=\"dataset\",\n",
        "    repo_id=\"vicpada/AzureResources\",\n",
        "    repo_type=\"dataset\",\n",
        ")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "state":{},
        "13f5df276e014dbda0f0d8b8b2d80d94": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1658b21204a04c6da554ad380c3d26f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "28986ca2cbc2439e97f1f02eecc9706e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38e3f8c2e78348a191c6de7ac2d02b62": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "44aa911946a6412faad36abfcf24a5d4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ff6592efda145cfb32c644f5b7bb7e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28986ca2cbc2439e97f1f02eecc9706e",
            "max": 44740,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1658b21204a04c6da554ad380c3d26f1",
            "value": 44740
          }
        },
        "6d808760aa88409d9066f2a363275672": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bde8f75e71d84f90b6fbcc40a15ccd9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e80f53dc9a784c29a3bd17c1569bd56a",
              "IPY_MODEL_4ff6592efda145cfb32c644f5b7bb7e1",
              "IPY_MODEL_e8db992d46f3404cb6c94fed540e89a4"
            ],
            "layout": "IPY_MODEL_e496685e94a34540908ad47423978d37"
          }
        },
        "e496685e94a34540908ad47423978d37": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e80f53dc9a784c29a3bd17c1569bd56a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44aa911946a6412faad36abfcf24a5d4",
            "placeholder": "​",
            "style": "IPY_MODEL_38e3f8c2e78348a191c6de7ac2d02b62",
            "value": "documents.csv: 100%"
          }
        },
        "e8db992d46f3404cb6c94fed540e89a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d808760aa88409d9066f2a363275672",
            "placeholder": "​",
            "style": "IPY_MODEL_13f5df276e014dbda0f0d8b8b2d80d94",
            "value": " 44.7k/44.7k [00:00&lt;00:00, 184kB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}