diff --git a/.gitattributes b/.gitattributes index 9a73c94914761281449889e719f0c828c715c6ff..0b55a456faf80b9ea3d510c7260398a707c5d010 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,3 +34,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text scratch_agent/uploads/documents/LLM_based_QA_chatbot_builder.pdf filter=lfs diff=lfs merge=lfs -text +scratch_VLM/Documentation_Scratch/Scratch[[:space:]]3.0[[:space:]]Block[[:space:]]Descriptions_.docx filter=lfs diff=lfs merge=lfs -text +scratch_VLM/Documentation_Scratch/Scratch[[:space:]]3.0[[:space:]]Block[[:space:]]types[[:space:]]and[[:space:]]shape[[:space:]]details.docx filter=lfs diff=lfs merge=lfs -text +scratch_VLM/Documentation_Scratch/Scratch[[:space:]]3.0[[:space:]]BlocK_shape_explainations.docx filter=lfs diff=lfs merge=lfs -text +scratch_VLM/game_samples/Duck-Pocalypse.sb3 filter=lfs diff=lfs merge=lfs -text +scratch_VLM/game_samples/Pong[[:space:]]Game.sb3 filter=lfs diff=lfs merge=lfs -text +scratch_VLM/scratch_agent/uploads/documents/LLM_based_QA_chatbot_builder.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/scratch_VLM/Documentation_Scratch/Scratch 3.0 BlocK_shape_explainations.docx b/scratch_VLM/Documentation_Scratch/Scratch 3.0 BlocK_shape_explainations.docx new file mode 100644 index 0000000000000000000000000000000000000000..4ed7b8e92dc6a5b2b1d033786abefcb86b683dd8 --- /dev/null +++ b/scratch_VLM/Documentation_Scratch/Scratch 3.0 BlocK_shape_explainations.docx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f9cce92af68910293e634d66843348ef184085c23ad169fa03f3bb316ff0df2 +size 6260523 diff --git a/scratch_VLM/Documentation_Scratch/Scratch 3.0 Block Descriptions_.docx b/scratch_VLM/Documentation_Scratch/Scratch 3.0 Block Descriptions_.docx new file mode 100644 index 0000000000000000000000000000000000000000..7e04fe3e97dc1883ee184a74d3e06460398629cc --- /dev/null +++ b/scratch_VLM/Documentation_Scratch/Scratch 3.0 Block Descriptions_.docx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:620110acff1fa8734b14ce5adb03371e060b0345be3da3d27910c82cb18f6ea2 +size 6228386 diff --git a/scratch_VLM/Documentation_Scratch/Scratch 3.0 Block types and shape details.docx b/scratch_VLM/Documentation_Scratch/Scratch 3.0 Block types and shape details.docx new file mode 100644 index 0000000000000000000000000000000000000000..1345acda2d77e957aaf4a2310aae4242f9ba4f88 --- /dev/null +++ b/scratch_VLM/Documentation_Scratch/Scratch 3.0 Block types and shape details.docx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5bbdc3833a363e83a63fd360c5eee2123b80ec192fe4ef6bb925af4f02e9df0 +size 6260523 diff --git a/scratch_VLM/all_assets_function file/all_controls.sb3.zip b/scratch_VLM/all_assets_function file/all_controls.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..8628211d0535d35815bce0a7039891cc7f1260e1 --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_controls.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17f245eebb94d2f79f3feb2db80b89450a636abfbb4c1f137b8703e357fc6b25 +size 42536 diff --git a/scratch_VLM/all_assets_function file/all_events.sb3.zip b/scratch_VLM/all_assets_function file/all_events.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..7d9943b4715e99dfc8bb9188d826d52484535799 --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_events.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fa3a7ea4f394932b68ea5b46e7a96352085d0395e2ed1a566831cc7a16d85e4 +size 42416 diff --git a/scratch_VLM/all_assets_function file/all_list_variable.sb3.zip b/scratch_VLM/all_assets_function file/all_list_variable.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..a3882577713e03795062752504e02b4cc056b84e --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_list_variable.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07d2e5a8594a9e5f3d04bd2907c509a4893b52495b9e726941de6913142e56e4 +size 42503 diff --git a/scratch_VLM/all_assets_function file/all_looks.sb3.zip b/scratch_VLM/all_assets_function file/all_looks.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..c94b846313b8ec43ac0bc5bc242ca0a583cac71b --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_looks.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec2a6d01c610e1691a266b74ec343a6df86f61304d28631b4566f9caab3e690 +size 42918 diff --git a/scratch_VLM/all_assets_function file/all_motions.sb3.zip b/scratch_VLM/all_assets_function file/all_motions.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..4d5b280c0a2a93810a38bc135f27630fcd3fc7e8 --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_motions.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ce3143c179f4a1735d419077a778bc8b01805ad15cbab9aab074a818915454e +size 43002 diff --git a/scratch_VLM/all_assets_function file/all_operators.sb3.zip b/scratch_VLM/all_assets_function file/all_operators.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..ae2541391a5f77f20bf745807eb244d07cd57eae --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_operators.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41a0106bba8b5ede17f2e3041693415b96c3fcadd48ab4af5269555176104f5 +size 42712 diff --git a/scratch_VLM/all_assets_function file/all_sensing.sb3.zip b/scratch_VLM/all_assets_function file/all_sensing.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..a808ad0accfde93239c68c2ef490a6c23a9cb927 --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_sensing.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b937daca2ff2df0054e34a57f396ba31775211c15e9425e58dac4e5c4a93f5a +size 42938 diff --git a/scratch_VLM/all_assets_function file/all_sounds.sb3.zip b/scratch_VLM/all_assets_function file/all_sounds.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..1e32cbe9da16a6559d7c9fe6a2c04fa048c39f96 --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_sounds.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9984ee3d014ad326a7d0c34e7722841ffe0a4c309beea3052d65839310575e4c +size 42436 diff --git a/scratch_VLM/all_assets_function file/all_variables.sb3.zip b/scratch_VLM/all_assets_function file/all_variables.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..f2c0f741fc684c64cbe0ca04951bf55ecb1ef90a --- /dev/null +++ b/scratch_VLM/all_assets_function file/all_variables.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5a8e6f4c7247adb28780b0b253592f7f9f4e8150da13c2250ba346f81bc6027 +size 42182 diff --git a/scratch_VLM/cat_collecting_coin.sb3 b/scratch_VLM/cat_collecting_coin.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..467912007a23df244fc3334e162a05edfe2b45c0 Binary files /dev/null and b/scratch_VLM/cat_collecting_coin.sb3 differ diff --git a/scratch_VLM/cat_jumping.sb3 b/scratch_VLM/cat_jumping.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..ae4a6216f096e2a55c60b2e2c281714ce0f7c6c8 Binary files /dev/null and b/scratch_VLM/cat_jumping.sb3 differ diff --git a/scratch_VLM/game_samples.zip b/scratch_VLM/game_samples.zip new file mode 100644 index 0000000000000000000000000000000000000000..5c153e9c3d67a7fb2befee0dda1037de7239a2f5 --- /dev/null +++ b/scratch_VLM/game_samples.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8808da3f45e4e8410308f9d3821b8304c2788cb8e77d146866fca50f828fe0b3 +size 50013760 diff --git a/scratch_VLM/game_samples/Clicker Game.sb3 b/scratch_VLM/game_samples/Clicker Game.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..abfeb7d66f7df7efabae0ce23e55b5de30889925 Binary files /dev/null and b/scratch_VLM/game_samples/Clicker Game.sb3 differ diff --git a/scratch_VLM/game_samples/Clicker Game.sb3.zip b/scratch_VLM/game_samples/Clicker Game.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..a9965d6d249cbc7ef6ee6ba836d84f71b34344e5 --- /dev/null +++ b/scratch_VLM/game_samples/Clicker Game.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0060036767174635610012bc52acb568c9026e052f3f6135078609364e5ee561 +size 39237 diff --git a/scratch_VLM/game_samples/Don't Get Clicked! .sb3.zip b/scratch_VLM/game_samples/Don't Get Clicked! .sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..3ee47a54073dce5dee4c40feba19f30521a9d8db --- /dev/null +++ b/scratch_VLM/game_samples/Don't Get Clicked! .sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48cb01f0c8a6ba0969ad40a751433c61134d2131d39ddad01d204f8d20f4789 +size 54112 diff --git a/scratch_VLM/game_samples/Don't Get Clicked!.sb3 b/scratch_VLM/game_samples/Don't Get Clicked!.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..26935c9a72245f4f57dfda8c756bafea74a250fc Binary files /dev/null and b/scratch_VLM/game_samples/Don't Get Clicked!.sb3 differ diff --git a/scratch_VLM/game_samples/Duck-Pocalypse.sb3 b/scratch_VLM/game_samples/Duck-Pocalypse.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..d89fe87ddfd195746d86df75818ddf3dcb5e2f58 --- /dev/null +++ b/scratch_VLM/game_samples/Duck-Pocalypse.sb3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ce47a53047a3e1c3fc10f6217423c94363aea0de3aed222981b51848fbf1f7 +size 24191466 diff --git a/scratch_VLM/game_samples/Duck-Pocalypse.sb3.zip b/scratch_VLM/game_samples/Duck-Pocalypse.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..d89fe87ddfd195746d86df75818ddf3dcb5e2f58 --- /dev/null +++ b/scratch_VLM/game_samples/Duck-Pocalypse.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ce47a53047a3e1c3fc10f6217423c94363aea0de3aed222981b51848fbf1f7 +size 24191466 diff --git a/scratch_VLM/game_samples/Mouse Trail .sb3.zip b/scratch_VLM/game_samples/Mouse Trail .sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..411293e36f73a8ac633ab9dfd40b0a2089a6ae50 --- /dev/null +++ b/scratch_VLM/game_samples/Mouse Trail .sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e4a5ef2e10477c357b24e860f540118f6a3f68717846db2a09f883101e420b +size 43327 diff --git a/scratch_VLM/game_samples/Mouse Trail.sb3 b/scratch_VLM/game_samples/Mouse Trail.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..3f976a0804a743989fa2590392cd89a2dd16f8a3 Binary files /dev/null and b/scratch_VLM/game_samples/Mouse Trail.sb3 differ diff --git a/scratch_VLM/game_samples/Pong Game.sb3 b/scratch_VLM/game_samples/Pong Game.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..8be243d5f6e909825e9edaa463f0d8798bf536df --- /dev/null +++ b/scratch_VLM/game_samples/Pong Game.sb3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6615d559c51aa00fa08b653f383efccbfd7309770d79bd27dbef805481c0cb8 +size 573744 diff --git a/scratch_VLM/game_samples/Pong Game.sb3.zip b/scratch_VLM/game_samples/Pong Game.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..8be243d5f6e909825e9edaa463f0d8798bf536df --- /dev/null +++ b/scratch_VLM/game_samples/Pong Game.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6615d559c51aa00fa08b653f383efccbfd7309770d79bd27dbef805481c0cb8 +size 573744 diff --git a/scratch_VLM/game_samples/cat_jumping.sb3 b/scratch_VLM/game_samples/cat_jumping.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..ae4a6216f096e2a55c60b2e2c281714ce0f7c6c8 Binary files /dev/null and b/scratch_VLM/game_samples/cat_jumping.sb3 differ diff --git a/scratch_VLM/game_samples/cat_jumping.sb3.zip b/scratch_VLM/game_samples/cat_jumping.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..f2a5273a6828e8b735f3103726658cd079887fca --- /dev/null +++ b/scratch_VLM/game_samples/cat_jumping.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:678b5fe3121fcb48432d01114e2f37c35afa46620a89b58330887c7c1273854d +size 51462 diff --git a/scratch_VLM/game_samples/catch_cup_cake.sb3 b/scratch_VLM/game_samples/catch_cup_cake.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..872ae4e9926b0c780c72f8c4113e9e64fcb283c5 Binary files /dev/null and b/scratch_VLM/game_samples/catch_cup_cake.sb3 differ diff --git a/scratch_VLM/game_samples/catch_cup_cake.sb3.zip b/scratch_VLM/game_samples/catch_cup_cake.sb3.zip new file mode 100644 index 0000000000000000000000000000000000000000..3c114677393e912f46dd823ef250d75d8eeea9c0 --- /dev/null +++ b/scratch_VLM/game_samples/catch_cup_cake.sb3.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3806b61bd0c54a90933d26486738954fe3b9ef108b41ba0a957ab50585da1e5e +size 50039 diff --git a/scratch_VLM/jumping_Cat_action_new_unique.sb3 b/scratch_VLM/jumping_Cat_action_new_unique.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..ad71610ee0e2f7e419427018876bd544b42dcbc3 Binary files /dev/null and b/scratch_VLM/jumping_Cat_action_new_unique.sb3 differ diff --git a/scratch_VLM/jumping_Cat_action_node.sb3 b/scratch_VLM/jumping_Cat_action_node.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..47747284856eafac20cf10be1981cdf8c730dedc Binary files /dev/null and b/scratch_VLM/jumping_Cat_action_node.sb3 differ diff --git a/scratch_VLM/jumping_Cat_behave_node.sb3 b/scratch_VLM/jumping_Cat_behave_node.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..84ae66d79efc8e409f540a4a7ef7aa6413b1fa62 Binary files /dev/null and b/scratch_VLM/jumping_Cat_behave_node.sb3 differ diff --git a/scratch_VLM/jumping_Cat_edited.sb3/0fb9be3e8397c983338cb71dc84d0b25.svg b/scratch_VLM/jumping_Cat_edited.sb3/0fb9be3e8397c983338cb71dc84d0b25.svg new file mode 100644 index 0000000000000000000000000000000000000000..5ff997fd11132a3505e71ce46f5e14e34dbb4430 --- /dev/null +++ b/scratch_VLM/jumping_Cat_edited.sb3/0fb9be3e8397c983338cb71dc84d0b25.svg @@ -0,0 +1,42 @@ + + + + costume2.1 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/jumping_Cat_edited.sb3/1727f65b5f22d151685b8e5917456a60.wav b/scratch_VLM/jumping_Cat_edited.sb3/1727f65b5f22d151685b8e5917456a60.wav new file mode 100644 index 0000000000000000000000000000000000000000..f5be3a1606476bfbee8a4321f41de6a1a0ead520 Binary files /dev/null and b/scratch_VLM/jumping_Cat_edited.sb3/1727f65b5f22d151685b8e5917456a60.wav differ diff --git a/scratch_VLM/jumping_Cat_edited.sb3/5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg b/scratch_VLM/jumping_Cat_edited.sb3/5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg new file mode 100644 index 0000000000000000000000000000000000000000..3a3908a99eb18cdeaacf1222caec50828656871e --- /dev/null +++ b/scratch_VLM/jumping_Cat_edited.sb3/5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg @@ -0,0 +1,29 @@ + + + + Slice 1 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/jumping_Cat_edited.sb3/83a9787d4cb6f3b7632b4ddfebf74367.wav b/scratch_VLM/jumping_Cat_edited.sb3/83a9787d4cb6f3b7632b4ddfebf74367.wav new file mode 100644 index 0000000000000000000000000000000000000000..fc3b2724a9c7cfef378eeb65499d44236ad2add8 Binary files /dev/null and b/scratch_VLM/jumping_Cat_edited.sb3/83a9787d4cb6f3b7632b4ddfebf74367.wav differ diff --git a/scratch_VLM/jumping_Cat_edited.sb3/83c36d806dc92327b9e7049a565c6bff.wav b/scratch_VLM/jumping_Cat_edited.sb3/83c36d806dc92327b9e7049a565c6bff.wav new file mode 100644 index 0000000000000000000000000000000000000000..45742d5ef6f09d05b0f0788cb055ffe54abfd9ad Binary files /dev/null and b/scratch_VLM/jumping_Cat_edited.sb3/83c36d806dc92327b9e7049a565c6bff.wav differ diff --git a/scratch_VLM/jumping_Cat_edited.sb3/bcf454acf82e4504149f7ffe07081dbc.svg b/scratch_VLM/jumping_Cat_edited.sb3/bcf454acf82e4504149f7ffe07081dbc.svg new file mode 100644 index 0000000000000000000000000000000000000000..03df23e29ad059d88e559d48bf5e2717870455f3 --- /dev/null +++ b/scratch_VLM/jumping_Cat_edited.sb3/bcf454acf82e4504149f7ffe07081dbc.svg @@ -0,0 +1,42 @@ + + + + costume1.1 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/jumping_Cat_edited.sb3/cd21514d0531fdffb22204e0ec5ed84a.svg b/scratch_VLM/jumping_Cat_edited.sb3/cd21514d0531fdffb22204e0ec5ed84a.svg new file mode 100644 index 0000000000000000000000000000000000000000..15f73119b9c3271c6c411fc4233090e37f42ec37 --- /dev/null +++ b/scratch_VLM/jumping_Cat_edited.sb3/cd21514d0531fdffb22204e0ec5ed84a.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/scratch_VLM/jumping_Cat_edited.sb3/e7c147730f19d284bcd7b3f00af19bb6.svg b/scratch_VLM/jumping_Cat_edited.sb3/e7c147730f19d284bcd7b3f00af19bb6.svg new file mode 100644 index 0000000000000000000000000000000000000000..0326edc8bdd7da45b33bac907c15ae1c0d1026c1 --- /dev/null +++ b/scratch_VLM/jumping_Cat_edited.sb3/e7c147730f19d284bcd7b3f00af19bb6.svg @@ -0,0 +1,19 @@ + + + + blue sky + Created with Sketch. + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/jumping_Cat_edited.sb3/project.json b/scratch_VLM/jumping_Cat_edited.sb3/project.json new file mode 100644 index 0000000000000000000000000000000000000000..5bf8ab6e5bdca1ea1aa731d4809be9acb9fc960b --- /dev/null +++ b/scratch_VLM/jumping_Cat_edited.sb3/project.json @@ -0,0 +1,799 @@ +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "objName": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ + { + "name": "Blue Sky", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "e7c147730f19d284bcd7b3f00af19bb6", + "md5ext": "e7c147730f19d284bcd7b3f00af19bb6.svg", + "rotationCenterX": 240, + "rotationCenterY": 180 + } + ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Sprite1", + "objName": "Sprite1", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": { + "hatBlockID": { + "opcode": "event_whenflagclicked", + "next": "goToXYBlockID", + "parent": null, + "topLevel": true, + "shadow": false, + "x": 0, + "y": 0 + }, + "goToXYBlockID": { + "opcode": "motion_gotoxy", + "inputs": { + "X": [ + 1, + "shadowNum0ID_X" + ], + "Y": [ + 1, + "shadowNum0ID_Y" + ] + }, + "next": "setDirectionBlockID", + "parent": "hatBlockID", + "topLevel": false, + "shadow": false, + "x": 100, + "y": -50 + }, + + "shadowNum0ID_X": { + "opcode": "math_number", + "fields": { + "NUM": [ + "0", + null + ] + }, + "parent": "goToXYBlockID", + "shadow": true, + "topLevel": false + }, + "shadowNum0ID_Y": { + "opcode": "math_number", + "fields": { + "NUM": [ + "0", + null + ] + }, + "parent": "goToXYBlockID", + "shadow": true, + "topLevel": false + }, + "setDirectionBlockID": { + "opcode": "motion_setdirection", + "inputs": { + "DIRECTION": [ + 1, + "shadowNum90ID" + ] + }, + "next": "foreverBlockID", + "parent": "goToXYBlockID", + "topLevel": false, + "shadow": false, + "x": 100, + "y": -100 + }, + "shadowNum90ID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "90", + null + ] + }, + "parent": "setDirectionBlockID", + "shadow": true, + "topLevel": false + }, + "foreverBlockID": { + "opcode": "control_forever", + "inputs": { + "SUBSTACK": [ + 2, + "moveStepsBlockID" + ] + }, + "next": null, + "parent": "setDirectionBlockID", + "topLevel": false, + "shadow": false, + "x": 150, + "y": -150 + }, + "moveStepsBlockID": { + "opcode": "motion_movesteps", + "inputs": { + "STEPS": [ + 1, + "shadowNum5ID" + ] + }, + "next": "ifBlockID", + "parent": "foreverBlockID", + "topLevel": false, + "shadow": false, + "x": 200, + "y": -200 + }, + "shadowNum5ID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "5", + null + ] + }, + "parent": "moveStepsBlockID", + "shadow": true, + "topLevel": false + }, + "ifBlockID": { + "opcode": "control_if", + "inputs": { + "CONDITION": [ + 1, + "touchingEdgeBlockID" + ], + "SUBSTACK": [ + 2, + "bounceEdgeBlockID" + ] + }, + "next": null, + "parent": "foreverBlockID", + "topLevel": false, + "shadow": false, + "x": 250, + "y": -250 + }, + "touchingEdgeBlockID": { + "opcode": "sensing_touchingobject", + "inputs": { + "TOUCHINGOBJECTMENU": [ + 1, + "shadowEdgeID" + ] + }, + "parent": "ifBlockID", + "next": null, + "topLevel": false, + "shadow": false, + "x": 300, + "y": -300 + }, + "shadowEdgeID": { + "opcode": "sensing_touchingobjectmenu", + "fields": { + "TOUCHINGOBJECTMENU": [ + "edge", + null + ] + }, + "parent": "touchingEdgeBlockID", + "shadow": true, + "topLevel": false + }, + "bounceEdgeBlockID": { + "opcode": "motion_ifonedgebounce", + "next": null, + "parent": "ifBlockID", + "topLevel": false, + "shadow": false, + "x": 300, + "y": -350 + }, + "keyPressHatID": { + "opcode": "event_whenkeypressed", + "fields": { + "KEY": [ + "space", + null + ] + }, + "next": "changeY10ID", + "parent": null, + "topLevel": true, + "shadow": false, + "x": 0, + "y": 150 + }, + "changeY10ID": { + "opcode": "motion_changeyby", + "inputs": { + "DY": [ + 1, + "shadowNum10ID" + ] + }, + "next": "wait01ID", + "parent": "keyPressHatID", + "topLevel": false, + "shadow": false, + "x": 50, + "y": -50 + }, + "shadowNum10ID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "10", + null + ] + }, + "parent": "changeY10ID", + "shadow": true, + "topLevel": false + }, + "wait01ID": { + "opcode": "control_wait", + "inputs": { + "DURATION": [ + 1, + "shadowNum01ID" + ] + }, + "next": "changeY-10ID", + "parent": "changeY10ID", + "topLevel": false, + "shadow": false, + "x": 50, + "y": -100 + }, + "shadowNum01ID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "0.1", + null + ] + }, + "parent": "wait01ID", + "shadow": true, + "topLevel": false + }, + "changeY-10ID": { + "opcode": "motion_changeyby", + "inputs": { + "DY": [ + 1, + "shadowNum-10ID" + ] + }, + "next": "repeatUntilSpaceID", + "parent": "wait01ID", + "topLevel": false, + "shadow": false, + "x": 50, + "y": -150 + }, + "shadowNum-10ID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "-10", + null + ] + }, + "parent": "changeY-10ID", + "shadow": true, + "topLevel": false + }, + "repeatUntilSpaceID": { + "opcode": "control_repeat_until", + "inputs": { + "CONDITION": [ + 1, + "spaceKeyPressedID" + ] + }, + "next": null, + "parent": "changeY-10ID", + "topLevel": false, + "shadow": false, + "x": 100, + "y": -200 + }, + "spaceKeyPressedID": { + "opcode": "sensing_keypressed", + "inputs": { + "KEY_OPTION": [ + 1, + "shadowSpaceKeyID" + ] + }, + "parent": "repeatUntilSpaceID", + "shadow": false, + "topLevel": false + }, + "shadowSpaceKeyID": { + "opcode": "sensing_keypressed_keymenu", + "fields": { + "KEY_OPTION": [ + "space", + null + ] + }, + "parent": "spaceKeyPressedID", + "shadow": true, + "topLevel": false + }, + "move10StepsID": { + "opcode": "motion_movesteps", + "inputs": { + "STEPS": [ + 1, + "shadowNum10StepsID" + ] + }, + "parent": "repeatUntilSpaceID", + "next": "ifTouchingSoccerBallID", + "topLevel": false, + "shadow": false, + "x": 150, + "y": -250 + }, + "shadowNum10StepsID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "10", + null + ] + }, + "parent": "move10StepsID", + "shadow": true, + "topLevel": false + }, + "ifTouchingSoccerBallID": { + "opcode": "control_if", + "inputs": { + "CONDITION": [ + 1, + "touchingSoccerBallID" + ], + "SUBSTACK": [ + 2, + "loseLifeID" + ] + }, + "parent": "repeatUntilSpaceID", + "next": null, + "topLevel": false, + "shadow": false, + "x": 200, + "y": -300 + }, + "touchingSoccerBallID": { + "opcode": "sensing_touchingobject", + "inputs": { + "TOUCHINGOBJECTMENU": [ + 1, + "shadowSoccerBallID" + ] + }, + "parent": "ifTouchingSoccerBallID", + "shadow": false, + "topLevel": false + }, + "shadowSoccerBallID": { + "opcode": "sensing_touchingobjectmenu", + "fields": { + "TOUCHINGOBJECTMENU": [ + "soccer ball", + null + ] + }, + "parent": "touchingSoccerBallID", + "shadow": true, + "topLevel": false + }, + "loseLifeID": { + "opcode": "data_changevariableby", + "inputs": { + "VALUE": [ + 1, + "shadowNum-1ID" + ] + }, + "parent": "ifTouchingSoccerBallID", + "next": null, + "topLevel": false, + "shadow": false, + "x": 250, + "y": -350 + }, + "shadowNum-1ID": { + "opcode": "math_number", + "fields": { + "NUM": [ + "-1", + null + ] + }, + "parent": "loseLifeID", + "shadow": true, + "topLevel": false + } + }, + "comments": {}, + "currentCostume": 0, + "costumes": [ + { + "name": "Sprite1", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "bcf454acf82e4504149f7ffe07081dbc", + "md5ext": "bcf454acf82e4504149f7ffe07081dbc.svg", + "rotationCenterX": 0, + "rotationCenterY": 0 + } + ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, + "y": -200, + "size": 100, + "direction": 90, + "draggable": false, + "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "soccer ball", + "objName": "soccer ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": { + "hatBlockID": { + "opcode": "event_whenflagclicked", + "next": "goToRandomPosBlockID", + "parent": null, + "topLevel": true, + "shadow": false, + "x": 0, + "y": 0 + }, + "goToRandomPosBlockID": { + "opcode": "motion_gotoxy", + "inputs": { + "X": [ + 1, + "randomXBlockID" + ], + "Y": [ + 1, + "randomYBlockID" + ] + }, + "next": "foreverBlockID", + "parent": "hatBlockID", + "topLevel": false, + "shadow": false, + "x": 0, + "y": 50 + }, + "randomXBlockID": { + "opcode": "operators_random", + "inputs": { + "FROM": [ + 1, + "shadowRandomXFrom" + ], + "TO": [ + 1, + "shadowRandomXTo" + ] + }, + "parent": "goToRandomPosBlockID", + "shadow": true, + "topLevel": false + }, + + "shadowRandomXFrom": { + "opcode": "math_number", + "fields": { + "NUM": ["240", null] + }, + "parent": "randomXBlockID", + "shadow": true, + "topLevel": false + }, + "shadowRandomXTo": { + "opcode": "math_number", + "fields": { + "NUM": ["-240", null] + }, + "parent": "randomXBlockID", + "shadow": true, + "topLevel": false + }, + "randomYBlockID": { + "opcode": "operators_random", + "inputs": { + "FROM": [ + 1, + "shadowRandomYFrom" + ], + "TO": [ + 1, + "shadowRandomYTo" + ] + }, + "parent": "goToRandomPosBlockID", + "shadow": true, + "topLevel": false + }, + + "shadowRandomYFrom": { + "opcode": "math_number", + "fields": { + "NUM": ["-100", null] + }, + "parent": "randomYBlockID", + "shadow": true, + "topLevel": false + }, + "shadowRandomYTo": { + "opcode": "math_number", + "fields": { + "NUM": ["100", null] + }, + "parent": "randomYBlockID", + "shadow": true, + "topLevel": false + }, + "foreverBlockID": { + "opcode": "control_forever", + "inputs": { + "SUBSTACK": [ + 2, + "glideToRandomPosBlockID" + ] + }, + "next": null, + "parent": "goToRandomPosBlockID", + "topLevel": false, + "shadow": false, + "x": 0, + "y": 100 + }, + "glideToRandomPosBlockID": { + "opcode": "motion_glidesecstoxy", + "inputs": { + "SECS": [ + 1, + "shadowNum2SecsID" + ], + "X": [ + 1, + "randomXBlockID2" + ], + "Y": [ + 1, + "randomYBlockID2" + ] + }, + "next": "ifTouchingSprite1BlockID", + "parent": "foreverBlockID", + "topLevel": false, + "shadow": false, + "x": 0, + "y": 150 + }, + + "shadowNum2SecsID": { + "opcode": "math_number", + "fields": { + "NUM": ["2", null] + }, + "parent": "glideToRandomPosBlockID", + "shadow": true, + "topLevel": false + }, + "randomXBlockID2": { + "opcode": "operators_random", + "inputs": { + "FROM": [ + 1, + "shadowRandomX2From" + ], + "TO": [ + 1, + "shadowRandomX2To" + ] + }, + "parent": "glideToRandomPosBlockID", + "shadow": true, + "topLevel": false + }, + + "shadowRandomX2From": { + "opcode": "math_number", + "fields": { + "NUM": ["-240", null] + }, + "parent": "randomXBlockID2", + "shadow": true, + "topLevel": false + }, + "shadowRandomX2To": { + "opcode": "math_number", + "fields": { + "NUM": ["240", null] + }, + "parent": "randomXBlockID2", + "shadow": true, + "topLevel": false + }, + "randomYBlockID2": { + "opcode": "operators_random", + "inputs": { + "FROM": [ + 1, + "shadowRandomY2From" + ], + "TO": [ + 1, + "shadowRandomY2To" + ] + }, + "parent": "glideToRandomPosBlockID", + "shadow": true, + "topLevel": false + }, + + "shadowRandomY2From": { + "opcode": "math_number", + "fields": { + "NUM": ["-100", null] + }, + "parent": "randomYBlockID2", + "shadow": true, + "topLevel": false + }, + "shadowRandomY2To": { + "opcode": "math_number", + "fields": { + "NUM": ["100", null] + }, + "parent": "randomYBlockID2", + "shadow": true, + "topLevel": false + }, + "ifTouchingSprite1BlockID": { + "opcode": "control_if", + "inputs": { + "CONDITION": [ + 1, + "touchingSprite1BlockID" + ], + "SUBSTACK": [ + 2, + "sayGameOverBlockID" + ] + }, + "next": null, + "parent": "glideToRandomPosBlockID", + "topLevel": false, + "shadow": false, + "x": 0, + "y": 200 + }, + "touchingSprite1BlockID": { + "opcode": "sensing_touchingobject", + "inputs": { + "TOUCHINGOBJECTMENU": [ + 1, + "sprite1MenuBlockID" + ] + }, + "parent": "ifTouchingSprite1BlockID", + "shadow": false, + "topLevel": false + }, + "sprite1MenuBlockID": { + "opcode": "sensing_touchingobjectmenu", + "fields": { + "TOUCHINGOBJECTMENU": [ + "Sprite1", + null + ] + }, + "parent": "touchingSprite1BlockID", + "shadow": true, + "topLevel": false + }, + "sayGameOverBlockID": { + "opcode": "looks_say", + "inputs": { + "MESSAGE": [ + 1, + "shadowGameOverMessage" + ] + }, + "next": null, + "parent": "ifTouchingSprite1BlockID", + "topLevel": false, + "shadow": false, + "x": 0, + "y": 250 + }, + + "shadowGameOverMessage": { + "opcode": "text_reporter", + "fields": { + "TEXT": ["Game Over", null] + }, + "parent": "sayGameOverBlockID", + "shadow": true, + "topLevel": false + } + }, + "comments": {}, + "currentCostume": 0, + "costumes": [ + { + "name": "soccer ball", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "5d973d7a3a8be3f3bd6e1cd0f73c32b5", + "md5ext": "5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg", + "rotationCenterX": 0, + "rotationCenterY": 0 + } + ], + "sounds": [], + "volume": 100, + "layerOrder": 3, + "visible": true, + "x": 240, + "y": 0, + "size": 100, + "direction": 90, + "draggable": false, + "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36" + } +} \ No newline at end of file diff --git a/scratch_VLM/jumping_Cat_edited_initial_positions.sb3 b/scratch_VLM/jumping_Cat_edited_initial_positions.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..0307e786d45ea4b7a50cd98b77dc6d79b80b1cff Binary files /dev/null and b/scratch_VLM/jumping_Cat_edited_initial_positions.sb3 differ diff --git a/scratch_VLM/jumping_Cat_enhance_action.sb3 b/scratch_VLM/jumping_Cat_enhance_action.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..9d1c4c45d8a8ed73f5af22a4f455f90597a40d03 Binary files /dev/null and b/scratch_VLM/jumping_Cat_enhance_action.sb3 differ diff --git a/scratch_VLM/jumping_Cat_enhance_action_2.sb3 b/scratch_VLM/jumping_Cat_enhance_action_2.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..e02ec0ab018f159a571c9ed87a312893b69e9f31 Binary files /dev/null and b/scratch_VLM/jumping_Cat_enhance_action_2.sb3 differ diff --git a/scratch_VLM/jumping_Cat_enhance_action_3.sb3 b/scratch_VLM/jumping_Cat_enhance_action_3.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..22b8017166f8b3a3e765c185bd47f1df636c2189 Binary files /dev/null and b/scratch_VLM/jumping_Cat_enhance_action_3.sb3 differ diff --git a/scratch_VLM/jumping_Cat_enhance_action_4.sb3 b/scratch_VLM/jumping_Cat_enhance_action_4.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..03ab6c5d44808cdd8410490376bf5a841110dcc1 Binary files /dev/null and b/scratch_VLM/jumping_Cat_enhance_action_4.sb3 differ diff --git a/scratch_VLM/jumping_Cat_enhance_action_5.sb3 b/scratch_VLM/jumping_Cat_enhance_action_5.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..c28cf170e82b9db43e999d144c7da65378bca280 Binary files /dev/null and b/scratch_VLM/jumping_Cat_enhance_action_5.sb3 differ diff --git a/scratch_VLM/jumping_Cat_enhance_v7.sb3 b/scratch_VLM/jumping_Cat_enhance_v7.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..c9e8290f38d5e6dfa6deeb4f48d64e74e42586fe Binary files /dev/null and b/scratch_VLM/jumping_Cat_enhance_v7.sb3 differ diff --git a/scratch_VLM/jumping_Cat_refine_v2.sb3 b/scratch_VLM/jumping_Cat_refine_v2.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..ac2284a1a2de5dbac2e98f6d0343815c1cc92480 Binary files /dev/null and b/scratch_VLM/jumping_Cat_refine_v2.sb3 differ diff --git a/scratch_VLM/jumping_Cat_refined_v1.sb3 b/scratch_VLM/jumping_Cat_refined_v1.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..5d0f2466eae7e47d8838f3b9af18eebfa2b2c94e Binary files /dev/null and b/scratch_VLM/jumping_Cat_refined_v1.sb3 differ diff --git a/scratch_VLM/jumping_Cat_refined_v3.sb3 b/scratch_VLM/jumping_Cat_refined_v3.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..422ad9db06f67bea954e2f4d00373e776a3006fc Binary files /dev/null and b/scratch_VLM/jumping_Cat_refined_v3.sb3 differ diff --git a/scratch_VLM/jumping_Cat_refined_v4.sb3 b/scratch_VLM/jumping_Cat_refined_v4.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..42195abd7269a2add89603f58e67a6fdf8b3058e Binary files /dev/null and b/scratch_VLM/jumping_Cat_refined_v4.sb3 differ diff --git a/scratch_VLM/jumping_Cat_refined_v5.sb3 b/scratch_VLM/jumping_Cat_refined_v5.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..07be024b1eeaa6ed138745d38380d532c449d037 Binary files /dev/null and b/scratch_VLM/jumping_Cat_refined_v5.sb3 differ diff --git a/scratch_VLM/jumping_Cat_struc.sb3 b/scratch_VLM/jumping_Cat_struc.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..1b81cec22672ba33f4aad6207b294c800434f049 Binary files /dev/null and b/scratch_VLM/jumping_Cat_struc.sb3 differ diff --git a/scratch_VLM/scratch_agent/.env b/scratch_VLM/scratch_agent/.env new file mode 100644 index 0000000000000000000000000000000000000000..29db2a5ac45d92ebe6f0f04e2c2cae9eafa22324 --- /dev/null +++ b/scratch_VLM/scratch_agent/.env @@ -0,0 +1,8 @@ +MISTRAL_API_KEY= "4ObQF29a1JbbYQxDafYXxxoTF02tC4DB" +GROQ_API_KEY = "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" +allow_unsafe_werkzeug=True +TAVILY_API_KEY= "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" +PINECONE_API = "pcsk_3GWkuM_S6YsnKKUUYgkjwDf2J44rtUq9z2KSkPko8eqnQ6WFpM5UQhHihWmp7vHw712MZm" +OPENROUTER_BASE_URL= "https://openrouter.ai/api/v1" +OPENROUTER_API_KEY= "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" +GROQ_API_KEY_2="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" \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/README.md b/scratch_VLM/scratch_agent/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3fd279a274f0aca05d342fe3444c7fe9dc97e8be --- /dev/null +++ b/scratch_VLM/scratch_agent/README.md @@ -0,0 +1,89 @@ +# General Agent with Audio, Image, and File Processing + +This is a general-purpose agent built with LangChain/LangGraph that includes advanced tools for processing various types of data. + +## Tool Categories + +The agent includes tools for working with: + +- **Web Content**: Search web pages, news articles, Wikipedia, ArXiv papers +- **Files**: Read PDFs, DOCXs, Excel files +- **Media**: Process images, transcribe audio, extract text from YouTube videos +- **Code**: Analyze code structure, read code files, analyze functions +- **Math**: Basic math operations + +## Testing Your Installation + +Before using the agent, you should check if all dependencies are installed and tools are working correctly: + +1. **Check and install dependencies**: + ``` + python fix_dependencies.py + ``` + This script will check for missing Python packages and system dependencies. + +2. **Test all tools**: + ``` + python test_all_tools.py + ``` + This will test all tools and report any issues. + +3. **Test image and audio processing specifically**: + ``` + python test_image_audio.py + ``` + This focuses on testing media processing tools and provides detailed troubleshooting steps. + +## System Requirements + +For full functionality, you'll need: + +- **Python 3.8+** +- **Tesseract OCR** (for image text extraction) +- **FFmpeg** (for audio processing) +- **Internet connection** (for web search, YouTube, etc.) +- **API Keys**: GROQ_API_KEY must be set in .env file or environment variables + +## Agent Structure + +This agent uses a streamlined 3-node graph structure: + +1. **PerceptionAgent**: Handles web searches, looking up information +2. **ActionAgent**: Performs calculations, file operations, code analysis +3. **EvaluationAgent**: Ensures answers are properly formatted + +## Common Issues + +If you encounter issues: + +1. **Web Scraping Errors**: The agent has robust error handling for 403 Forbidden errors +2. **Audio Processing Errors**: Make sure FFmpeg is installed and in your PATH +3. **Image Processing Errors**: Make sure Tesseract OCR is installed and in your PATH +4. **GROQ API Rate Limits**: The agent includes automatic rate limiting and retry mechanisms + +## Running GAIA Tests + +To test if the agent can properly handle factual questions with GAIA format: + +``` +python test_factual_questions.py +``` + +## Testing Individual Tools + +```python +from agent import multiply, add, subtract, divide, modulus # Math tools +from agent import web_search, wiki_search # Web tools +from agent import read_text_from_pdf, read_text_from_docx # Document tools +from agent import image_processing # Image tools +from agent import transcribe_audio # Audio tools +from agent import analyze_code, read_code_file # Code tools + +# Test a tool directly +result = multiply(5, 7) +print(result) # 35 + +# Process an image +image_description = image_processing("Describe this image", "path/to/image.jpg") +print(image_description) +``` \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/__pycache__/agent.cpython-312.pyc b/scratch_VLM/scratch_agent/__pycache__/agent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..396b501ac82c92801503f843245f36b641fdf106 Binary files /dev/null and b/scratch_VLM/scratch_agent/__pycache__/agent.cpython-312.pyc differ diff --git a/scratch_VLM/scratch_agent/__pycache__/retry_groq.cpython-312.pyc b/scratch_VLM/scratch_agent/__pycache__/retry_groq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d46a9696423eb50ec2748446cf0857357ea1d54b Binary files /dev/null and b/scratch_VLM/scratch_agent/__pycache__/retry_groq.cpython-312.pyc differ diff --git a/scratch_VLM/scratch_agent/action_node.py b/scratch_VLM/scratch_agent/action_node.py new file mode 100644 index 0000000000000000000000000000000000000000..2ceb5afe86d992bb0d657fa39b3faa47f3e26edc --- /dev/null +++ b/scratch_VLM/scratch_agent/action_node.py @@ -0,0 +1,452 @@ +import json +import uuid +import logging +from typing import Dict, Any, List + +# Assume GameState is a TypedDict or similar for clarity +# from typing import TypedDict +# class GameState(TypedDict): +# description: str +# project_json: Dict[str, Any] +# action_plan: Dict[str, Any] +# sprite_initial_positions: Dict[str, Any] + +# Placeholder for actual GameState in your application +GameState = Dict[str, Any] + +logger = logging.getLogger(__name__) + +# --- Mock Agent for demonstration --- +class MockAgent: + def invoke(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """ + Mocks an LLM agent invocation. In a real scenario, this would call your + actual LLM API (e.g., through LangChain, LlamaIndex, etc.). + """ + user_message = payload["messages"][-1]["content"] + print(f"\n--- Mock Agent Received Prompt (partial) ---\n{user_message[:500]}...\n------------------------------------------") + + # Simplified mock responses for demonstration purposes + # In a real scenario, the LLM would generate actual Scratch block JSON + if "Propose a high-level action flow" in user_message: + return { + "messages": [{ + "content": json.dumps({ + "action_overall_flow": { + "Sprite1": { + "description": "Basic movement and interaction", + "plans": [ + { + "event": "when flag clicked", + "logic": "forever loop: move 10 steps, if touching Edge then turn 15 degrees" + }, + { + "event": "when space key pressed", + "logic": "say Hello! for 2 seconds" + } + ] + }, + "Ball": { + "description": "Simple bouncing behavior", + "plans": [ + { + "event": "when flag clicked", + "logic": "move 5 steps, if on edge bounce" + } + ] + } + } + }) + }] + } + elif "You are an AI assistant generating Scratch 3.0 block JSON" in user_message: + # This mock response is highly simplified. A real LLM would generate + # valid Scratch blocks based on the provided relevant catalog and plan. + # We're just demonstrating the *mechanism* of filtering the catalog. + if "Sprite1" in user_message: + return { + "messages": [{ + "content": json.dumps({ + f"block_id_{generate_block_id()}": { + "opcode": "event_whenflagclicked", + "next": f"block_id_{generate_block_id()}_forever", + "parent": None, + "inputs": {}, "fields": {}, "shadow": False, "topLevel": True, "x": 100, "y": 100 + }, + f"block_id_{generate_block_id()}_forever": { + "opcode": "control_forever", + "next": None, + "parent": f"block_id_{generate_block_id()}", + "inputs": { + "SUBSTACK": [2, f"block_id_{generate_block_id()}_move"] + }, "fields": {}, "shadow": False, "topLevel": False + }, + f"block_id_{generate_block_id()}_move": { + "opcode": "motion_movesteps", + "next": f"block_id_{generate_block_id()}_if", + "parent": f"block_id_{generate_block_id()}_forever", + "inputs": { + "STEPS": [1, [4, "10"]] + }, "fields": {}, "shadow": False, "topLevel": False + }, + f"block_id_{generate_block_id()}_if": { + "opcode": "control_if", + "next": None, + "parent": f"block_id_{generate_block_id()}_forever", + "inputs": { + "CONDITION": [2, f"block_id_{generate_block_id()}_touching"], + "SUBSTACK": [2, f"block_id_{generate_block_id()}_turn"] + }, "fields": {}, "shadow": False, "topLevel": False + }, + f"block_id_{generate_block_id()}_touching": { + "opcode": "sensing_touchingobject", + "next": None, + "parent": f"block_id_{generate_block_id()}_if", + "inputs": { + "TOUCHINGOBJECTMENU": [1, f"block_id_{generate_block_id()}_touching_menu"] + }, "fields": {}, "shadow": False, "topLevel": False + }, + f"block_id_{generate_block_id()}_touching_menu": { + "opcode": "sensing_touchingobjectmenu", + "next": None, + "parent": f"block_id_{generate_block_id()}_touching", + "inputs": {}, + "fields": {"TOUCHINGOBJECTMENU": ["_edge_", None]}, + "shadow": True, "topLevel": False + }, + f"block_id_{generate_block_id()}_turn": { + "opcode": "motion_turnright", + "next": None, + "parent": f"block_id_{generate_block_id()}_if", + "inputs": { + "DEGREES": [1, [4, "15"]] + }, "fields": {}, "shadow": False, "topLevel": False + }, + f"block_id_{generate_block_id()}_say": { + "opcode": "looks_sayforsecs", + "next": None, + "parent": None, # This block would typically be part of a separate script + "inputs": { + "MESSAGE": [1, [10, "Hello!"]], + "SECS": [1, [4, "2"]] + }, "fields": {}, "shadow": False, "topLevel": True, "x": 300, "y": 100 + } + }) + }] + } + elif "Ball" in user_message: + return { + "messages": [{ + "content": json.dumps({ + f"block_id_{generate_block_id()}": { + "opcode": "event_whenflagclicked", + "next": f"block_id_{generate_block_id()}_moveball", + "parent": None, + "inputs": {}, "fields": {}, "shadow": False, "topLevel": True, "x": 100, "y": 100 + }, + f"block_id_{generate_block_id()}_moveball": { + "opcode": "motion_movesteps", + "next": f"block_id_{generate_block_id()}_edgebounce", + "parent": f"block_id_{generate_block_id()}", + "inputs": { + "STEPS": [1, [4, "5"]] + }, "fields": {}, "shadow": False, "topLevel": False + }, + f"block_id_{generate_block_id()}_edgebounce": { + "opcode": "motion_ifonedgebounce", + "next": None, + "parent": f"block_id_{generate_block_id()}_moveball", + "inputs": {}, "fields": {}, "shadow": False, "topLevel": False + } + }) + }] + } + return {"messages": [{"content": "[]"}]} # Default empty response + +agent = MockAgent() + +# Helper function to generate a unique block ID +def generate_block_id(): + return str(uuid.uuid4())[:10].replace('-', '') # Shorten for readability, ensure uniqueness + +# Placeholder for your extract_json_from_llm_response function +def extract_json_from_llm_response(response_string): + try: + # Assuming the LLM response is ONLY the JSON string within triple backticks + json_match = response_string.strip().replace("```json", "").replace("```", "").strip() + return json.loads(json_match) + except json.JSONDecodeError as e: + logger.error(f"Failed to decode JSON from LLM response: {e}") + logger.error(f"Raw response: {response_string}") + raise ValueError("Invalid JSON response from LLM") + +# --- GLOBAL CATALOG OF ALL SCRATCH BLOCKS --- +# This is where you would load your block_content.json +# For demonstration, I'm using your provided snippets and adding some common ones. +# In a real application, you'd load this once at startup. +ALL_SCRATCH_BLOCKS_CATALOG = { + "motion_movesteps": { + "opcode": "motion_movesteps", "next": None, "parent": None, + "inputs": {"STEPS": [1, [4, "10"]]}, "fields": {}, "shadow": False, "topLevel": True, "x": 464, "y": -416 + }, + "motion_turnright": { + "opcode": "motion_turnright", "next": None, "parent": None, + "inputs": {"DEGREES": [1, [4, "15"]]}, "fields": {}, "shadow": False, "topLevel": True, "x": 467, "y": -316 + }, + "motion_ifonedgebounce": { + "opcode": "motion_ifonedgebounce", "next": None, "parent": None, + "inputs": {}, "fields": {}, "shadow": False, "topLevel": True, "x": 467, "y": -316 + }, + "event_whenflagclicked": { + "opcode": "event_whenflagclicked", "next": None, "parent": None, + "inputs": {}, "fields": {}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "event_whenkeypressed": { + "opcode": "event_whenkeypressed", "next": None, "parent": None, + "inputs": {}, "fields": {"KEY_OPTION": ["space", None]}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "control_forever": { + "opcode": "control_forever", "next": None, "parent": None, + "inputs": {"SUBSTACK": [2, "some_id"]}, "fields": {}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "control_if": { + "opcode": "control_if", "next": None, "parent": None, + "inputs": {"CONDITION": [2, "some_id"], "SUBSTACK": [2, "some_id_2"]}, "fields": {}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "looks_sayforsecs": { + "opcode": "looks_sayforsecs", "next": None, "parent": None, + "inputs": {"MESSAGE": [1, [10, "Hello!"]], "SECS": [1, [4, "2"]]}, "fields": {}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "looks_say": { + "opcode": "looks_say", "next": None, "parent": None, + "inputs": {"MESSAGE": [1, [10, "Hello!"]]}, "fields": {}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "sensing_touchingobject": { + "opcode": "sensing_touchingobject", "next": None, "parent": None, + "inputs": {"TOUCHINGOBJECTMENU": [1, "some_id"]}, "fields": {}, "shadow": False, "topLevel": True, "x": 10, "y": 10 + }, + "sensing_touchingobjectmenu": { + "opcode": "sensing_touchingobjectmenu", "next": None, "parent": None, + "inputs": {}, "fields": {"TOUCHINGOBJECTMENU": ["_mouse_", None]}, "shadow": True, "topLevel": True, "x": 10, "y": 10 + }, + # Add more blocks from your block_content.json here... +} + +# --- Heuristic-based block selection --- +def get_relevant_blocks_for_plan(action_plan: Dict[str, Any], all_blocks_catalog: Dict[str, Any]) -> Dict[str, Any]: + """ + Analyzes the natural language action plan and selects relevant Scratch blocks + from the comprehensive catalog. This is a heuristic approach and might need + to be refined based on your specific use cases and LLM capabilities. + """ + relevant_opcodes = set() + + # Always include common event blocks + relevant_opcodes.add("event_whenflagclicked") + relevant_opcodes.add("event_whenkeypressed") # Could be more specific if key is mentioned + + # Keyword to opcode mapping (can be expanded) + keyword_map = { + "move": "motion_movesteps", + "steps": "motion_movesteps", + "turn": "motion_turnright", + "rotate": "motion_turnright", + "bounce": "motion_ifonedgebounce", + "edge": "motion_ifonedgebounce", + "forever": "control_forever", + "loop": "control_forever", + "if": "control_if", + "condition": "control_if", + "say": "looks_say", + "hello": "looks_say", # Simple example, might need more context + "touching": "sensing_touchingobject", + "mouse pointer": "sensing_touchingobjectmenu", + "edge": "sensing_touchingobjectmenu", # For touching edge + } + + # Iterate through the action plan to find keywords + for sprite_name, sprite_actions in action_plan.get("action_overall_flow", {}).items(): + for plan in sprite_actions.get("plans", []): + event_logic = plan.get("event", "").lower() + " " + plan.get("logic", "").lower() + + # Check for direct opcode matches (if the LLM somehow outputs opcodes in its plan) + for opcode in all_blocks_catalog.keys(): + if opcode in event_logic: + relevant_opcodes.add(opcode) + + # Check for keywords + for keyword, opcode in keyword_map.items(): + if keyword in event_logic: + relevant_opcodes.add(opcode) + # Add associated shadow blocks if known + if opcode == "sensing_touchingobject": + relevant_opcodes.add("sensing_touchingobjectmenu") + if opcode == "event_whenkeypressed": + relevant_opcodes.add("event_whenkeypressed") # It's already there but good to be explicit + + # Construct the filtered catalog + relevant_blocks_catalog = { + opcode: all_blocks_catalog[opcode] + for opcode in relevant_opcodes if opcode in all_blocks_catalog + } + return relevant_blocks_catalog + +# --- New Action Planning Node --- +def plan_sprite_actions(state: GameState): + logger.info("--- Running PlanSpriteActionsNode ---") + + planning_prompt = ( + f"You are an AI assistant tasked with planning Scratch 3.0 block code for a game. " + f"The game description is: '{state['description']}'.\n\n" + f"Here are the sprites currently in the project: {', '.join(target['name'] for target in state['project_json']['targets'] if not target['isStage']) if len(state['project_json']['targets']) > 1 else 'None'}.\n" + f"Initial positions: {json.dumps(state.get('sprite_initial_positions', {}), indent=2)}\n\n" + f"Consider the main actions and interactions required for each sprite. " + f"Think step-by-step about what each sprite needs to *do*, *when* it needs to do it (events), " + f"and if any actions need to *repeat* or depend on *conditions*.\n\n" + f"Propose a high-level action flow for each sprite in the following JSON format. " + f"Do NOT generate Scratch block JSON yet. Only describe the logic using natural language or simplified pseudo-code.\n\n" + f"Example format:\n" + f"```json\n" + f"{{\n" + f" \"action_overall_flow\": {{\n" + f" \"Sprite1\": {{\n" + f" \"description\": \"Main character actions\",\n" + f" \"plans\": [\n" + f" {{\n" + f" \"event\": \"when flag clicked\",\n" + f" \"logic\": \"forever loop: move 10 steps, if on edge bounce\"\n" + f" }},\n" + f" {{\n" + f" \"event\": \"when space key pressed\",\n" + f" \"logic\": \"change y by 10, wait 0.1 seconds, change y by -10\"\n" + f" }}\n" + f" ]\n" + f" }},\n" + f" \"Ball\": {{\n" + f" \"description\": \"Projectile movement\",\n" + f" \"plans\": [\n" + f" {{\n" + f" \"event\": \"when I start as a clone\",\n" + f" \"logic\": \"glide 1 sec to random position, if touching Sprite1 then stop this script\"\n" + f" }}\n" + f" ]\n" + f" }}\n" + f" }}\n" + f"}}\n" + f"```\n\n" + f"Return ONLY the JSON object for the action overall flow." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM [PlanSpriteActionsNode]:", raw_response) + action_plan = extract_json_from_llm_response(raw_response) + logger.info("Sprite action plan generated by PlanSpriteActionsNode.") + return {"action_plan": action_plan} + except Exception as e: + logger.error(f"Error in PlanSpriteActionsNode: {e}") + raise + +# --- Updated Action Node Builder (to consume the plan and build blocks) --- +def build_action_nodes(state: GameState): + logger.info("--- Running ActionNodeBuilder ---") + + action_plan = state.get("action_plan", {}) + if not action_plan: + raise ValueError("No action plan found in state. Run PlanSpriteActionsNode first.") + + # Convert the Scratch project JSON to a mutable Python object + project_json = state["project_json"] + targets = project_json["targets"] + + # We need a way to map sprite names to their actual target objects in project_json + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + + # --- NEW: Get only the relevant blocks for the entire action plan --- + relevant_scratch_blocks_catalog = get_relevant_blocks_for_plan(action_plan, ALL_SCRATCH_BLOCKS_CATALOG) + logger.info(f"Filtered {len(relevant_scratch_blocks_catalog)} relevant blocks out of {len(ALL_SCRATCH_BLOCKS_CATALOG)} total.") + + + # Iterate through the planned actions for each sprite + for sprite_name, sprite_actions in action_plan.get("action_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + # Ensure 'blocks' field exists for the sprite + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + # Generate block JSON based on the detailed action plan for this sprite + # This is where the LLM's role becomes crucial: translating logic to blocks + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON based on a provided plan. " + f"The current sprite is '{sprite_name}'.\n" + f"Its planned actions are:\n" + f"```json\n{json.dumps(sprite_actions, indent=2)}\n```\n\n" + f"Here is a **curated catalog of only the most relevant Scratch 3.0 blocks** for this plan:\n" + f"```json\n{json.dumps(relevant_scratch_blocks_catalog, indent=2)}\n```\n\n" + f"Current Scratch project JSON (for context, specifically this sprite's existing blocks if any):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions:**\n" + f"1. For each planned event and its associated logic, generate the corresponding Scratch 3.0 block JSON.\n" + f"2. **Generate unique block IDs** for every new block. Use a format like 'block_id_abcdef12'.\n" # Updated ID format hint + f"3. Properly link blocks using `next` and `parent` fields to form execution stacks. Hat blocks (`topLevel: true`, `parent: null`).\n" + f"4. Correctly fill `inputs` and `fields` based on the catalog and the plan's logic (e.g., specific values for motion, keys for events, conditions for controls).\n" + f"5. For C-blocks (like `control_repeat`, `control_forever`, `control_if`), use the `SUBSTACK` input to link to the first block inside its loop/conditional.\n" + f"6. If the plan involves operators (e.g., 'if touching Sprite1'), use the appropriate operator blocks from the catalog and link them correctly as `CONDITION` inputs.\n" + f"7. Ensure that any shadow blocks (e.g., for dropdowns like `motion_goto_menu`, `sensing_touchingobjectmenu`) are generated with `shadow: true` and linked correctly as inputs to their parent block.\n" + f"8. Return ONLY the **updated 'blocks' dictionary** for this specific sprite. Do NOT return the full project JSON. ONLY the `blocks` dictionary." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + print(f"Raw response from LLM [ActionNodeBuilder - {sprite_name}]:", raw_response) + generated_blocks = extract_json_from_llm_response(raw_response) + current_sprite_target["blocks"].update(generated_blocks) # Merge new blocks + logger.info(f"Action blocks added for sprite '{sprite_name}' by ActionNodeBuilder.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}': {e}") + # Depending on robustness needed, you might continue or re-raise + raise + + return {"project_json": project_json} + +# --- Example Usage (to demonstrate the flow) --- +if __name__ == "__main__": + # Initialize a mock game state + initial_game_state = { + "description": "A simple game where a sprite moves and says hello.", + "project_json": { + "targets": [ + {"isStage": True, "name": "Stage", "blocks": {}}, + {"isStage": False, "name": "Sprite1", "blocks": {}}, + {"isStage": False, "name": "Ball", "blocks": {}} + ] + }, + "sprite_initial_positions": {} + } + + # Step 1: Plan Sprite Actions + try: + state_after_planning = plan_sprite_actions(initial_game_state) + initial_game_state.update(state_after_planning) + print("\n--- Game State After Planning ---") + print(json.dumps(initial_game_state, indent=2)) + except Exception as e: + print(f"Planning failed: {e}") + exit() + + # Step 2: Build Action Nodes (Generate Blocks) + try: + state_after_building = build_action_nodes(initial_game_state) + initial_game_state.update(state_after_building) + print("\n--- Game State After Building Blocks ---") + # Print only the blocks for a specific sprite to keep output manageable + for target in initial_game_state["project_json"]["targets"]: + if not target["isStage"]: + print(f"\nBlocks for {target['name']}:") + print(json.dumps(target.get('blocks', {}), indent=2)) + + except Exception as e: + print(f"Building blocks failed: {e}") \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/agent.py b/scratch_VLM/scratch_agent/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..76c915f75058254e66372ed1b9d751798af36e36 --- /dev/null +++ b/scratch_VLM/scratch_agent/agent.py @@ -0,0 +1,1647 @@ +#─── Basic imports ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +import os +import math +import sqlite3 +import fitz # PyMuPDF for PDF parsing +import re + +from dotenv import load_dotenv +# Load environment variables from .env file +load_dotenv() # This line ensures .env variables are loaded + +from langgraph.graph import START, StateGraph, MessagesState, END +from langgraph.prebuilt import tools_condition +from langgraph.prebuilt import ToolNode +from langgraph.constants import START +from langchain_core.tools import tool +from langchain.schema import SystemMessage +#from langchain.chat_models import init_chat_model +#from langgraph.prebuilt import create_react_agent + +from langchain.embeddings import HuggingFaceEmbeddings +#from langchain.vectorstores import Pinecone +from langchain.tools.retriever import create_retriever_tool +#import pinecone +#from pinecone import Pinecone as PineconeClient, ServerlessSpec +#from pinecone import Index # the blocking‐call client constructor +#from pinecone import Pinecone as PineconeClient, ServerlessSpec +from langchain.embeddings import HuggingFaceEmbeddings +from langchain_community.vectorstores.pinecone import Pinecone as LC_Pinecone + +# ─── Langchain Frameworks ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#from langchain.tools import Tool +from langchain.chat_models import ChatOpenAI +from langchain_groq import ChatGroq +from langchain_mistralai import ChatMistralAI +from langchain.agents import initialize_agent, AgentType +from langchain.schema import Document +from langchain.chains import RetrievalQA +from langchain.embeddings import OpenAIEmbeddings +from langchain_community.embeddings import HuggingFaceEmbeddings +from langchain.vectorstores import FAISS +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.prompts import PromptTemplate +from langchain_community.document_loaders import TextLoader, PyMuPDFLoader +from langchain_community.document_loaders.wikipedia import WikipediaLoader +from langchain_community.document_loaders.arxiv import ArxivLoader +from langchain_experimental.tools.python.tool import PythonREPLTool + + +# ─── Memory ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +from langchain.agents import initialize_agent, AgentType +from langchain.tools import Tool +from typing import List, Callable +from langchain.schema import BaseMemory, AIMessage, HumanMessage, SystemMessage +from langchain.schema import HumanMessage, SystemMessage +from langchain.llms.base import LLM +from langchain.memory.chat_memory import BaseChatMemory +from pydantic import PrivateAttr +from langchain_core.messages import get_buffer_string + +# ─── Image Processing ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +from PIL import Image +import pytesseract +from transformers import pipeline +from groq import Groq +import requests +from io import BytesIO +from transformers import pipeline, TrOCRProcessor, VisionEncoderDecoderModel +import requests +import base64 +from PIL import UnidentifiedImageError + +# ─── Browser var ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +from typing import List, Dict +import json +from io import BytesIO +#from langchain.tools import tool # or langchain_core.tools +from playwright.sync_api import sync_playwright +from duckduckgo_search import DDGS +import time +import random +import logging +from functools import lru_cache, wraps +import requests +from playwright.sync_api import sync_playwright +from bs4 import BeautifulSoup +import tenacity +from tenacity import retry, stop_after_attempt, wait_exponential + +# Initialize logger +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Additional imports for new functionality +import pandas as pd +from PyPDF2 import PdfReader +import docx +import pytesseract +import speech_recognition as sr +from pydub import AudioSegment +from pytube import YouTube +from newspaper import Article +from langchain.document_loaders import ArxivLoader +from langchain_community.document_loaders.youtube import YoutubeLoader, TranscriptFormat + +from playwright.sync_api import sync_playwright +# Attempt to import Playwright for dynamic page rendering +try: + from playwright.sync_api import sync_playwright + _playwright_available = True +except ImportError: + _playwright_available = False + +# Define forbidden keywords for basic NSFW filtering +_forbidden = ["porn", "sex", "xxx", "nude", "erotic"] + +# ─── LLM Setup ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# Load OpenAI API key from environment (required for LLM and embeddings) + +# API Keys from .env file +os.environ.setdefault("OPENAI_API_KEY", "") # Set your own key or env var +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") +os.environ["MISTRAL_API_KEY"] = os.getenv("MISTRAL_API_KEY", "default_key_or_placeholder") + +# Tavily API Key +TAVILY_API_KEY = os.getenv("TAVILY_API_KEY", "default_key_or_placeholder") +_forbidden = ["nsfw", "porn", "sex", "explicit"] +_playwright_available = True # set False to disable Playwright + +# Globals for RAG system +vector_store = None +rag_chain = None +DB_PATH = None # will be set when a .db is uploaded +DOC_PATH = None # will be set when a document is uploaded +IMG_PATH = None # will be set when an image is uploaded +OTH_PATH = None # will be set when an other file is uploaded + + +# ─── LLMS ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +#llm = ChatOpenAI(model_name="gpt-3.5-turbo", streaming=True, temperature=0) +from tenacity import retry, stop_after_attempt, wait_exponential + +# Import the RetryingChatGroq client +from retry_groq import RetryingChatGroq + +# Use the retrying version instead +llm = RetryingChatGroq(model="deepseek-r1-distill-llama-70b", streaming=False, temperature=0) +#llm = ChatMistralAI(model="mistral-large-latest", streaming=True, temperature=0) + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for multiply ────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +@tool(parse_docstring=True) +def multiply(a: int, b: int) -> int: + """ + Multiply two numbers. + + Args: + a (int): The first factor. + b (int): The second factor. + + Returns: + int: The product of a and b. + """ + try: + # Direct calculation without relying on LangChain handling + result = a * b + return result + except Exception as e: + return f"Error in multiplication: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for add ────────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +@tool(parse_docstring=True) +def add(a: int, b: int) -> int: + """ + Add two numbers. + + Args: + a (int): The first factor. + b (int): The second factor. + + Returns: + int: The addition of a and b. + """ + try: + # Direct calculation without relying on LangChain handling + result = a + b + return result + except Exception as e: + return f"Error in addition: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for subtract ────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +@tool(parse_docstring=True) +def subtract(a: int, b: int) -> int: + """ + Subtract two numbers. + + Args: + a (int): The first factor. + b (int): The second factor. + + Returns: + int: The subtraction of a and b. + """ + try: + # Direct calculation without relying on LangChain handling + result = a - b + return result + except Exception as e: + return f"Error in subtraction: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for divide ────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +@tool(parse_docstring=True) +def divide(a: int, b: int) -> int: + """ + Divide two numbers. + + Args: + a (int): The numerator. + b (int): The denominator. + + Returns: + float: The result of a divided by b. + + Raises: + ValueError: If b is zero. + """ + try: + if b == 0: + return "Error: Cannot divide by zero." + # Direct calculation without relying on LangChain handling + result = a / b + return result + except Exception as e: + return f"Error in division: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for modulus ────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +@tool(parse_docstring=True) +def modulus(a: int, b: int) -> int: + """ + Get the modulus (remainder) of two numbers. + + Args: + a (int): The dividend. + b (int): The divisor. + + Returns: + int: The remainder when a is divided by b. + """ + try: + if b == 0: + return "Error: Cannot calculate modulus with zero divisor." + # Direct calculation without relying on LangChain handling + result = a % b + return result + except Exception as e: + return f"Error in modulus calculation: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for browsing ────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +def with_retry(max_attempts: int = 3, backoff_base: int = 2): + """ + Decorator for retrying a function with exponential backoff on exception. + """ + def decorator(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + for attempt in range(max_attempts): + try: + return fn(*args, **kwargs) + except Exception as e: + wait = backoff_base ** attempt + random.uniform(0, 1) + logger.warning(f"{fn.__name__} failed (attempt {attempt+1}/{max_attempts}): {e}") + if attempt < max_attempts - 1: + time.sleep(wait) + logger.error(f"{fn.__name__} failed after {max_attempts} attempts.") + return [] + return wrapper + return decorator + +@with_retry() +@lru_cache(maxsize=128) +def tavily_search(query: str, top_k: int = 3) -> List[Dict]: + """Call Tavily API and return a list of result dicts.""" + if not TAVILY_API_KEY: + logger.info("[Tavily] No API key set. Skipping Tavily search.") + return [] + url = "https://api.tavily.com/search" + headers = { + "Authorization": f"Bearer {TAVILY_API_KEY}", + "Content-Type": "application/json", + } + payload = {"query": query, "num_results": top_k} + resp = requests.post(url, headers=headers, json=payload, timeout=10) + resp.raise_for_status() + data = resp.json() + results = [] + for item in data.get("results", []): + results.append({ + "title": item.get("title", ""), + "url": item.get("url", ""), + "content": item.get("content", "")[:200], + "source": "Tavily" + }) + return results + +@with_retry() +@lru_cache(maxsize=128) +def duckduckgo_search(query: str, top_k: int = 3) -> List[Dict]: + """Query DuckDuckGo and return up to top_k raw SERP hits.""" + results = [] + try: + with DDGS(timeout=15) as ddgs: # Increase timeout from default + for hit in ddgs.text(query, safesearch="On", max_results=top_k, timeout=15): + results.append({ + "title": hit.get("title", ""), + "url": hit.get("href") or hit.get("url", ""), + "content": hit.get("body", ""), + "source": "DuckDuckGo" + }) + if len(results) >= top_k: + break + except Exception as e: + logger.warning(f"DuckDuckGo search failed: {e}") + # Don't re-raise - just return empty results to allow fallbacks to work + + return results + +# Additional fallback search alternative +def simple_google_search(query: str, top_k: int = 3) -> List[Dict]: + """Simplified Google search as a fallback when other methods fail.""" + try: + # Encode the query + import urllib.parse + import bs4 + + encoded_query = urllib.parse.quote(query) + url = f"https://www.google.com/search?q={encoded_query}" + + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Referer": "https://www.google.com/", + "Connection": "keep-alive", + } + + response = requests.get(url, headers=headers, timeout=20) + response.raise_for_status() + + soup = bs4.BeautifulSoup(response.text, "html.parser") + results = [] + + # Extract search results + for result in soup.select("div.g")[:top_k]: + title_elem = result.select_one("h3") + link_elem = result.select_one("a") + snippet_elem = result.select_one("div.VwiC3b") + + if title_elem and link_elem and snippet_elem and "href" in link_elem.attrs: + href = link_elem["href"] + if href.startswith("/url?q="): + href = href.split("/url?q=")[1].split("&")[0] + + if href.startswith("http"): + results.append({ + "title": title_elem.get_text(), + "url": href, + "content": snippet_elem.get_text(), + "source": "Google" + }) + + return results + + except Exception as e: + logger.warning(f"Simple Google search failed: {e}") + return [] + +def hybrid_search(query: str, top_k: int = 3) -> List[Dict]: + """Combine multiple search sources with fallbacks.""" + # Try primary search methods first + results = [] + + # Start with Tavily if API key is available + if TAVILY_API_KEY and TAVILY_API_KEY != "default_key_or_placeholder": + try: + tavily_results = tavily_search(query, top_k) + results.extend(tavily_results) + logger.info(f"Retrieved {len(tavily_results)} results from Tavily") + except Exception as e: + logger.warning(f"Tavily search failed: {e}") + + # If we don't have enough results, try DuckDuckGo + if len(results) < top_k: + try: + ddg_results = duckduckgo_search(query, top_k - len(results)) + results.extend(ddg_results) + logger.info(f"Retrieved {len(ddg_results)} results from DuckDuckGo") + except Exception as e: + logger.warning(f"DuckDuckGo search failed: {e}") + + # If we still don't have enough results, try Google + if len(results) < top_k: + try: + google_results = simple_google_search(query, top_k - len(results)) + results.extend(google_results) + logger.info(f"Retrieved {len(google_results)} results from Google") + except Exception as e: + logger.warning(f"Google search failed: {e}") + + # If all search methods failed, return a dummy result + if not results: + results.append({ + "title": "Search Failed", + "url": "", + "content": f"Sorry, I couldn't find results for '{query}'. Please try refining your search terms or check your internet connection.", + "source": "No results" + }) + + return results[:top_k] # Ensure we only return top_k results + +def format_search_docs(search_docs: List[Dict]) -> Dict[str, str]: + """ + Turn a list of {source, page, content} dicts into one big + string with … entries separated by `---`. + """ + formatted_search_docs = "\n\n---\n\n".join( + [ + f'\n' + f'{doc.get("content", "")}\n' + f'' + for doc in search_docs + ] + ) + return {"web_results": formatted_search_docs} + + +@tool(parse_docstring=True) +def web_search(query: str, top_k: int = 3) -> Dict[str, str]: + """ + Perform a hybrid web search combining multiple search engines with robust fallbacks. + + Args: + query: The search query string to look up. + top_k: The maximum number of search results to return (default is 3). + + Returns: + A dictionary mapping result indices to XML-like blocks, each containing: + - source: The URL of the webpage. + - page: Placeholder for page identifier (empty string by default). + - content: The first 200 words of the page text, cleaned of HTML tags. + """ + try: + # Use our robust hybrid search to get initial results + search_results = hybrid_search(query, top_k) + results = [] + + # Process each search result to get better content + for hit in search_results: + url = hit.get("url") + if not url: + continue + + # Start with the snippet from search + content = hit.get("content", "") + title = hit.get("title", "") + + # Try to scrape additional content if possible + try: + # Use a random user agent to avoid blocking + headers = { + "User-Agent": random.choice([ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 Edg/97.0.1072.62" + ]), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Referer": "https://www.google.com/", + "DNT": "1", + "Connection": "keep-alive" + } + + # Higher timeout for better reliability + resp = requests.get(url, timeout=15, headers=headers) + + # Only process if successful + if resp.status_code == 200: + soup = BeautifulSoup(resp.text, "html.parser") + + # Try to find main content + main_content = soup.find('main') or soup.find('article') or soup.find('div', class_='content') + + # If we found main content, use it + if main_content: + extracted_text = main_content.get_text(separator=" ", strip=True) + # Take first 200 words + content = " ".join(extracted_text.split()[:200]) + else: + # Otherwise use all text + all_text = soup.get_text(separator=" ", strip=True) + content = " ".join(all_text.split()[:200]) + + # Use content from page only if it's substantial + if len(content) < 50: + content = hit.get("content", "")[:200] + + # Random delay between 0.5-1.5 seconds to avoid rate limits + time.sleep(0.5 + random.random()) + + except requests.exceptions.HTTPError as e: + logger.warning(f"HTTP error when scraping {url}: {e}") + # Keep the search snippet as a fallback + except requests.exceptions.RequestException as e: + logger.warning(f"Request error when scraping {url}: {e}") + # Keep the search snippet as a fallback + except Exception as e: + logger.warning(f"Unexpected error when scraping {url}: {e}") + # Keep the search snippet as a fallback + + # Filter out inappropriate content + if any(f in content.lower() for f in _forbidden): + continue + + # Add to results + results.append({ + "source": url, + "page": "", + "content": content + }) + + # Return formatted search docs + return format_search_docs(results[:top_k]) + except Exception as e: + logger.error(f"Web search failed: {e}") + # Return a helpful error message + return format_search_docs([{ + "source": "Error", + "page": "", + "content": f"Search failed with error: {e}. Please try again with different search terms." + }]) + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for File System ─────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +@tool(parse_docstring=True) +def download_file(url: str, dest_path: str) -> str: + """ + Download a file from a given URL and save it locally. + + Args: + url: The direct URL of the file to download. + dest_path: The local path to save the downloaded file. + + Returns: + The destination path where the file was saved. + """ + r = requests.get(url, stream=True) + r.raise_for_status() + with open(dest_path, 'wb') as f: + for chunk in r.iter_content(8192): + f.write(chunk) + return dest_path + +@tool(parse_docstring=True) +def process_excel_to_text(file_path: str) -> str: + """ + Convert an Excel file into CSV-formatted text. + + Args: + file_path: Path to the Excel (.xlsx) file. + + Returns: + A string of CSV-formatted content extracted from the Excel file. + """ + try: + # Check if file exists + import os + if not os.path.exists(file_path): + return f"Error: Excel file '{file_path}' does not exist." + + # Try different engines + engines = ['openpyxl', 'xlrd', None] + + for engine in engines: + try: + # For engine=None, pandas will try to auto-detect + if engine: + df = pd.read_excel(file_path, engine=engine) + else: + df = pd.read_excel(file_path) + return df.to_csv(index=False) + except Exception as e: + print(f"Excel engine {engine} failed: {e}") + last_error = e + continue + + # If we got here, all engines failed + return f"Error processing Excel file: {str(last_error)}" + except Exception as e: + return f"Error with Excel file: {str(e)}" + +@tool(parse_docstring=True) +def read_text_from_pdf(file_path: str, question: str = None) -> str: + """ + Extract text from a PDF file, chunking large documents if needed. + + Args: + file_path: Path to the PDF file. + question: Optional question to help retrieve relevant parts of long documents. + + Returns: + The extracted text content, potentially chunked if the document is large. + """ + try: + # Check if file exists + import os + if not os.path.exists(file_path): + return f"Error: PDF file '{file_path}' does not exist." + + reader = PdfReader(file_path) + full_text = "\n".join([page.extract_text() or "" for page in reader.pages]) + + # If a question is provided, use retrieval to get relevant parts + if question and len(full_text) > 5000: # Only chunk if text is large + return process_large_document(full_text, question) + + return full_text + except Exception as e: + return f"Error reading PDF: {str(e)}" + +@tool(parse_docstring=True) +def read_text_from_docx(file_path: str, question: str = None) -> str: + """ + Extract text from a DOCX (Word) document, chunking large documents if needed. + + Args: + file_path: Path to the DOCX file. + question: Optional question to help retrieve relevant parts of long documents. + + Returns: + The extracted text, potentially chunked if the document is large. + """ + try: + # Check if file exists + import os + if not os.path.exists(file_path): + return f"Error: File '{file_path}' does not exist." + + try: + doc = docx.Document(file_path) + full_text = "\n".join([para.text for para in doc.paragraphs]) + except Exception as docx_err: + # Handle "Package not found" error specifically + if "Package not found" in str(docx_err): + # Try to read raw text if possible + try: + import zipfile + from xml.etree.ElementTree import XML + + WORD_NAMESPACE = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}' + PARA = WORD_NAMESPACE + 'p' + TEXT = WORD_NAMESPACE + 't' + + with zipfile.ZipFile(file_path) as docx_file: + with docx_file.open('word/document.xml') as document: + tree = XML(document.read()) + paragraphs = [] + for paragraph in tree.iter(PARA): + texts = [node.text for node in paragraph.iter(TEXT) if node.text] + if texts: + paragraphs.append(''.join(texts)) + full_text = '\n'.join(paragraphs) + except Exception as e: + return f"Error reading DOCX file: {str(e)}" + else: + return f"Error reading DOCX file: {str(docx_err)}" + + # If a question is provided, use retrieval to get relevant parts + if question and len(full_text) > 5000: # Only chunk if text is large + return process_large_document(full_text, question) + + return full_text + except Exception as e: + return f"Error reading DOCX file: {str(e)}" + + +@tool(parse_docstring=True) +def transcribe_audio(file_path: str) -> str: + """ + Transcribe speech from a local audio file to text. + + Args: + file_path: Path to the audio file. + + Returns: + Transcribed text using Google Web Speech API. + """ + try: + # Check if file exists + import os + if not os.path.exists(file_path): + return f"Error: Audio file '{file_path}' does not exist." + + # For non-WAV files, convert to WAV first + if not file_path.lower().endswith('.wav'): + try: + from pydub import AudioSegment + temp_wav = os.path.splitext(file_path)[0] + "_temp.wav" + audio = AudioSegment.from_file(file_path) + audio.export(temp_wav, format="wav") + file_path = temp_wav + except Exception as e: + return f"Failed to convert audio to WAV format: {str(e)}" + + recognizer = sr.Recognizer() + with sr.AudioFile(file_path) as src: + audio = recognizer.record(src) + return recognizer.recognize_google(audio) + except Exception as e: + if "Audio file could not be read" in str(e): + return f"Error: Audio format not supported. Try converting to WAV, MP3, OGG, or FLAC." + return f"Error transcribing audio: {str(e)}" + +@tool(parse_docstring=True) +def youtube_audio_processing(youtube_url: str) -> str: + """ + Download and transcribe audio from a YouTube video. + + Args: + youtube_url: URL of the YouTube video. + + Returns: + Transcription text extracted from the video's audio. + """ + yt = YouTube(youtube_url) + audio_stream = yt.streams.filter(only_audio=True).first() + out_file = audio_stream.download(output_path='.', filename='yt_audio') + wav_path = 'yt_audio.wav' + AudioSegment.from_file(out_file).export(wav_path, format='wav') + return transcribe_audio(wav_path) + +@tool(parse_docstring=True) +def extract_article_text(url: str, question: str = None) -> str: + """ + Download and extract the main article content from a webpage, chunking large articles if needed. + + Args: + url: The URL of the article to extract. + question: Optional question to help retrieve relevant parts of long articles. + + Returns: + The article's textual content, potentially chunked if large. + """ + try: + art = Article(url) + art.download() + art.parse() + full_text = art.text + + # If a question is provided, use retrieval to get relevant parts + if question and len(full_text) > 5000: # Only chunk if text is large + return process_large_document(full_text, question) + + return full_text + except Exception as e: + return f"Error extracting article: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────── Tool for ArXiv ──────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +@tool(parse_docstring=True) +def arvix_search(query: str) -> Dict[str, str]: + """ + Search for academic papers on ArXiv. + + Args: + query: The search term to look for in ArXiv. + + Returns: + A dictionary of up to 3 relevant paper entries in JSON format. + """ + papers = ArxivLoader(query=query, load_max_docs=3).load() + results = [] + for doc in papers: + try: + # Handle different metadata formats that might be returned + source = doc.metadata.get("source", "ArXiv") + doc_id = doc.metadata.get("id", doc.metadata.get("entry_id", "")) + result = { + "source": source, + "id": doc_id, + "summary": doc.page_content[:1000] if hasattr(doc, "page_content") else str(doc)[:1000], + } + results.append(result) + except Exception as e: + # Add error information as a fallback + results.append({ + "source": "ArXiv Error", + "id": "error", + "summary": f"Error processing paper: {str(e)}" + }) + + return {"arvix_results": json.dumps(results)} + +@tool(parse_docstring=True) +def answer_youtube_video_question( + youtube_url: str, + question: str, + chunk_size_seconds: int = 30 +) -> str: + """ + Answer a question based on a YouTube video's transcript. + + Args: + youtube_url: URL of the YouTube video. + question: The question to be answered using video content. + chunk_size_seconds: Duration of each transcript chunk. + + Returns: + The answer to the question generated from the video transcript. + """ + loader = YoutubeLoader.from_youtube_url( + youtube_url, + add_video_info=True, + transcript_format=TranscriptFormat.CHUNKS, + chunk_size_seconds=chunk_size_seconds, + ) + documents = loader.load() + embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-mpnet-base-v2') + vectorstore = FAISS.from_documents(documents, embeddings) + llm = RetryingChatGroq(model="deepseek-r1-distill-llama-70b", streaming=False) + qa_chain = RetrievalQA.from_chain_type(llm=llm, retriever=vectorstore.as_retriever()) + return qa_chain.run(question) + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────── Tool for Python REPL tool ──────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +python_repl = PythonREPLTool() + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────── Tool for Wiki ──────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +@tool(parse_docstring=True) +def wiki_search(query: str) -> str: + """ + Search Wikipedia for information on a given topic. + + Args: + query: The search term for Wikipedia. + + Returns: + A JSON string with up to 3 summary results. + """ + # load up to top_k pages + pages = WikipediaLoader(query=query, load_max_docs=3).load() + results: List[Dict] = [] + for doc in pages: + results.append({ + "source": doc.metadata["source"], + "page": doc.metadata.get("page", ""), + "content": doc.page_content[:1000], # truncate if you like + }) + return {"wiki_results": format_search_docs(results)} + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────── Tool for Image (understading, captioning & classification) ───────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +def _load_image(img_path: str, resize_to=(512, 512)) -> Image.Image: + """ + Load, verify, convert, and resize an image. + Raises ValueError on failure. + """ + if not img_path: + raise ValueError("No image path provided.") + try: + with Image.open(img_path) as img: + img.verify() + img = Image.open(img_path).convert("RGB") + img = img.resize(resize_to) + return img + except UnidentifiedImageError: + raise ValueError(f"File at {img_path} is not a valid image.") + except Exception as e: + raise ValueError(f"Failed to load image at {img_path}: {e}") + +def _encode_image_to_base64(img_path: str) -> str: + """ + Load an image, save optimized PNG into memory, and base64‑encode it. + """ + img = _load_image(img_path) + buffer = BytesIO() + img.save(buffer, format="PNG", optimize=True) + return base64.b64encode(buffer.getvalue()).decode("utf-8") + +@tool +def image_processing(prompt: str, img_path: str) -> str: + """Process an image using a vision LLM, with OCR fallback. + + Args: + prompt: Instruction or question related to the image. + img_path: Path to the image file. + + Returns: + The model's response or fallback OCR result. + """ + try: + import os + # Check if file exists + if not os.path.exists(img_path): + return f"Error: Image file '{img_path}' does not exist." + + try: + b64 = _encode_image_to_base64(img_path) + # Build a single markdown string with inline base64 image + md = f"{prompt}\n\n![](data:image/png;base64,{b64})" + message = HumanMessage(content=md) + # Use RetryingChatGroq with Llama 4 Maverick for vision + llm = RetryingChatGroq(model="meta-llama/llama-4-maverick-17b-128e-instruct", streaming=False, temperature=0) + try: + resp = llm.invoke([message]) + if hasattr(resp, 'content'): + return resp.content.strip() + elif isinstance(resp, str): + return resp.strip() + else: + # Handle dictionary or other response types + return str(resp) + except Exception as invoke_err: + print(f"[LLM invoke error] {invoke_err}") + # Fall back to OCR + raise ValueError("LLM invocation failed") + except Exception as llama_err: + print(f"[LLM vision failed] {llama_err}") + try: + img = _load_image(img_path) + return pytesseract.image_to_string(img).strip() + except Exception as ocr_err: + print(f"[OCR fallback failed] {ocr_err}") + return "Unable to process the image. Please check the file and try again." + except Exception as e: + # Catch any other errors + print(f"[image_processing error] {e}") + return f"Error processing image: {str(e)}" + +python_repl_tool = PythonREPLTool() + +@tool +def echo(text: str) -> str: + """Echo back the input text. + + Args: + text: The string to be echoed. + + Returns: + The same text that was provided as input. + """ + return text + +# ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Langgraph Agent ─────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + +# Build graph function +from langchain_core.tools import tool +from langchain.chat_models import ChatOpenAI +from langgraph.prebuilt.chat_agent_executor import create_react_agent, AgentState +from langchain.chat_models import init_chat_model + + + +def build_graph(provider: str = "groq"): + """Construct and compile the multi‑agent GAIA workflow StateGraph. + + This graph wires together three React‑style agents into a streamlined pipeline: + PerceptionAgent → ActionAgent → EvaluationAgent (with appropriate entry/exit points) + + The agents have the following responsibilities: + - PerceptionAgent: Handles web searches, Wikipedia, ArXiv, and image processing + - ActionAgent: Performs calculations, file operations, and code analysis + - EvaluationAgent: Reviews results and ensures the final answer is properly formatted + + Args: + provider: The name of the LLM provider. Must be "groq". + + Returns: + CompiledGraph: A compiled LangGraph state machine ready for invocation. + + Raises: + ValueError: If `provider` is anything other than "groq". + """ + try: + if provider != "groq": + raise ValueError("Invalid provider. Expected 'groq'.") + + # Initialize LLM + try: + logger.info("Initializing LLM with model: deepseek-r1-distill-llama-70b") + api_key = os.getenv("GROQ_API_KEY") + if not api_key or api_key == "default_key_or_placeholder": + logger.error("GROQ_API_KEY is not set or is using placeholder value") + raise ValueError("GROQ_API_KEY environment variable is not set properly. Please set a valid API key.") + + llm = RetryingChatGroq(model="deepseek-r1-distill-llama-70b", temperature=0) + logger.info("LLM initialized successfully") + except Exception as e: + logger.error(f"Error initializing LLM: {str(e)}") + raise + + # General system message for agents + sys_msg = SystemMessage(content=""" + You are a general AI assistant. I will ask you a question. Report your thoughts, and finish your answer with the following template: + + FINAL ANSWER: [YOUR FINAL ANSWER] + + YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma-separated list of numbers and/or strings. + + If you are asked for a number, don't use commas or units (e.g., $, %, kg) unless specified otherwise. + + If you are asked for a string, don't use articles (a, an, the), and don't use abbreviations (e.g., for states). + + If you are asked for a comma-separated list, apply the above rules to each element in the list. + """.strip()) + + # Special system message for the evaluation agent with stricter formatting requirements + eval_sys_msg = SystemMessage(content=""" + You are a specialized evaluation agent. Your job is to review the work done by other agents + and provide a final, properly formatted answer. + + IMPORTANT: You MUST ALWAYS format your answer using this exact template: + + FINAL ANSWER: [concise answer] + + Rules for formatting the answer: + 1. The answer must be extremely concise - use as few words as possible + 2. For numeric answers, provide only the number without units unless units are specifically requested + 3. For text answers, avoid articles (a, an, the) and unnecessary words + 4. For list answers, use a comma-separated format + 5. NEVER explain your reasoning in the FINAL ANSWER section + 6. NEVER skip the "FINAL ANSWER:" prefix + + Example good answers: + FINAL ANSWER: 42 + FINAL ANSWER: Paris + FINAL ANSWER: 1912, 1945, 1989 + + Example bad answers (don't do these): + - Based on my analysis, the answer is 42. + - I think it's Paris because that's the capital of France. + - The years were 1912, 1945, and 1989. + + Remember: ALWAYS include "FINAL ANSWER:" followed by the most concise answer possible. + """.strip()) + + # Define tools for each agent + logger.info("Setting up agent tools") + perception_tools = [web_search, wiki_search, news_article_search, arvix_search, image_processing, echo] + execution_tools = [ + multiply, add, subtract, divide, modulus, + download_file, process_excel_to_text, + read_text_from_pdf, read_text_from_docx, + transcribe_audio, youtube_audio_processing, + extract_article_text, answer_youtube_video_question, + python_repl_tool, analyze_code, read_code_file, analyze_python_function + ] + + # ─────────────── Agent Creation ─────────────── + logger.info("Creating agents") + try: + # Create agents with proper error handling + PerceptionAgent = create_react_agent( + model=llm, + tools=perception_tools, + prompt=sys_msg, + state_schema=AgentState, + name="PerceptionAgent" + ) + logger.info("Created PerceptionAgent successfully") + + # Combined Planning and Execution agent for better efficiency + ActionAgent = create_react_agent( + model=llm, + tools=execution_tools, # Has access to all execution tools + prompt=sys_msg, + state_schema=AgentState, + name="ActionAgent" + ) + logger.info("Created ActionAgent successfully") + + # Evaluation agent with stricter prompt + EvaluationAgent = create_react_agent( + model=llm, + tools=[], # No tools needed for evaluation + prompt=eval_sys_msg, # Use the specialized evaluation prompt + state_schema=AgentState, + name="EvaluationAgent" + ) + logger.info("Created EvaluationAgent successfully") + except Exception as e: + logger.error(f"Error creating agent: {str(e)}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + raise + + # Build the StateGraph + logger.info("Building StateGraph") + try: + builder = StateGraph(AgentState) + + # Add agent nodes first + builder.add_node("PerceptionAgent", PerceptionAgent) + builder.add_node("ActionAgent", ActionAgent) + builder.add_node("EvaluationAgent", EvaluationAgent) + + # Define the flow with a starting edge + builder.set_entry_point("PerceptionAgent") + + # Add the edges for the simpler linear flow + builder.add_edge("PerceptionAgent", "ActionAgent") + builder.add_edge("ActionAgent", "EvaluationAgent") + + # Set EvaluationAgent as the end node + builder.set_finish_point("EvaluationAgent") + + logger.info("Compiling StateGraph") + return builder.compile() + except Exception as e: + logger.error(f"Error building graph: {str(e)}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + raise + except Exception as e: + logger.error(f"Overall error in build_graph: {str(e)}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + raise + +def get_final_answer(text): + """Extract just the FINAL ANSWER from the model's response. + + Args: + text: The full text response from the LLM + + Returns: + str: The extracted answer without the "FINAL ANSWER:" prefix + """ + # Log the raw text for debugging if needed + logger.debug(f"Extracting answer from: {text[:200]}...") + + if not text: + logger.warning("Empty response received") + return "No answer provided." + + # Method 1: Look for "FINAL ANSWER:" with most comprehensive pattern matching + pattern = r'(?:^|\n)FINAL ANSWER:\s*(.*?)(?:\n\s*$|$)' + match = re.search(pattern, text, re.DOTALL | re.IGNORECASE) + if match: + # Return just the answer part, cleaned up + logger.debug("Found answer using pattern 1") + return match.group(1).strip() + + # Method 2: Try looking for variations on the final answer format + for variant in ["FINAL ANSWER:", "FINAL_ANSWER:", "Final Answer:", "Answer:"]: + lines = text.split('\n') + for i, line in enumerate(reversed(lines)): + if variant in line: + # Extract everything after the variant text + logger.debug(f"Found answer using variant: {variant}") + answer = line[line.find(variant) + len(variant):].strip() + if answer: + return answer + # If the answer is on the next line, return that + if i > 0: + next_line = lines[len(lines) - i] + if next_line.strip(): + return next_line.strip() + + # Method 3: Look for phrases that suggest an answer + for phrase in ["The answer is", "The result is", "We get", "Therefore,", "In conclusion,"]: + phrase_pos = text.find(phrase) + if phrase_pos != -1: + # Try to extract everything after the phrase until the end of the sentence + sentence_end = text.find(".", phrase_pos) + if sentence_end != -1: + logger.debug(f"Found answer using phrase: {phrase}") + return text[phrase_pos + len(phrase):sentence_end].strip() + + # Method 4: Fall back to taking the last paragraph with actual content + paragraphs = text.strip().split('\n\n') + for para in reversed(paragraphs): + para = para.strip() + if para and not para.startswith("I ") and not para.lower().startswith("to "): + logger.debug("Using last meaningful paragraph") + # If paragraph is very long, try to extract a concise answer + if len(para) > 100: + sentences = re.split(r'[.!?]', para) + for sentence in reversed(sentences): + sent = sentence.strip() + if sent and len(sent) > 5 and not sent.startswith("I "): + return sent + return para + + # Method 5: Last resort - just return the last line with content + lines = text.strip().split('\n') + for line in reversed(lines): + line = line.strip() + if line and len(line) > 3: + logger.debug("Using last line with content") + return line + + # If everything fails, warn and return the truncated response + logger.warning("Could not find a properly formatted answer") + return text[:100] + "..." if len(text) > 100 else text + +# test +if __name__ == "__main__": + question = "When was a picture of St. Thomas Aquinas first added to the Wikipedia page on the Principle of double effect?" + # Build the graph + graph = build_graph(provider="groq") + # Run the graph + messages = [HumanMessage(content=question)] + messages = graph.invoke({"messages": messages}) + for m in messages["messages"]: + m.pretty_print() + +# ─────────────────────────────────────────────── Tool for Code Analysis ─────────────────────────────────────────────────────────────── +@tool +def analyze_code(code_string: str) -> str: + """Analyze a string of code to understand its structure, functionality, and potential issues. + + Args: + code_string: The code to analyze as a string. + + Returns: + A structured analysis of the code including functions, classes, and key operations. + """ + try: + import ast + + # Try to parse with Python's AST module + try: + parsed = ast.parse(code_string) + + # Extract functions and classes + functions = [node.name for node in ast.walk(parsed) if isinstance(node, ast.FunctionDef)] + classes = [node.name for node in ast.walk(parsed) if isinstance(node, ast.ClassDef)] + imports = [node.names[0].name for node in ast.walk(parsed) if isinstance(node, ast.Import)] + imports.extend([f"{node.module}.{name.name}" if node.module else name.name + for node in ast.walk(parsed) if isinstance(node, ast.ImportFrom) + for name in node.names]) + + # Count various node types for complexity assessment + num_loops = len([node for node in ast.walk(parsed) + if isinstance(node, (ast.For, ast.While))]) + num_conditionals = len([node for node in ast.walk(parsed) + if isinstance(node, (ast.If, ast.IfExp))]) + + analysis = { + "language": "Python", + "functions": functions, + "classes": classes, + "imports": imports, + "complexity": { + "functions": len(functions), + "classes": len(classes), + "loops": num_loops, + "conditionals": num_conditionals + } + } + return str(analysis) + except SyntaxError: + # If not valid Python, try some simple pattern matching + if "{" in code_string and "}" in code_string: + if "function" in code_string or "=>" in code_string: + language = "JavaScript/TypeScript" + elif "func" in code_string or "struct" in code_string: + language = "Go or Rust" + elif "public" in code_string or "private" in code_string or "class" in code_string: + language = "Java/C#/C++" + else: + language = "Unknown C-like language" + elif "<" in code_string and ">" in code_string and ("/>" in code_string or " str: + """Read a code file and return its contents with proper syntax detection. + + Args: + file_path: Path to the code file. + + Returns: + The file contents and detected language. + """ + try: + # Check if file exists + import os + if not os.path.exists(file_path): + return f"Error: File '{file_path}' does not exist." + + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Try to detect language from extension + ext = os.path.splitext(file_path)[1].lower() + + language_map = { + '.py': 'Python', + '.js': 'JavaScript', + '.ts': 'TypeScript', + '.html': 'HTML', + '.css': 'CSS', + '.java': 'Java', + '.c': 'C', + '.cpp': 'C++', + '.cs': 'C#', + '.go': 'Go', + '.rs': 'Rust', + '.php': 'PHP', + '.rb': 'Ruby', + '.sh': 'Shell', + '.bat': 'Batch', + '.ps1': 'PowerShell', + '.sql': 'SQL', + '.json': 'JSON', + '.xml': 'XML', + '.yaml': 'YAML', + '.yml': 'YAML', + } + + language = language_map.get(ext, 'Unknown') + + return f"File content ({language}):\n\n{content}" + except Exception as e: + return f"Error reading file: {str(e)}" + +@tool +def analyze_python_function(function_name: str, code_string: str) -> str: + """Extract and analyze a specific function from Python code. + + Args: + function_name: The name of the function to analyze. + code_string: The complete code containing the function. + + Returns: + Analysis of the function including parameters, return type, and docstring. + """ + try: + import ast + import inspect + from types import CodeType, FunctionType + + # Parse the code string + parsed = ast.parse(code_string) + + # Find the function definition + function_def = None + for node in ast.walk(parsed): + if isinstance(node, ast.FunctionDef) and node.name == function_name: + function_def = node + break + + if not function_def: + return f"Function '{function_name}' not found in the provided code." + + # Extract parameters + params = [] + for arg in function_def.args.args: + param_name = arg.arg + # Get annotation if it exists + if arg.annotation: + if isinstance(arg.annotation, ast.Name): + param_type = arg.annotation.id + elif isinstance(arg.annotation, ast.Attribute): + param_type = f"{arg.annotation.value.id}.{arg.annotation.attr}" + else: + param_type = "complex_type" + params.append(f"{param_name}: {param_type}") + else: + params.append(param_name) + + # Extract return type if it exists + return_type = None + if function_def.returns: + if isinstance(function_def.returns, ast.Name): + return_type = function_def.returns.id + elif isinstance(function_def.returns, ast.Attribute): + return_type = f"{function_def.returns.value.id}.{function_def.returns.attr}" + else: + return_type = "complex_return_type" + + # Extract docstring + docstring = ast.get_docstring(function_def) + + # Create a summary + summary = { + "function_name": function_name, + "parameters": params, + "return_type": return_type, + "docstring": docstring, + "decorators": [d.id if isinstance(d, ast.Name) else "complex_decorator" for d in function_def.decorator_list], + "line_count": len(function_def.body) + } + + # Create a more explicit string representation that ensures key terms are included + result = f"Function '{function_name}' analysis:\n" + result += f"- Parameters: {', '.join(params)}\n" + result += f"- Return type: {return_type or 'None specified'}\n" + result += f"- Docstring: {docstring or 'None'}\n" + result += f"- Line count: {len(function_def.body)}" + + return result + except Exception as e: + return f"Error analyzing function: {str(e)}" + +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +# ─────────────────────────────────────────────── Tool for News Article Retrieval ────────────────────────────────────────────────────────────────────── +# ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +@tool +def news_article_search(query: str, top_k: int = 3) -> Dict[str, str]: + """Search for and retrieve news articles with robust error handling for news sites. + + Args: + query: The news topic or keywords to search for. + top_k: Maximum number of articles to retrieve. + + Returns: + A dictionary with search results formatted as XML-like document entries. + """ + # First, get URLs from DuckDuckGo with "news" focus + results = [] + news_sources = [ + "bbc.com", "reuters.com", "apnews.com", "nasa.gov", + "space.com", "universetoday.com", "nature.com", "science.org", + "scientificamerican.com", "nytimes.com", "theguardian.com" + ] + + # Find news from reliable sources + try: + with DDGS() as ddgs: + search_query = f"{query} site:{' OR site:'.join(news_sources)}" + for hit in ddgs.text(search_query, safesearch="On", max_results=top_k*2): + url = hit.get("href") or hit.get("url", "") + if not url: + continue + + # Add the search snippet first as a fallback + result = { + "source": url, + "page": "", + "content": hit.get("body", "")[:250], + "title": hit.get("title", "") + } + + # Try to get better content via a more robust method + try: + headers = { + "User-Agent": random.choice([ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" + ]), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Referer": "https://www.google.com/", + "DNT": "1", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1" + } + + # Add a short delay between requests + time.sleep(1 + random.random()) + + # Try to use newspaper3k for more reliable article extraction + from newspaper import Article + article = Article(url) + article.download() + article.parse() + + # If we got meaningful content, update the result + if article.text and len(article.text) > 100: + # Get a summary - first paragraph + some highlights + paragraphs = article.text.split('\n\n') + first_para = paragraphs[0] if paragraphs else "" + summary = first_para[:300] + if len(paragraphs) > 1: + summary += "... " + paragraphs[1][:200] + + result["content"] = summary + if article.title: + result["title"] = article.title + + except Exception as article_err: + logger.warning(f"Article extraction failed for {url}: {article_err}") + # Fallback to simple requests-based extraction + try: + resp = requests.get(url, timeout=12, headers=headers) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # Try to get main content + main_content = soup.find('main') or soup.find('article') or soup.find('div', class_='content') + + if main_content: + content = " ".join(main_content.get_text(separator=" ", strip=True).split()[:250]) + result["content"] = content + except Exception as req_err: + logger.warning(f"Fallback extraction failed for {url}: {req_err}") + # Keep the original snippet as fallback + + results.append(result) + if len(results) >= top_k: + break + + except Exception as e: + logger.error(f"News search failed: {e}") + return format_search_docs([{ + "source": "Error", + "page": "", + "content": f"Failed to retrieve news articles for '{query}': {str(e)}" + }]) + + if not results: + # Fallback to regular web search + logger.info(f"No news results found, falling back to web_search for {query}") + return web_search(query, top_k) + + return format_search_docs(results[:top_k]) + +# ───────────────────────────────────────────────────────────── Document Chunking Utilities ────────────────────────────────────────────────────────── +def chunk_document(text: str, chunk_size: int = 1000, overlap: int = 100) -> List[str]: + """ + Split a large document into smaller chunks with overlap to maintain context across chunks. + + Args: + text: The document text to split into chunks + chunk_size: Maximum size of each chunk in characters + overlap: Number of characters to overlap between chunks + + Returns: + List of text chunks + """ + # If text is smaller than chunk_size, return it as is + if len(text) <= chunk_size: + return [text] + + chunks = [] + start = 0 + + while start < len(text): + # Get chunk with overlap + end = min(start + chunk_size, len(text)) + + # Try to find sentence boundary for cleaner breaks + if end < len(text): + # Look for sentence endings: period, question mark, or exclamation followed by space + for sentence_end in ['. ', '? ', '! ']: + last_period = text[start:end].rfind(sentence_end) + if last_period != -1: + end = start + last_period + 2 # +2 to include the period and space + break + + # Add chunk to list + chunks.append(text[start:end]) + + # Move start position, accounting for overlap + start = end - overlap if end < len(text) else len(text) + + return chunks + +# Document processing utility that uses chunking +def process_large_document(text: str, question: str, llm=None) -> str: + """ + Process a large document by chunking it and using retrieval to find relevant parts. + + Args: + text: The document text to process + question: The question being asked about the document + llm: Optional language model to use (defaults to agent's LLM) + + Returns: + Summarized answer based on relevant chunks + """ + if not llm: + llm = RetryingChatGroq(model="deepseek-r1-distill-llama-70b", streaming=False, temperature=0) + + # Split document into chunks + chunks = chunk_document(text) + + # If document is small enough, don't bother with retrieval + if len(chunks) <= 1: + return text + + # For larger documents, create embeddings to find relevant chunks + try: + from langchain_community.embeddings import HuggingFaceEmbeddings + from langchain.vectorstores import FAISS + from langchain.schema import Document + + # Create documents with chunk content + documents = [Document(page_content=chunk, metadata={"chunk_id": i}) for i, chunk in enumerate(chunks)] + + # Create embeddings and vector store + embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") + vectorstore = FAISS.from_documents(documents, embeddings) + + # Get most relevant chunks + relevant_chunks = vectorstore.similarity_search(question, k=2) # Get top 2 most relevant chunks + + # Join the relevant chunks + relevant_text = "\n\n".join([doc.page_content for doc in relevant_chunks]) + + # Option 1: Return relevant chunks directly + return relevant_text + + # Option 2: Summarize with LLM (commented out for now) + # prompt = f"Using only the following information, answer the question: '{question}'\n\nInformation:\n{relevant_text}" + # response = llm.invoke([HumanMessage(content=prompt)]) + # return response.content + + except Exception as e: + # Fall back to first chunk if retrieval fails + logger.warning(f"Retrieval failed: {e}. Falling back to first chunk.") + return chunks[0] \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/agent2.py b/scratch_VLM/scratch_agent/agent2.py new file mode 100644 index 0000000000000000000000000000000000000000..654c793cabf087d0af8eeae5e14551c03f7bd465 --- /dev/null +++ b/scratch_VLM/scratch_agent/agent2.py @@ -0,0 +1,259 @@ +import os +from dotenv import load_dotenv + +from langchain_core.tools import tool +from langgraph.prebuilt import tools_condition, ToolNode +from langgraph.graph import START, StateGraph, MessagesState + +from langchain_community.tools.tavily_search import TavilySearchResults +from langchain_community.document_loaders import WikipediaLoader, ArxivLoader +from langchain_core.messages import SystemMessage, HumanMessage +from langchain_community.vectorstores import FAISS +from langchain_huggingface import HuggingFaceEmbeddings +from langchain_groq import ChatGroq +from langchain.tools.retriever import create_retriever_tool + +"""LangGraph Agent""" +import os +from dotenv import load_dotenv +from langgraph.graph import START, StateGraph, MessagesState +from langgraph.prebuilt import tools_condition +from langgraph.prebuilt import ToolNode +from langchain_google_genai import ChatGoogleGenerativeAI +from langchain_groq import ChatGroq +from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFaceEmbeddings +from langchain_community.tools.tavily_search import TavilySearchResults +from langchain_community.document_loaders import WikipediaLoader +from langchain_community.document_loaders import ArxivLoader +from langchain_community.vectorstores import SupabaseVectorStore,FAISS +from langchain_core.messages import SystemMessage, HumanMessage +from langchain_core.tools import tool +from langchain.tools.retriever import create_retriever_tool +#from supabase.client import Client, create_client + +# ────────────────────────────────────────────────────────── +# ENV +# ────────────────────────────────────────────────────────── +load_dotenv() +# API Keys from .env file +os.environ.setdefault("OPENAI_API_KEY", "") # Set your own key or env var +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") +os.environ["MISTRAL_API_KEY"] = os.getenv("MISTRAL_API_KEY", "default_key_or_placeholder") + +# Tavily API Key +TAVILY_API_KEY = os.getenv("TAVILY_API_KEY", "default_key_or_placeholder") +_forbidden = ["nsfw", "porn", "sex", "explicit"] +_playwright_available = True # set False to disable Playwright + +embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") # dim = 768 + +@tool +def multiply(a: int, b: int) -> int: + """Multiply two numbers. + + Args: + a: first int + b: second int + """ + return a * b + +@tool +def add(a: int, b: int) -> int: + """Add two numbers. + + Args: + a: first int + b: second int + """ + return a + b + +@tool +def subtract(a: int, b: int) -> int: + """Subtract two numbers. + + Args: + a: first int + b: second int + """ + return a - b + +@tool +def divide(a: int, b: int) -> int: + """Divide two numbers. + + Args: + a: first int + b: second int + """ + if b == 0: + raise ValueError("Cannot divide by zero.") + return a / b + +@tool +def modulus(a: int, b: int) -> int: + """Get the modulus of two numbers. + + Args: + a: first int + b: second int + """ + return a % b + +@tool +def wiki_search(query: str) -> str: + """Search Wikipedia for a query and return maximum 2 results. + + Args: + query: The search query.""" + search_docs = WikipediaLoader(query=query, load_max_docs=2).load() + formatted_search_docs = "\n\n---\n\n".join( + [ + f'\n{doc.page_content}\n' + for doc in search_docs + ]) + return {"wiki_results": formatted_search_docs} + +@tool +def web_search(query: str) -> str: + """Search Tavily for a query and return maximum 3 results. + + Args: + query: The search query.""" + search_docs = TavilySearchResults(max_results=3).invoke(query=query) + formatted_search_docs = "\n\n---\n\n".join( + [ + f'\n{doc.page_content}\n' + for doc in search_docs + ]) + return {"web_results": formatted_search_docs} + +@tool +def arvix_search(query: str) -> str: + """Search Arxiv for a query and return maximum 3 result. + + Args: + query: The search query.""" + search_docs = ArxivLoader(query=query, load_max_docs=3).load() + formatted_search_docs = "\n\n---\n\n".join( + [ + f'\n{doc.page_content[:1000]}\n' + for doc in search_docs + ]) + return {"arvix_results": formatted_search_docs} + + + +# load the system prompt from the file +with open("system_prompt.txt", "r", encoding="utf-8") as f: + system_prompt = f.read() + +# System message +sys_msg = SystemMessage(content=system_prompt) + +# build a retriever +embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") # dim=768 + +INDEX_PATH = "faiss_index" +if os.path.exists(INDEX_PATH): + vector_store = FAISS.load_local(INDEX_PATH, embeddings, allow_dangerous_deserialization=True) +else: + vector_store = FAISS.from_texts(["__init__"], embeddings) + vector_store.save_local(INDEX_PATH) + +create_retriever_tool = create_retriever_tool( + retriever=vector_store.as_retriever(), + name="Question Search", + description="A tool to retrieve similar questions from a local FAISS vector store." +) + + + +tools = [ + multiply, + add, + subtract, + divide, + modulus, + wiki_search, + web_search, + arvix_search, +] + +# Build graph function +def build_graph(provider: str = "groq"): + """Build the graph""" + # Load environment variables from .env file + if provider == "google": + # Google Gemini + llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0) + elif provider == "groq": + # Groq https://console.groq.com/docs/models + try: + llm = ChatGroq(model="deepseek-r1-distill-llama-70b", temperature=0) # optional : qwen-qwq-32b gemma2-9b-it + except Exception as e: + print(f"Error initializing Groq: {str(e)}") + raise + elif provider == "huggingface": + # TODO: Add huggingface endpoint + llm = ChatHuggingFace( + llm=HuggingFaceEndpoint( + url="https://api-inference.huggingface.co/models/Meta-DeepLearning/llama-2-7b-chat-hf", + temperature=0, + ), + ) + else: + raise ValueError("Invalid provider. Choose 'google', 'groq' or 'huggingface'.") + + # Bind tools to LLM + llm_with_tools = llm.bind_tools(tools) + + # Node + def assistant(state: MessagesState): + """Assistant node""" + try: + if not state["messages"] or not state["messages"][-1].content: + raise ValueError("Empty message content") + return {"messages": [llm_with_tools.invoke(state["messages"])]} + except Exception as e: + print(f"Error in assistant node: {str(e)}") + raise + + def retriever(state: MessagesState): + """Retriever node""" + try: + if not state["messages"] or not state["messages"][0].content: + raise ValueError("Empty message content") + similar_question = vector_store.similarity_search(state["messages"][0].content) + example_msg = HumanMessage( + content=f"Here I provide a similar question and answer for reference: \n\n{similar_question[0].page_content}", + ) + return {"messages": [sys_msg] + state["messages"] + [example_msg]} + except Exception as e: + print(f"Error in retriever node: {str(e)}") + raise + + builder = StateGraph(MessagesState) + builder.add_node("retriever", retriever) + builder.add_node("assistant", assistant) + builder.add_node("tools", ToolNode(tools)) + builder.add_edge(START, "retriever") + builder.add_edge("retriever", "assistant") + builder.add_conditional_edges( + "assistant", + tools_condition, + ) + builder.add_edge("tools", "assistant") + + # Compile graph + return builder.compile() + +# test +if __name__ == "__main__": + question = "When was a picture of St. Thomas Aquinas first added to the Wikipedia page on the Principle of double effect?" + # Build the graph + graph = build_graph(provider="groq") + # Run the graph + messages = [HumanMessage(content=question)] + messages = graph.invoke({"messages": messages}) + for m in messages["messages"]: + m.pretty_print() \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/app.py b/scratch_VLM/scratch_agent/app.py new file mode 100644 index 0000000000000000000000000000000000000000..046336e8839ae62936f41129c5d854320663b84d --- /dev/null +++ b/scratch_VLM/scratch_agent/app.py @@ -0,0 +1,997 @@ +from flask import Flask, request, jsonify, render_template, send_from_directory +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +from PIL import Image +import os, json, re +import shutil +import uuid +from langgraph.graph import StateGraph, END +import logging +from typing import Dict, TypedDict, Optional + +# --- Configure logging --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +# --- LLM / Vision model setup --- +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + logger.critical("GROQ_API_KEY environment variable is not set or invalid. Please set it to proceed.") + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +llm = ChatGroq( + model="deepseek-r1-distill-llama-70b", + temperature=0.0, +) + +# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema. When generating or modifying the `blocks` section, pay extremely close attention to the following: + +**Scratch Target and Block Schema Rules:** +1. **Target Structure:** + * Every target (Stage or Sprite) must have an `objName` property. For the Stage, it's typically `"objName": "Stage"`. For sprites, it's their name (e.g., `"objName": "Cat"`). + * `variables` dictionary: This dictionary maps unique variable IDs to arrays `[variable_name, initial_value]`. + Example: `"variables": { "myVarId123": ["score", 0] }` + +2. **Block Structure:** Every block must have `opcode`, `parent`, `next`, `inputs`, `fields`, `shadow`, `topLevel`. + * `parent`: ID of the block it's stacked on (or `null` for top-level). + * `next`: ID of the block directly below it (or `null` for end of stack). + * `topLevel`: `true` if it's a hat block or a standalone block, `false` otherwise. + * `shadow`: `true` if it's a shadow block (e.g., a default number input), `false` otherwise. + +3. **`inputs` vs. `fields`:** This is critical. + * **`inputs`**: Used for blocks or values *plugged into* a block (e.g., condition for an `if` block, target for `go to`). Values are **arrays**. + * `[1, ]`: Represents a Reporter or Boolean block *plugged in* (e.g., `operator_equals`). + * `[1, [, ""]]`: For a shadow block with a literal value (e.g., `[1, ["num", "10"]]` for a number input). The `type` can be "num", "str", "bool", "colour", etc. + * `[2, ]`: For a C-block's substack (e.g., `SUBSTACK` in `control_if`). + * **`fields`**: Used for dropdown menus or direct internal values *within* a block (e.g., key selected in `when key pressed`, variable name in `set variable`). Values are always **arrays** `["", null]`. + * Example for `event_whenkeypressed`: `"KEY": ["space", null]` + * Example for `data_setvariableto`: `"VARIABLE": ["score", null]` + +4. **Unique IDs:** All block IDs, variable IDs, and list IDs must be unique strings (e.g., "myBlock123", "myVarId456"). Do NOT use placeholder strings like "block_id_here". + +5. **No Nested `blocks` Dictionary:** The `blocks` dictionary should only appear once per `target` (sprite/stage). Do NOT nest a `blocks` dictionary inside an individual block definition. Blocks that are part of a substack (like inside an `if` or `forever` loop) are linked via the `SUBSTACK` input, where the input value is `[2, "ID_of_first_block_in_substack"]`. + +6. **Asset Properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY` should be correct for costumes/sounds. + +**When responding, you must:** +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, field, and variable declaration strictly follows the Scratch 3.0 schema and the rules above. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" + +agent = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT +) + +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + backdrops = [] + sprites = [] + try: + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + logger.info("Successfully listed static assets.") + except Exception as e: + logger.error(f"Error listing static assets: {e}") + return jsonify({"error": "Failed to list assets"}), 500 + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + logger.warning(f"Could not determine image dimensions for {filename}: {e}. Setting center to 0,0.") + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- New endpoint to fetch project.json --- +@app.route("/get_project/", methods=["GET"]) +def get_project(project_id): + project_folder = os.path.join("generated_projects", project_id) + project_json_path = os.path.join(project_folder, "project.json") + + try: + if os.path.exists(project_json_path): + logger.info(f"Serving project.json for project ID: {project_id}") + return send_from_directory(project_folder, "project.json", as_attachment=True, download_name=f"{project_id}.json") + else: + logger.warning(f"Project JSON not found for ID: {project_id}") + return jsonify({"error": "Project not found"}), 404 + except Exception as e: + logger.error(f"Error serving project.json for ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve project"}), 500 + +# --- New endpoint to fetch assets --- +@app.route("/get_asset//", methods=["GET"]) +def get_asset(project_id, filename): + project_folder = os.path.join("generated_projects", project_id) + asset_path = os.path.join(project_folder, filename) + + try: + if os.path.exists(asset_path): + logger.info(f"Serving asset '{filename}' for project ID: {project_id}") + return send_from_directory(project_folder, filename) + else: + logger.warning(f"Asset '{filename}' not found for project ID: {project_id}") + return jsonify({"error": "Asset not found"}), 404 + except Exception as e: + logger.error(f"Error serving asset '{filename}' for project ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve asset"}), 500 + + +### LangGraph Workflow Definition + +# Define a state for your graph using TypedDict +class GameState(TypedDict): + project_json: dict + description: str + project_id: str + sprite_initial_positions: dict # Add this as well, as it's part of your state + action_plan: Optional[Dict] + behavior_plan: Optional[Dict] + +# Helper function to extract JSON from LLM response +def extract_json_from_llm_response(raw_response: str) -> dict: + m = re.search(r"```json\n(.+?)```", raw_response, re.S) + body = m.group(1) if m else raw_response + try: + json_data = json.loads(body) + logger.debug("Successfully extracted and parsed JSON from LLM response.") + return json_data + except json.JSONDecodeError as e: + logger.error(f"LLM did not return valid JSON. Error: {e}\nRaw response: {raw_response}") + raise ValueError(f"LLM did not return valid JSON: {e}\nRaw response: {raw_response}") + except Exception as e: + logger.error(f"An unexpected error occurred during JSON extraction: {e}\nRaw response: {raw_response}") + raise +# Helper function to update project JSON with sprite positions +import copy +def update_project_with_sprite_positions(project_json: dict, sprite_positions: dict) -> dict: + """ + Update the 'x' and 'y' coordinates of sprites in the Scratch project JSON. + + Args: + project_json (dict): Original Scratch project JSON. + sprite_positions (dict): Dict mapping sprite names to {'x': int, 'y': int}. + + Returns: + dict: Updated project JSON with new sprite positions. + """ + updated_project = copy.deepcopy(project_json) + + for target in updated_project.get("targets", []): + if not target.get("isStage", False): + sprite_name = target.get("name") + if sprite_name in sprite_positions: + pos = sprite_positions[sprite_name] + if "x" in pos and "y" in pos: + target["x"] = pos["x"] + target["y"] = pos["y"] + + return updated_project + +# +# --- Global variable for the block catalog --- +ALL_SCRATCH_BLOCKS_CATALOG = {} +BLOCK_CATALOG_PATH = "block_content.json" # Define the path to your JSON file + +# Helper function to load the block catalog from a JSON file +def _load_block_catalog(file_path: str) -> Dict: + """Loads the Scratch block catalog from a specified JSON file.""" + try: + with open(file_path, 'r') as f: + catalog = json.load(f) + logger.info(f"Successfully loaded block catalog from {file_path}") + return catalog + except FileNotFoundError: + logger.error(f"Error: Block catalog file not found at {file_path}") + # Return an empty dict or raise an error, depending on desired behavior + return {} + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON from {file_path}: {e}") + return {} + except Exception as e: + logger.error(f"An unexpected error occurred while loading {file_path}: {e}") + return {} + + +# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally +ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH) + +# Helper function to generate a unique block ID +def generate_block_id(): + """Generates a short, unique ID for a Scratch block.""" + return str(uuid.uuid4())[:10].replace('-', '') # Shorten for readability, ensure uniqueness + +# Placeholder for your extract_json_from_llm_response function +def extract_json_from_llm_response(response_string: str) -> Dict: + """ + Extracts a JSON object from an LLM response string. + Assumes the JSON is enclosed within triple backticks (```json ... ```). + """ + try: + # Find the start and end of the JSON block + start_marker = "```json" + end_marker = "```" + start_index = response_string.find(start_marker) + end_index = response_string.rfind(end_marker) + + if start_index != -1 and end_index != -1 and start_index < end_index: + # Extract the substring between the markers + json_str = response_string[start_index + len(start_marker):end_index].strip() + return json.loads(json_str) + else: + # If markers are not found, try to parse the whole string as JSON + # This handles cases where the LLM might omit markers + logger.warning("JSON markers not found in LLM response. Attempting to parse raw string.") + return json.loads(response_string.strip()) + except json.JSONDecodeError as e: + logger.error(f"Failed to decode JSON from LLM response: {e}") + logger.error(f"Raw response: {response_string}") + raise ValueError("Invalid JSON response from LLM") + except Exception as e: + logger.error(f"An unexpected error occurred in extract_json_from_llm_response: {e}") + raise + + +# Node 1: Analysis User Query and Initial Positions +def parse_query_and_set_initial_positions(state: GameState): + logger.info("--- Running ParseQueryNode ---") + + llm_query_prompt = ( + f"Based on the user's game description: '{state['description']}', " + f"and the current Scratch project JSON below, " + f"determine the most appropriate initial 'x' and 'y' coordinates for each sprite. " + f"Return ONLY a JSON object with a single key 'sprite_initial_positions' mapping sprite names to their {{'x': int, 'y': int}} coordinates.\n\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Example Json output:\n" + f"{{\n" + f" \"sprite_initial_positions\": {{\n" + f" \"Sprite1\": {{\"x\": -160, \"y\": -110}},\n" + f" \"Sprite2\": {{\"x\": 240, \"y\": -135}}\n" + f" }}\n" + f"}}" + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_query_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM:", raw_response) + + updated_data = extract_json_from_llm_response(raw_response) + sprite_positions = updated_data.get("sprite_initial_positions", {}) + + new_project_json = update_project_with_sprite_positions(state["project_json"], sprite_positions) + print("Updated project JSON with sprite positions:", json.dumps(new_project_json, indent=2)) + return {"project_json": new_project_json, "sprite_initial_positions": sprite_positions} + #return {"project_json": new_project_json} + + except Exception as e: + logger.error(f"Error in ParseQueryNode: {e}") + raise + +# Node 3: Sprite Action Plan Builder +def plan_sprite_actions(state: GameState): + logger.info("--- Running PlanSpriteActionsNode ---") + + + # Assuming 'agent' is defined globally or passed into this scope + planning_prompt = ( + f"You are an AI assistant tasked with planning Scratch 3.0 block code for a game. " + f"The game description is: '{state['description']}'.\n\n" + f"Here are the sprites currently in the project: {', '.join(target['name'] for target in state['project_json']['targets'] if not target['isStage'] if state['project_json']['targets']) or 'None'}.\n" + f"Initial positions: {json.dumps(state.get('sprite_initial_positions', {}), indent=2)}\n\n" + f"Consider the main actions and interactions required for each sprite. " + f"Think step-by-step about what each sprite needs to *do*, *when* it needs to do it (events), " + f"and if any actions need to *repeat* or depend on *conditions*.\n\n" + f"Propose a high-level action flow for each sprite in the following JSON format. " + f"Do NOT generate Scratch block JSON yet. Only describe the logic using natural language or simplified pseudo-code.\n\n" + f"Example format:\n" + f"```json\n" + f"{{\n" + f" \"action_overall_flow\": {{\n" + f" \"Sprite1\": {{\n" + f" \"description\": \"Main character actions\",\n" + f" \"plans\": [\n" + f" {{\n" + f" \"event\": \"when flag clicked\",\n" + f" \"logic\": \"forever loop: move 10 steps, if on edge bounce\"\n" + f" }},\n" + f" {{\n" + f" \"event\": \"when space key pressed\",\n" + f" \"logic\": \"change y by 10, wait 0.1 seconds, change y by -10\"\n" + f" }}\n" + f" ]\n" + f" }},\n" + f" \"Ball\": {{\n" + f" \"description\": \"Projectile movement\",\n" + f" \"plans\": [\n" + f" {{\n" + f" \"event\": \"when I start as a clone\",\n" + f" \"logic\": \"glide 1 sec to random position, if touching Sprite1 then stop this script\"\n" + f" }}\n" + f" ]\n" + f" }}\n" + f" }}\n" + f"}}\n" + f"```\n\n" + f"Return ONLY the JSON object for the action overall flow." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM [PlanSpriteActionsNode]:", raw_response) + action_plan = extract_json_from_llm_response(raw_response) + logger.info("Sprite action plan generated by PlanSpriteActionsNode.") + return {"action_plan": action_plan} + except Exception as e: + logger.error(f"Error in PlanSpriteActionsNode: {e}") + raise + +# Node 4: Action Node Builder +def build_action_nodes(state: GameState): + logger.info("--- Running ActionNodeBuilder ---") + + # Assuming 'agent' is defined globally or passed into this scope + # from your_module import agent + global agent # Declare agent as global if it's imported in the main script scope + + action_plan = state.get("action_plan", {}) + if not action_plan: + raise ValueError("No action plan found in state. Run PlanSpriteActionsNode first.") + + # Convert the Scratch project JSON to a mutable Python object + project_json = state["project_json"] + targets = project_json["targets"] + + # We need a way to map sprite names to their actual target objects in project_json + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + + # Iterate through the planned actions for each sprite + for sprite_name, sprite_actions_data in action_plan.get("action_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + # Iterate through each planned event/script for the current sprite + for plan_entry in sprite_actions_data.get("plans", []): + event_description = plan_entry["event"] + logic_description = plan_entry["logic"] + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is:\n" + f" Event: '{event_description}'\n" + f" Logic: '{logic_description}'\n\n" + f"Here is the comprehensive catalog of all available Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(ALL_SCRATCH_BLOCKS_CATALOG, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON:**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'abc123def4').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Fill `inputs` for parameters and substacks:**\n" + f" * If a block takes a direct value input (e.g., `motion_movesteps` takes 'STEPS'), provide the input value as an array `[type_code, value_or_block_id]`. The `type_code` for the outer array must be a **number**: `1` (for a primitive/reporter input), `2` (for a block ID input), or `3` (for a variable/list reference).\n" + f" * `[1, [4, \"10\"]]` for directly embedding a simple number or string literal as a primitive. The inner `4` indicates a number primitive.\n" + f" * `[1, \"SHADOW_BLOCK_ID\"]` for inputs where a separate `shadow: true` block is used (e.g., a `math_number` block for a number, or a menu/message block for a dropdown/broadcast message).\n" + f" * `[2, \"CONNECTED_BLOCK_ID\"]` for a block that is plugged into this input (e.g., an operator block).\n" + f" * **For C-blocks (e.g., `control_forever`, `control_if`, `control_repeat`):** Use the `SUBSTACK` input. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_SUBSTACK_ID\"]`. The `2` indicates that the input is a block ID, and `FIRST_BLOCK_IN_SUBSTACK_ID` is the ID of the first block inside the loop/conditional structure.\n" + f"5. **Define Shadow Blocks for Literals, Dropdowns, and Broadcast Messages:** When a block requires a number, string literal, a selection from a dropdown menu, or a broadcast message, you must define a separate `shadow: true` block and link it.\n" + f" * **For a number literal (e.g., '10'):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_NUM_ID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, link to this shadow block: `\"INPUT_NAME\": [1, \"UNIQUE_SHADOW_NUM_ID\"]`\n" + f" * **For a dropdown menu selection (e.g., 'space' key for `when key pressed`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_MENU_ID\": {{\n" + f" \"opcode\": \"event_whenkeypressed_keymenu\", // Example opcode for a key menu shadow\n" + f" \"fields\": {{\n" + f" \"KEY_OPTION\": [\"space\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, link to this shadow block: `\"KEY_OPTION\": [1, \"UNIQUE_SHADOW_MENU_ID\"]`\n" + f" * **For a Broadcast Message (e.g., 'gameover' for `broadcast message`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_BROADCAST_ID\": {{\n" + f" \"opcode\": \"event_broadcast_menu\", // This opcode specifically handles the message name\n" + f" \"fields\": {{\n" + f" \"BROADCAST_OPTION\": [\"gameover\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BROADCAST_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent `event_broadcast` or `event_broadcastandwait` block's `inputs`, link to this shadow block: `\"BROADCAST\": [1, \"UNIQUE_SHADOW_BROADCAST_ID\"]`\n" + f"6. **`fields` for direct dropdown values/text:** If the block directly uses a dropdown value or text field without an `inputs` connection, use the `fields` dictionary. Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + print(f"Raw response from LLM [ActionNodeBuilder - {sprite_name} - {event_description}]:", raw_response) + + # Directly extract the dictionary of blocks + generated_blocks = extract_json_from_llm_response(raw_response) + + # Check for and unwrap any accidental 'blocks' nesting + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Action blocks added for sprite '{sprite_name}', script '{event_description}' by ActionNodeBuilder.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_description}': {e}") + raise + print("Updated project JSON with action nodes:", json.dumps(project_json, indent=2)) + return {"project_json": project_json} + +# Node 5: Behaviour Collector (Sensing/Collision) +def plan_behaviors_node(state: GameState): + logger.info("--- Running PlanBehaviorsNode ---") + + # Get relevant context from the state + description = state.get("description","") + action_plan = state.get("action_plan", {}) + initial_positions = state.get("initial_positions", {}) # Assuming this will be populated by a prior node if needed + + # Construct the prompt with comprehensive context + planning_prompt = ( + "Generate a detailed plan for the game's behaviors based on the following context:\n\n" + f"The game description is: '{state['description']}'.\n\n" + f"Action Plan: {json.dumps(action_plan, indent=2)}\n" + f"Initial Positions: {json.dumps(initial_positions, indent=2)}\n\n" + "Your task is to define the game's behaviors, including scoring, collision detection, " + "game state management (e.g., win/loss conditions, resets), and any necessary sprite " + "initialization (like starting positions or visibility)." + "The output should be a JSON object with a single key 'behavior_overall_flow'. " + "The value should be a dictionary where keys are sprite names (e.g., 'Stage', 'Player', 'Enemy', 'Coin') " + "and values are dictionaries containing a 'plans' list. " + "Each plan in the list should have an 'event' (a high-level trigger) and 'logic' (a natural language description " + "of the behavior, suitable for block generation)." + "\n\nExample structure for 'behavior_overall_flow':\n" + "```json\n" + "{\n" + " \"behavior_overall_flow\": {\n" + " \"Stage\": {\n" + " \"plans\": [\n" + " {\"event\": \"Game Initialization\", \"logic\": \"When green flag clicked, reset score, lives, and timer.\"}\n" + " ]\n" + " },\n" + " \"Player\": {\n" + " \"plans\": [\n" + " {\"event\": \"Start Position\", \"logic\": \"When green flag clicked, go to x:0 y:0.\"},\n" + " {\"event\": \"Collision with Enemy\", \"logic\": \"If the Player is touching 'Enemy', change score by -1, change lives by -1, and broadcast 'hit'.\"},\n" + " {\"event\": \"Collect Coin\", \"logic\": \"If the Player is touching 'Coin', change score by 10, and hide 'Coin'.\"}\n" + " ]\n" + " },\n" + " \"Enemy\": {\n" + " \"plans\": [\n" + " {\"event\": \"Game Reset\", \"logic\": \"When 'hit' is received, move Enemy to a random position and show it.\"}\n" + " ]\n" + " },\n" + " \"Coin\": {\n" + " \"plans\": [\n" + " {\"event\": \"Game Reset (Coin)\", \"logic\": \"When green flag clicked, show the Coin.\"}\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Based on the provided context, generate the `behavior_overall_flow`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [PlanBehaviorsNode]: {raw_response[:200]}...") + + behavior_plan = extract_json_from_llm_response(raw_response) + if "behavior_overall_flow" not in behavior_plan: + raise ValueError("LLM response for behavior plan missing 'behavior_overall_flow' key.") + + state["behavior_plan"] = behavior_plan + logger.info("Behavior plan generated and added to state.") + return {"behavior_plan": behavior_plan} + except Exception as e: + logger.error(f"Error generating behavior plan: {e}") + raise + + + +def build_behavior_nodes(state: GameState): + logger.info("--- Running BehaviorNodeBuilder ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + + behavior_plan = state.get("behavior_plan", {}) + if not behavior_plan: + logger.warning("No behavior plan found in state. Skipping BehaviorNodeBuilder.") + return state + + for sprite_name, sprite_behaviors_data in behavior_plan.get("behavior_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + for plan_entry in sprite_behaviors_data.get("plans", []): + event_description = plan_entry["event"] + logic_description = plan_entry["logic"] + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is:\n" + f" Event: '{event_description}'\n" + f" Logic: '{logic_description}'\n\n" + f"Here is the comprehensive catalog of all available Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(ALL_SCRATCH_BLOCKS_CATALOG, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON:**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'abc123def4').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Fill `inputs` for parameters and substacks:**\n" + f" * If a block takes a direct value input (e.g., `motion_movesteps` takes 'STEPS'), provide the input value as an array `[type_code, value_or_block_id]`. The `type_code` for the outer array must be a **number**: `1` (for a primitive/reporter input), `2` (for a block ID input), or `3` (for a variable/list reference).\n" + f" * `[1, [4, \"10\"]]` for directly embedding a simple number or string literal as a primitive. The inner `4` indicates a number primitive.\n" + f" * `[1, \"SHADOW_BLOCK_ID\"]` for inputs where a separate `shadow: true` block is used (e.g., a `math_number` block for a number, or a menu/message block for a dropdown/broadcast message).\n" + f" * `[2, \"CONNECTED_BLOCK_ID\"]` for a block that is plugged into this input (e.g., an operator block).\n" + f" * **For C-blocks (e.g., `control_forever`, `control_if`, `control_repeat`):** Use the `SUBSTACK` input. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_SUBSTACK_ID\"]`. The `2` indicates that the input is a block ID, and `FIRST_BLOCK_IN_SUBSTACK_ID` is the ID of the first block inside the loop/conditional structure.\n" + f"5. **Define Shadow Blocks for Literals, Dropdowns, and Broadcast Messages:** When a block requires a number, string literal, a selection from a dropdown menu, or a broadcast message, you must define a separate `shadow: true` block and link it.\n" + f" * **For a number literal (e.g., '10'):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_NUM_ID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, link to this shadow block: `\"INPUT_NAME\": [1, \"UNIQUE_SHADOW_NUM_ID\"]`\n" + f" * **For a dropdown menu selection (e.g., 'space' key for `when key pressed`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_MENU_ID\": {{\n" + f" \"opcode\": \"event_whenkeypressed_keymenu\", // Example opcode for a key menu shadow\n" + f" \"fields\": {{\n" + f" \"KEY_OPTION\": [\"space\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, link to this shadow block: `\"KEY_OPTION\": [1, \"UNIQUE_SHADOW_MENU_ID\"]`\n" + f" * **For a Broadcast Message (e.g., 'gameover' for `broadcast message`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_BROADCAST_ID\": {{\n" + f" \"opcode\": \"event_broadcast_menu\", // This opcode specifically handles the message name\n" + f" \"fields\": {{\n" + f" \"BROADCAST_OPTION\": [\"gameover\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BROADCAST_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent `event_broadcast` or `event_broadcastandwait` block's `inputs`, link to this shadow block: `\"BROADCAST\": [1, \"UNIQUE_SHADOW_BROADCAST_ID\"]`\n" + f" * **For Sensing Menu Options (e.g., touching 'edge', 'mouse-pointer'):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_SENSING_MENU_ID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\", // Or other sensing menu opcodes like sensing_keyoptions\n" + f" \"fields\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [\"_mouse_\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_SENSING_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent `sensing_touchingobject` block's `inputs`, link: `\"TOUCHINGOBJECTMENU\": [1, \"UNIQUE_SHADOW_SENSING_MENU_ID\"]`\n" + f"6. **`fields` for direct dropdown values/text:** If the block directly uses a dropdown value or text field without an `inputs` connection, use the `fields` dictionary. Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + print(f"Raw response from LLM [BehaviorNodeBuilder - {sprite_name} - {event_description}]:", raw_response) + + generated_blocks = extract_json_from_llm_response(raw_response) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Behavior blocks added for sprite '{sprite_name}', script '{event_description}' by BehaviorNodeBuilder.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_description}': {e}") + raise + + logger.info("Updated project JSON with behavior nodes.") + print("Updated project JSON with behavior nodes:", json.dumps(project_json, indent=2)) + return {"project_json": project_json} + +# Node 4: Scoring Function +def add_scoring_function(state: GameState): + logger.info("--- Running ScoringNode ---") + if not state['parsed_game_elements'].get("scoring_goals"): + logger.info("No scoring goals identified, skipping scoring node.") + return {} # Return an empty dictionary if no changes are made + + llm_scoring_prompt = ( + f"Based on the game description '{state['description']}' " + f"add a scoring mechanism to the Scratch project JSON. " + f"**Crucially, add a 'score' variable to the Stage's 'variables' dictionary.** The format for variables in `targets[].variables` is " + f"a dictionary where keys are unique variable IDs (e.g., 'scoreVarId123') and values are arrays `[\"variable_name\", initial_value]`. " + f"Example: `\"variables\": {{ \"scoreVarId123\": [\"score\", 0] }}`.\n" + f"Then, add blocks to update it (e.g., 'data_setvariableto', 'data_changevariableby') " + f"based on positive actions identified in 'scoring_goals' (e.g., jumping over an obstacle). " + f"**Strictly adhere to the Scratch 3.0 block schema rules for `inputs` and `fields`:**\n" + f"- For `data_setvariableto` or `data_changevariableby`, the variable name is a `field`: `\"VARIABLE\": [\"score\", null]`.\n" + f"- The value for `data_setvariableto` or `data_changevariableby` is an `input` linking to a shadow block: `\"VALUE\": [1, [\"num\", \"0\"]]` or `\"VALUE\": [1, [\"num\", \"10\"]]`.\n" + f"Ensure all block IDs are unique and relationships are correct. No nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_scoring_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Scoring function added by ScoringNode.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ScoringNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ScoringNode: {e}") + raise + +# Node 5: Reset Option +def add_reset_option(state: GameState): + logger.info("--- Running ResetNode ---") + if not state['parsed_game_elements'].get("reset_triggers"): + logger.info("No reset triggers identified, skipping reset node.") + return {} # Return an empty dictionary if no changes are made + + llm_reset_prompt = ( + f"Based on the game description '{state['description']}'" + f"add a reset mechanism to the Scratch project JSON. " + f"This should typically involve an 'event_whenkeypressed' (e.g., 'r' key) or 'event_whenflagclicked' that resets sprite positions ('motion_gotoxy') " + f"and any variables (like score) to their initial states, potentially broadcasting a 'reset_game' message. " + f"**Ensure variable IDs and names are consistent with those in the Stage's 'variables' dictionary.**\n" + f"**Strictly adhere to the Scratch 3.0 block schema rules provided in the SYSTEM_PROMPT for `inputs` and `fields`:**\n" + f"- For `event_whenkeypressed`, the key is a `field`: `\"KEY\": [\"r\", null]`.\n" + f"- For `motion_gotoxy`, `X` and `Y` are `inputs` that link to number reporter shadow blocks: `\"X\": [1, [\"num\", \"-160\"]], \"Y\": [1, [\"num\", \"-110\"]]`.\n" + f"- For `data_setvariableto`, the variable name is a `field`: `\"VARIABLE\": [\"score\", null]`, and value is an `input` like `\"VALUE\": [1, [\"num\", \"0\"]]`.\n" + f"- For `event_broadcast`, the message is a `field`: `\"BROADCAST_OPTION\": [\"reset_game\", null]`.\n" + f"- Ensure all block IDs are unique and relationships are correct. No nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_reset_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Reset option added by ResetNode.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ResetNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ResetNode: {e}") + raise + +# Node 6: JSON Validation (and optional Refinement) +def validate_json(state: GameState): + logger.info("--- Running JSONValidatorNode ---") + validation_errors = [] + + try: + # Basic check if it's a valid JSON structure at least + json.dumps(state['project_json']) + + for i, target in enumerate(state['project_json'].get("targets", [])): + target_name = target.get('name', f'index {i}') + # Validate objName presence for all targets + if "objName" not in target: + validation_errors.append(f"Target '{target_name}' is missing 'objName'.") + + # Validate variables structure + if "variables" in target and isinstance(target["variables"], dict): + for var_id, var_data in target["variables"].items(): + if not isinstance(var_data, list) or len(var_data) != 2 or not isinstance(var_data[0], str): + validation_errors.append(f"Target '{target_name}' variable '{var_id}' has incorrect format: {var_data}. Expected [\"name\", initial_value].") + else: + validation_errors.append(f"Target '{target_name}' has missing or malformed 'variables' section.") + + + block_ids = set() + for block_id, block_data in target.get("blocks", {}).items(): + if block_id in block_ids: + validation_errors.append(f"Duplicate block ID found: '{block_id}' in target '{target_name}'.") + block_ids.add(block_id) + + # Validate inputs + for input_name, input_data in block_data.get("inputs", {}).items(): + if not isinstance(input_data, list): + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has non-list input '{input_name}': {input_data}. Should be a list.") + continue + + if len(input_data) < 2: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has malformed input '{input_name}': {input_data}. Expected at least 2 elements.") + continue + + if input_data[0] in [1, 2, 3] and isinstance(input_data[1], str) and input_data[1] not in block_ids and input_data[1] != "": + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent input block '{input_data[1]}' for input '{input_name}'.") + + # Validate fields + for field_name, field_data in block_data.get("fields", {}).items(): + if not isinstance(field_data, list) or len(field_data) < 1 or (len(field_data) > 1 and field_data[1] is not None): + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has malformed field '{field_name}': {field_data}. Should be [\"value\", null].") + + # Validate parent + if "parent" in block_data and block_data["parent"] is not None and block_data["parent"] not in block_ids: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent parent '{block_data['parent']}'.") + + # Validate next + if "next" in block_data and block_data["next"] is not None and block_data["next"] not in block_ids: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent next block '{block_data['next']}'.") + + # Check for nested 'blocks' (common LLM error) + if "blocks" in block_data and isinstance(block_data["blocks"], dict) and len(block_data["blocks"]) > 0: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has a nested 'blocks' dictionary. Blocks should only be at the target's top level and linked via inputs (e.g., SUBSTACK).") + + + except json.JSONDecodeError as e: + validation_errors.append(f"Structural JSON error (invalid format) during validation: {e}") + except Exception as e: + validation_errors.append(f"An unexpected error occurred during JSON validation: {e}") + + if validation_errors: + error_message = "; ".join(validation_errors) + logger.warning(f"JSON validation failed: {error_message}. Attempting refinement...") + + llm_refine_prompt = ( + f"The Scratch project JSON you generated failed validation due to the following issues: {error_message}. " + f"Please review and correct the JSON, ensuring all rules from your SYSTEM_PROMPT are followed, " + f"especially regarding unique block IDs, correct structure (parent/next/inputs/fields), and valid opcodes. " + f"Pay extremely close attention to the **exact array format for inputs and fields** (e.g., `\"FIELD_NAME\": [\"value\", null]` and `\"INPUT_NAME\": [1, [\"num\", \"value\"]]` or `[2, \"linked_block_id\"]`). " + f"Crucially, ensure the `variables` section within each target is formatted as a dictionary where keys are unique IDs and values are `[\"variable_name\", initial_value]`. " + f"Also, ensure no `blocks` dictionary is nested inside a block definition; use `SUBSTACK` input for C-blocks. " + f"The current problematic JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **corrected complete JSON**." + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_refine_prompt}]}) + raw_response = response["messages"][-1].content + refined_project_json = extract_json_from_llm_response(raw_response) + logger.info("JSON refined and updated by JSONValidatorNode.") + return {"project_json": refined_project_json} + except ValueError as e_refine: + logger.error(f"Refinement also failed to return valid JSON: {e_refine}") + raise ValueError(f"JSON validation and refinement failed: {e_refine}") + except Exception as e_refine: + logger.error(f"An unexpected error occurred during JSON refinement: {e_refine}") + raise ValueError(f"JSON validation and refinement failed due to unexpected error: {e_refine}") + else: + logger.info("JSON validation successful (all checks passed).") + return {} # Return an empty dictionary if no changes are needed or made in this pass + +# Build the LangGraph workflow +workflow = StateGraph(GameState) + +workflow.add_node("parse_query", parse_query_and_set_initial_positions) +workflow.add_node("plan_actions", plan_sprite_actions) +workflow.add_node("build_actions", build_action_nodes) +workflow.add_node("plan_behaviors", plan_behaviors_node) +workflow.add_node("build_behaviors", build_behavior_nodes) +workflow.add_node("add_scoring", add_scoring_function) +workflow.add_node("add_reset", add_reset_option) +workflow.add_node("validate_json", validate_json) + +# Define the flow (edges) +workflow.set_entry_point("parse_query") +workflow.add_edge("parse_query", "plan_actions") +workflow.add_edge("plan_actions", "build_actions") +workflow.add_edge("build_actions", "plan_behaviors") +workflow.add_edge("plan_behaviors", "build_behaviors") +workflow.add_edge("build_behaviors", "add_scoring") +workflow.add_edge("add_scoring", "add_reset") +workflow.add_edge("add_reset", "validate_json") +workflow.add_edge("validate_json", END) # End of the main flow + +app_graph = workflow.compile() + +### Modified `generate_game` Endpoint + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description", "") + backdrops = payload.get("backdrops", []) + sprites = payload.get("sprites", []) + + logger.info(f"Starting game generation for description: '{desc}'") + + # 1) Initial skeleton generation + project_skeleton = { + "targets": [ + { + "isStage": True, + "name":"Stage", + "objName": "Stage", # ADDED THIS FOR THE STAGE + "variables":{}, # This must be a dict: { "var_id": ["var_name", value] } + "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + for idx, s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project_skeleton["targets"].append({ + "isStage": False, + "name": s["name"], + "objName": s["name"], # objName is also important for sprites + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + logger.info("Initial project skeleton created.") + + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + + try: + os.makedirs(project_folder, exist_ok=True) + # Save initial skeleton and copy assets + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project_skeleton, f, indent=2) + logger.info(f"Initial project skeleton saved to {project_json_path}") + + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source backdrop asset not found: {src_path}") + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source sprite asset not found: {src_path}") + logger.info("Assets copied to project folder.") + + # Initialize the state for LangGraph as a dictionary matching the TypedDict structure + initial_state_dict: GameState = { # Type hint for clarity + "project_json": project_skeleton, + "description": desc, + "project_id": project_id, + "parsed_game_elements": {} # Initialize this as well + } + + # Run the LangGraph workflow with the dictionary input. + final_state_dict: GameState = app_graph.invoke(initial_state_dict) + + # Access elements from the final_state_dict + final_project_json = final_state_dict['project_json'] + + # Save the *final* filled project JSON, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(final_project_json, f, indent=2) + logger.info(f"Final project JSON saved to {project_json_path}") + + return jsonify({"message": "Game generated successfully", "project_id": project_id}) + + except Exception as e: + logger.error(f"Error during game generation workflow for project ID {project_id}: {e}", exc_info=True) + return jsonify(error=f"Error generating game: {e}"), 500 + + +if __name__=="__main__": + logger.info("Starting Flask application...") + app.run(debug=True,port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/blocks.json b/scratch_VLM/scratch_agent/blocks/blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..22c9e9ae7643832795da94fecc488814c8e393e4 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/blocks.json @@ -0,0 +1,2221 @@ +{ + "motion_movesteps": { + "opcode": "motion_movesteps", + "next": null, + "parent": null, + "inputs": { + "STEPS": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 464, + "y": -416 + }, + "motion_turnright": { + "opcode": "motion_turnright", + "next": null, + "parent": null, + "inputs": { + "DEGREES": [ + 1, + [ + 4, + "15" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 467, + "y": -316 + }, + "motion_turnleft": { + "opcode": "motion_turnleft", + "next": null, + "parent": null, + "inputs": { + "DEGREES": [ + 1, + [ + 4, + "15" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 464, + "y": -210 + }, + "motion_goto": { + "opcode": "motion_goto", + "next": null, + "parent": null, + "inputs": { + "TO": [ + 1, + "@iM=Z?~GCbpC}gT7KAKY" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 465, + "y": -95 + }, + "motion_goto_menu": { + "opcode": "motion_goto_menu", + "next": null, + "parent": "d|J?C902/xy6tD5,|dmB", + "inputs": {}, + "fields": { + "TO": [ + "_random_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "motion_gotoxy": { + "opcode": "motion_gotoxy", + "next": null, + "parent": null, + "inputs": { + "X": [ + 1, + [ + 4, + "0" + ] + ], + "Y": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 468, + "y": 12 + }, + "motion_glideto": { + "opcode": "motion_glideto", + "next": null, + "parent": null, + "inputs": { + "SECS": [ + 1, + [ + 4, + "1" + ] + ], + "TO": [ + 1, + "{id to destination position}" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 470, + "y": 129 + }, + "motion_glideto_menu": { + "opcode": "motion_glideto_menu", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "TO": [ + "_random_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "motion_glidesecstoxy": { + "opcode": "motion_glidesecstoxy", + "next": null, + "parent": null, + "inputs": { + "SECS": [ + 1, + [ + 4, + "1" + ] + ], + "X": [ + 1, + [ + 4, + "0" + ] + ], + "Y": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 476, + "y": 239 + }, + "motion_pointindirection": { + "opcode": "motion_pointindirection", + "next": null, + "parent": null, + "inputs": { + "DIRECTION": [ + 1, + [ + 8, + "90" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 493, + "y": 361 + }, + "motion_pointtowards": { + "opcode": "motion_pointtowards", + "next": null, + "parent": null, + "inputs": { + "TOWARDS": [ + 1, + "6xQl1pPk%9E~Znhm*:ng" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 492, + "y": 463 + }, + "motion_pointtowards_menu": { + "opcode": "motion_pointtowards_menu", + "next": null, + "parent": "Ucm$YBs*^9GFTGXCbal@", + "inputs": {}, + "fields": { + "TOWARDS": [ + "_mouse_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "motion_changexby": { + "opcode": "motion_changexby", + "next": null, + "parent": null, + "inputs": { + "DX": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 851, + "y": -409 + }, + "motion_setx": { + "opcode": "motion_setx", + "next": null, + "parent": null, + "inputs": { + "X": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 864, + "y": -194 + }, + "motion_changeyby": { + "opcode": "motion_changeyby", + "next": null, + "parent": null, + "inputs": { + "DY": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 861, + "y": -61 + }, + "motion_sety": { + "opcode": "motion_sety", + "next": null, + "parent": null, + "inputs": { + "Y": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 864, + "y": 66 + }, + "motion_ifonedgebounce": { + "opcode": "motion_ifonedgebounce", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1131, + "y": -397 + }, + "motion_setrotationstyle": { + "opcode": "motion_setrotationstyle", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "STYLE": [ + "left-right", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 1128, + "y": -287 + }, + "motion_xposition": { + "opcode": "motion_xposition", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1193, + "y": -136 + }, + "motion_yposition": { + "opcode": "motion_yposition", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1181, + "y": -64 + }, + "motion_direction": { + "opcode": "motion_direction", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1188, + "y": 21 + }, + "control_wait": { + "opcode": "control_wait", + "next": null, + "parent": null, + "inputs": { + "DURATION": [ + 1, + [ + 5, + "1" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 337, + "y": 129 + }, + "control_repeat": { + "opcode": "control_repeat", + "next": null, + "parent": null, + "inputs": { + "TIMES": [ + 1, + [ + 6, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 265 + }, + "control_forever": { + "opcode": "control_forever", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 334, + "y": 439 + }, + "control_if": { + "opcode": "control_if", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 331, + "y": 597 + }, + "control_if_else": { + "opcode": "control_if_else", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 335, + "y": 779 + }, + "control_wait_until": { + "opcode": "control_wait_until", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 676, + "y": 285 + }, + "control_repeat_until": { + "opcode": "control_repeat_until", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 692, + "y": 381 + }, + "control_stop": { + "opcode": "control_stop", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "STOP_OPTION": [ + "all", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 708, + "y": 545, + "mutation": { + "tagName": "mutation", + "children": [], + "hasnext": "false" + } + }, + "control_start_as_clone": { + "opcode": "control_start_as_clone", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 665, + "y": 672 + }, + "control_create_clone_of": { + "opcode": "control_create_clone_of", + "next": null, + "parent": null, + "inputs": { + "CLONE_OPTION": [ + 1, + "t))DW9(QSKB]3C/3Ou+J" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 648, + "y": 797 + }, + "control_create_clone_of_menu": { + "opcode": "control_create_clone_of_menu", + "next": null, + "parent": "80yo/}Cw++Z.;x[ohh|7", + "inputs": {}, + "fields": { + "CLONE_OPTION": [ + "_myself_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "control_delete_this_clone": { + "opcode": "control_delete_this_clone", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 642, + "y": 914 + }, + "event_whenflagclicked": { + "opcode": "event_whenflagclicked", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 166, + "y": -422 + }, + "event_whenkeypressed": { + "opcode": "event_whenkeypressed", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "space", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 151, + "y": -329 + }, + "event_whenthisspriteclicked": { + "opcode": "event_whenthisspriteclicked", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 156, + "y": -223 + }, + "event_whenbackdropswitchesto": { + "opcode": "event_whenbackdropswitchesto", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "BACKDROP": [ + "backdrop1", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 148, + "y": -101 + }, + "event_whengreaterthan": { + "opcode": "event_whengreaterthan", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": { + "WHENGREATERTHANMENU": [ + "LOUDNESS", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 150, + "y": 10 + }, + "event_whenbroadcastreceived": { + "opcode": "event_whenbroadcastreceived", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "BROADCAST_OPTION": [ + "message1", + "5O!nei;S$!c!=hCT}0:a" + ] + }, + "shadow": false, + "topLevel": true, + "x": 141, + "y": 118 + }, + "event_broadcast": { + "opcode": "event_broadcast", + "next": null, + "parent": null, + "inputs": { + "BROADCAST_INPUT": [ + 1, + [ + 11, + "message1", + "5O!nei;S$!c!=hCT}0:a" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 151, + "y": 229 + }, + "event_broadcastandwait": { + "opcode": "event_broadcastandwait", + "next": null, + "parent": null, + "inputs": { + "BROADCAST_INPUT": [ + 1, + [ + 11, + "message1", + "5O!nei;S$!c!=hCT}0:a" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 157, + "y": 340 + }, + "sensing_touchingobject": { + "opcode": "sensing_touchingobject", + "next": null, + "parent": null, + "inputs": { + "TOUCHINGOBJECTMENU": [ + 1, + "xSKW9a+wTnM~h~So8Jc]" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 359, + "y": 116 + }, + "sensing_touchingobjectmenu": { + "opcode": "sensing_touchingobjectmenu", + "next": null, + "parent": "Y(n,F@BYzwd4CiN|Bh[P", + "inputs": {}, + "fields": { + "TOUCHINGOBJECTMENU": [ + "_mouse_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sensing_touchingcolor": { + "opcode": "sensing_touchingcolor", + "next": null, + "parent": null, + "inputs": { + "COLOR": [ + 1, + [ + 9, + "#55b888" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 360, + "y": 188 + }, + "sensing_coloristouchingcolor": { + "opcode": "sensing_coloristouchingcolor", + "next": null, + "parent": null, + "inputs": { + "COLOR": [ + 1, + [ + 9, + "#d019f2" + ] + ], + "COLOR2": [ + 1, + [ + 9, + "#2b0de3" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 277 + }, + "sensing_askandwait": { + "opcode": "sensing_askandwait", + "next": null, + "parent": null, + "inputs": { + "QUESTION": [ + 1, + [ + 10, + "What's your name?" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 338, + "y": 354 + }, + "sensing_answer": { + "opcode": "sensing_answer", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 782, + "y": 111 + }, + "sensing_keypressed": { + "opcode": "sensing_keypressed", + "next": null, + "parent": null, + "inputs": { + "KEY_OPTION": [ + 1, + "SNlf@Im$sv%.6ULi-f3i" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 762, + "y": 207 + }, + "sensing_keyoptions": { + "opcode": "sensing_keyoptions", + "next": null, + "parent": "7$xEUO.2hH2R6vh!$(Uj", + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "space", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sensing_mousedown": { + "opcode": "sensing_mousedown", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 822, + "y": 422 + }, + "sensing_mousex": { + "opcode": "sensing_mousex", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 302, + "y": 528 + }, + "sensing_mousey": { + "opcode": "sensing_mousey", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 668, + "y": 547 + }, + "sensing_setdragmode": { + "opcode": "sensing_setdragmode", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "DRAG_MODE": [ + "draggable", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 950, + "y": 574 + }, + "sensing_loudness": { + "opcode": "sensing_loudness", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 658, + "y": 703 + }, + "sensing_timer": { + "opcode": "sensing_timer", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 459, + "y": 671 + }, + "sensing_resettimer": { + "opcode": "sensing_resettimer", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 462, + "y": 781 + }, + "sensing_of": { + "opcode": "sensing_of", + "next": null, + "parent": null, + "inputs": { + "OBJECT": [ + 1, + "t+o*y;iz,!O#aT|qM_+O" + ] + }, + "fields": { + "PROPERTY": [ + "backdrop #", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 997, + "y": 754 + }, + "sensing_of_object_menu": { + "opcode": "sensing_of_object_menu", + "next": null, + "parent": "[4I2wIG/tNc@LQ-;FbsB", + "inputs": {}, + "fields": { + "OBJECT": [ + "_stage_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sensing_current": { + "opcode": "sensing_current", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "CURRENTMENU": [ + "YEAR", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 627, + "y": 884 + }, + "sensing_dayssince2000": { + "opcode": "sensing_dayssince2000", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 959, + "y": 903 + }, + "sensing_username": { + "opcode": "sensing_username", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 833, + "y": 757 + }, + "operator_add": { + "opcode": "operator_add", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 128, + "y": 153 + }, + "operator_subtract": { + "opcode": "operator_subtract", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 134, + "y": 214 + }, + "operator_multiply": { + "opcode": "operator_multiply", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 134, + "y": 278 + }, + "operator_divide": { + "opcode": "operator_divide", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 138, + "y": 359 + }, + "operator_random": { + "opcode": "operator_random", + "next": null, + "parent": null, + "inputs": { + "FROM": [ + 1, + [ + 4, + "1" + ] + ], + "TO": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 311, + "y": 157 + }, + "operator_gt": { + "opcode": "operator_gt", + "next": null, + "parent": null, + "inputs": { + "OPERAND1": [ + 1, + [ + 10, + "" + ] + ], + "OPERAND2": [ + 1, + [ + 10, + "50" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 217 + }, + "operator_lt": { + "opcode": "operator_lt", + "next": null, + "parent": null, + "inputs": { + "OPERAND1": [ + 1, + [ + 10, + "" + ] + ], + "OPERAND2": [ + 1, + [ + 10, + "50" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 345, + "y": 286 + }, + "operator_equals": { + "opcode": "operator_equals", + "next": null, + "parent": null, + "inputs": { + "OPERAND1": [ + 1, + [ + 10, + "" + ] + ], + "OPERAND2": [ + 1, + [ + 10, + "50" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 345, + "y": 372 + }, + "operator_and": { + "opcode": "operator_and", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 701, + "y": 158 + }, + "operator_or": { + "opcode": "operator_or", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 705, + "y": 222 + }, + "operator_not": { + "opcode": "operator_not", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 734, + "y": 283 + }, + "operator_join": { + "opcode": "operator_join", + "next": null, + "parent": null, + "inputs": { + "STRING1": [ + 1, + [ + 10, + "apple " + ] + ], + "STRING2": [ + 1, + [ + 10, + "banana" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 663, + "y": 378 + }, + "operator_letter_of": { + "opcode": "operator_letter_of", + "next": null, + "parent": null, + "inputs": { + "LETTER": [ + 1, + [ + 6, + "1" + ] + ], + "STRING": [ + 1, + [ + 10, + "apple" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 664, + "y": 445 + }, + "operator_length": { + "opcode": "operator_length", + "next": null, + "parent": null, + "inputs": { + "STRING": [ + 1, + [ + 10, + "apple" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 664, + "y": 521 + }, + "operator_contains": { + "opcode": "operator_contains", + "next": null, + "parent": null, + "inputs": { + "STRING1": [ + 1, + [ + 10, + "apple" + ] + ], + "STRING2": [ + 1, + [ + 10, + "a" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 634, + "y": 599 + }, + "operator_mod": { + "opcode": "operator_mod", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 295, + "y": 594 + }, + "operator_round": { + "opcode": "operator_round", + "next": null, + "parent": null, + "inputs": { + "NUM": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 307, + "y": 674 + }, + "operator_mathop": { + "opcode": "operator_mathop", + "next": null, + "parent": null, + "inputs": { + "NUM": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": { + "OPERATOR": [ + "abs", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 280, + "y": 754 + }, + "data_setvariableto": { + "opcode": "data_setvariableto", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 10, + "0" + ] + ] + }, + "fields": { + "VARIABLE": [ + "my variable", + "`jEk@4|i[#Fk?(8x)AV.-my variable" + ] + }, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 241 + }, + "data_changevariableby": { + "opcode": "data_changevariableby", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 4, + "1" + ] + ] + }, + "fields": { + "VARIABLE": [ + "my variable", + "`jEk@4|i[#Fk?(8x)AV.-my variable" + ] + }, + "shadow": false, + "topLevel": true, + "x": 313, + "y": 363 + }, + "data_showvariable": { + "opcode": "data_showvariable", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "VARIABLE": [ + "my variable", + "`jEk@4|i[#Fk?(8x)AV.-my variable" + ] + }, + "shadow": false, + "topLevel": true, + "x": 415, + "y": 473 + }, + "data_hidevariable": { + "opcode": "data_hidevariable", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "VARIABLE": [ + "my variable", + "`jEk@4|i[#Fk?(8x)AV.-my variable" + ] + }, + "shadow": false, + "topLevel": true, + "x": 319, + "y": 587 + }, + "data_addtolist": { + "opcode": "data_addtolist", + "next": null, + "parent": null, + "inputs": { + "ITEM": [ + 1, + [ + 10, + "thing" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 385, + "y": 109 + }, + "data_deleteoflist": { + "opcode": "data_deleteoflist", + "next": null, + "parent": null, + "inputs": { + "INDEX": [ + 1, + [ + 7, + "1" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 384, + "y": 244 + }, + "data_deletealloflist": { + "opcode": "data_deletealloflist", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 387, + "y": 374 + }, + "data_insertatlist": { + "opcode": "data_insertatlist", + "next": null, + "parent": null, + "inputs": { + "ITEM": [ + 1, + [ + 10, + "thing" + ] + ], + "INDEX": [ + 1, + [ + 7, + "1" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 366, + "y": 527 + }, + "data_replaceitemoflist": { + "opcode": "data_replaceitemoflist", + "next": null, + "parent": null, + "inputs": { + "INDEX": [ + 1, + [ + 7, + "1" + ] + ], + "ITEM": [ + 1, + [ + 10, + "thing" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 365, + "y": 657 + }, + "data_itemoflist": { + "opcode": "data_itemoflist", + "next": null, + "parent": null, + "inputs": { + "INDEX": [ + 1, + [ + 7, + "1" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 862, + "y": 117 + }, + "data_itemnumoflist": { + "opcode": "data_itemnumoflist", + "next": null, + "parent": null, + "inputs": { + "ITEM": [ + 1, + [ + 10, + "thing" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 883, + "y": 238 + }, + "data_lengthoflist": { + "opcode": "data_lengthoflist", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 876, + "y": 342 + }, + "data_listcontainsitem": { + "opcode": "data_listcontainsitem", + "next": null, + "parent": null, + "inputs": { + "ITEM": [ + 1, + [ + 10, + "thing" + ] + ] + }, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 871, + "y": 463 + }, + "data_showlist": { + "opcode": "data_showlist", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 931, + "y": 563 + }, + "data_hidelist": { + "opcode": "data_hidelist", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "LIST": [ + "MY_LIST", + "o6`kIhtT{xWH+rX(5d,A" + ] + }, + "shadow": false, + "topLevel": true, + "x": 962, + "y": 716 + }, + "sound_playuntildone": { + "opcode": "sound_playuntildone", + "next": null, + "parent": null, + "inputs": { + "SOUND_MENU": [ + 1, + "4w%pR8G.yD%g-BwCj=uK" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 253, + "y": 17 + }, + "sound_sounds_menu": { + "opcode": "sound_sounds_menu", + "next": null, + "parent": "Pdc$U;s8e_uUfTX`}jOo", + "inputs": {}, + "fields": { + "SOUND_MENU": [ + "Meow", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sound_play": { + "opcode": "sound_play", + "next": null, + "parent": null, + "inputs": { + "SOUND_MENU": [ + 1, + "i1U{^VHb*2`9?l}=:L)/" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 245, + "y": 122 + }, + "sound_stopallsounds": { + "opcode": "sound_stopallsounds", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 253, + "y": 245 + }, + "sound_changeeffectby": { + "opcode": "sound_changeeffectby", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": { + "EFFECT": [ + "PITCH", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 653, + "y": 14 + }, + "sound_seteffectto": { + "opcode": "sound_seteffectto", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 4, + "100" + ] + ] + }, + "fields": { + "EFFECT": [ + "PITCH", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 653, + "y": 139 + }, + "sound_cleareffects": { + "opcode": "sound_cleareffects", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 651, + "y": 242 + }, + "sound_changevolumeby": { + "opcode": "sound_changevolumeby", + "next": null, + "parent": null, + "inputs": { + "VOLUME": [ + 1, + [ + 4, + "-10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 645, + "y": 353 + }, + "sound_setvolumeto": { + "opcode": "sound_setvolumeto", + "next": null, + "parent": null, + "inputs": { + "VOLUME": [ + 1, + [ + 4, + "100" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1108, + "y": 5 + }, + "sound_volume": { + "opcode": "sound_volume", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1136, + "y": 123 + }, + "looks_sayforsecs": { + "opcode": "looks_sayforsecs", + "next": null, + "parent": null, + "inputs": { + "MESSAGE": [ + 1, + [ + 10, + "Hello!" + ] + ], + "SECS": [ + 1, + [ + 4, + "2" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 408, + "y": 91 + }, + "looks_say": { + "opcode": "looks_say", + "next": null, + "parent": null, + "inputs": { + "MESSAGE": [ + 1, + [ + 10, + "Hello!" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 413, + "y": 213 + }, + "looks_thinkforsecs": { + "opcode": "looks_thinkforsecs", + "next": null, + "parent": null, + "inputs": { + "MESSAGE": [ + 1, + [ + 10, + "Hmm..." + ] + ], + "SECS": [ + 1, + [ + 4, + "2" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 413, + "y": 317 + }, + "looks_think": { + "opcode": "looks_think", + "next": null, + "parent": null, + "inputs": { + "MESSAGE": [ + 1, + [ + 10, + "Hmm..." + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 412, + "y": 432 + }, + "looks_switchcostumeto": { + "opcode": "looks_switchcostumeto", + "next": null, + "parent": null, + "inputs": { + "COSTUME": [ + 1, + "8;bti4wv(iH9nkOacCJ|" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 411, + "y": 555 + }, + "looks_costume": { + "opcode": "looks_costume", + "next": null, + "parent": "Q#a,6LPWHqo9-0Nu*[SV", + "inputs": {}, + "fields": { + "COSTUME": [ + "costume2", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "looks_nextcostume": { + "opcode": "looks_nextcostume", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 419, + "y": 687 + }, + "looks_switchbackdropto": { + "opcode": "looks_switchbackdropto", + "next": null, + "parent": null, + "inputs": { + "BACKDROP": [ + 1, + "-?yeX}29V*wd6W:unW0i" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 901, + "y": 91 + }, + "looks_backdrops": { + "opcode": "looks_backdrops", + "next": null, + "parent": "`Wm^p~l[(IWzc1|wNv*.", + "inputs": {}, + "fields": { + "BACKDROP": [ + "backdrop1", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "looks_changesizeby": { + "opcode": "looks_changesizeby", + "next": null, + "parent": null, + "inputs": { + "CHANGE": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 895, + "y": 192 + }, + "looks_setsizeto": { + "opcode": "looks_setsizeto", + "next": null, + "parent": null, + "inputs": { + "SIZE": [ + 1, + [ + 4, + "100" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 896, + "y": 303 + }, + "looks_changeeffectby": { + "opcode": "looks_changeeffectby", + "next": null, + "parent": null, + "inputs": { + "CHANGE": [ + 1, + [ + 4, + "25" + ] + ] + }, + "fields": { + "EFFECT": [ + "COLOR", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 892, + "y": 416 + }, + "looks_seteffectto": { + "opcode": "looks_seteffectto", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": { + "EFFECT": [ + "COLOR", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 902, + "y": 527 + }, + "looks_cleargraphiceffects": { + "opcode": "looks_cleargraphiceffects", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 902, + "y": 638 + }, + "looks_show": { + "opcode": "looks_show", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 908, + "y": 758 + }, + "looks_hide": { + "opcode": "looks_hide", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 455, + "y": 861 + }, + "looks_gotofrontback": { + "opcode": "looks_gotofrontback", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "FRONT_BACK": [ + "front", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 853, + "y": 878 + }, + "looks_goforwardbackwardlayers": { + "opcode": "looks_goforwardbackwardlayers", + "next": null, + "parent": null, + "inputs": { + "NUM": [ + 1, + [ + 7, + "1" + ] + ] + }, + "fields": { + "FORWARD_BACKWARD": [ + "forward", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 851, + "y": 999 + }, + "looks_costumenumbername": { + "opcode": "looks_costumenumbername", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "NUMBER_NAME": [ + "number", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 458, + "y": 1007 + }, + "looks_backdropnumbername": { + "opcode": "looks_backdropnumbername", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "NUMBER_NAME": [ + "number", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 1242, + "y": 753 + }, + "looks_size": { + "opcode": "looks_size", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1249, + "y": 876 + } +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/boolean_blocks.json b/scratch_VLM/scratch_agent/blocks/boolean_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..9f7f4c0127bb5b5e93d02f4653b2c20c42c45d96 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/boolean_blocks.json @@ -0,0 +1,233 @@ +{ + "block_category": "Boolean Blocks", + "description": "Boolean blocks are hexagonal in shape. They represent conditions that evaluate to either 'true' or 'false' and are typically used as inputs for control flow blocks.", + "blocks": [ + { + "block_name": "<() < ()>", + "block_type": "operator", + "op_code": "operator_lt", + "block_shape": "Boolean Block", + "functionality": "Checks if the first value is less than the second.", + "inputs": [ + {"name": "OPERAND1", "type": "any"}, + {"name": "OPERAND2", "type": "any"} + ], + "example_standalone": "<(score) < (10)>", + "example_with_other_blocks": [ + { + "script": "if <(score) < (10)> then\n say [Keep trying!]", + "explanation": "This script causes the sprite to say 'Keep trying!' if the 'score' variable is less than 10." + } + ] + }, + { + "block_name": "<() = ()>", + "block_type": "operator", + "op_code": "operator_equals", + "block_shape": "Boolean Block", + "functionality": "Checks if two values are equal.", + "inputs": [ + {"name": "OPERAND1", "type": "any"}, + {"name": "OPERAND2", "type": "any"} + ], + "example_standalone": "<(answer) = (5)>", + "example_with_other_blocks": [ + { + "script": "if <(answer) = (5)> then\n say [Correct!]", + "explanation": "This script makes the sprite say 'Correct!' if the value of the 'answer' variable is exactly 5." + } + ] + }, + { + "block_name": "<() > ()>", + "block_type": "operator", + "op_code": "operator_gt", + "block_shape": "Boolean Block", + "functionality": "Checks if the first value is greater than the second.", + "inputs": [ + {"name": "OPERAND1", "type": "any"}, + {"name": "OPERAND2", "type": "any"} + ], + "example_standalone": "<(health) > (0)>", + "example_with_other_blocks": [ + { + "script": "if <(health) > (0)> then\n move (10) steps\nelse\n stop [all v]\nend", + "explanation": "This script moves the sprite if its 'health' is greater than 0; otherwise, it stops all scripts." + } + ] + }, + { + "block_name": "<<> and <>>", + "block_type": "operator", + "op_code": "operator_and", + "block_shape": "Boolean Block", + "functionality": "Returns 'true' if both provided Boolean conditions are 'true'.", + "inputs": [ + {"name": "OPERAND1", "type": "boolean"}, + {"name": "OPERAND2", "type": "boolean"} + ], + "example_standalone": "< and >", + "example_with_other_blocks": [ + { + "script": "if < and > then\n say [You're clicking me!]", + "explanation": "This script makes the sprite say 'You're clicking me!' only if the mouse button is pressed AND the mouse pointer is touching the sprite." + } + ] + }, + { + "block_name": "<<> or <>>", + "block_type": "operator", + "op_code": "operator_or", + "block_shape": "Boolean Block", + "functionality": "Returns 'true' if at least one of the provided Boolean conditions is 'true'.", + "inputs": [ + {"name": "OPERAND1", "type": "boolean"}, + {"name": "OPERAND2", "type": "boolean"} + ], + "example_standalone": "< or >", + "example_with_other_blocks": [ + { + "script": "if < or > then\n change x by (-10)\nend", + "explanation": "This script moves the sprite left if either the left arrow key OR the 'a' key is pressed." + } + ] + }, + { + "block_name": ">", + "block_type": "operator", + "op_code": "operator_not", + "block_shape": "Boolean Block", + "functionality": "Returns 'true' if the provided Boolean condition is 'false', and 'false' if it is 'true'.", + "inputs": [ + {"name": "OPERAND", "type": "boolean"} + ], + "example_standalone": ">", + "example_with_other_blocks": [ + { + "script": "if > then\n say [I'm safe!]\nend", + "explanation": "This script makes the sprite say 'I'm safe!' if it is NOT touching 'Sprite2'." + } + ] + }, + { + "block_name": "<() contains ()?>", + "block_type": "operator", + "op_code": "operator_contains", + "block_shape": "Boolean Block", + "functionality": "Checks if one string contains another string.", + "inputs": [ + {"name": "STRING1", "type": "string"}, + {"name": "STRING2", "type": "string"} + ], + "example_standalone": "<[apple] contains [a]?>", + "example_with_other_blocks": [ + { + "script": "if <[answer] contains [yes]?> then\n say [Great!]", + "explanation": "This script makes the sprite say 'Great!' if the 'answer' variable contains the substring 'yes'." + } + ] + }, + { + "block_name": "", + "block_type": "Sensing", + "op_code": "sensing_touchingobject", + "block_shape": "Boolean Block", + "functionality": "Checks if its sprite is touching the mouse-pointer, edge, or another specified sprite.", + "inputs": [ + {"name": "TOUCHINGOBJECTMENU", "type": "dropdown", "options": ["mouse-pointer", "edge", "Sprite1", "..." ]} + ], + "example_standalone": "", + "example_with_other_blocks": [ + { + "script": "if then\n bounce off edge\nend", + "explanation": "This script makes the sprite reverse direction if it comes into contact with the edge of the stage." + } + ] + }, + { + "block_name": "", + "block_type": "Sensing", + "op_code": "sensing_touchingcolor", + "block_shape": "Boolean Block", + "functionality": "Checks whether its sprite is touching a specified color.", + "inputs": [ + {"name": "COLOR", "type": "color"} + ], + "example_standalone": "", + "example_with_other_blocks": [ + { + "script": "if then\n change [health v] by (-1)\nend", + "explanation": "This script decreases the 'health' variable by 1 if the sprite touches any red color on the stage." + } + ] + }, + { + "block_name": "", + "block_type": "Sensing", + "op_code": "sensing_coloristouchingcolor", + "block_shape": "Boolean Block", + "functionality": "Checks whether a specific color on its sprite is touching another specified color on the stage or another sprite.", + "inputs": [ + {"name": "COLOR1", "type": "color"}, + {"name": "COLOR2", "type": "color"} + ], + "example_standalone": "", + "example_with_other_blocks": [ + { + "script": "if then\n say [Collision!]\nend", + "explanation": "This script makes the sprite say 'Collision!' if a green part of the sprite touches a red color elsewhere in the project." + } + ] + }, + { + "block_name": "", + "block_type": "Sensing", + "op_code": "sensing_keypressed", + "block_shape": "Boolean Block", + "functionality": "Checks if a specified keyboard key is currently being pressed.", + "inputs": [ + {"name": "KEY_OPTION", "type": "dropdown", "options": ["space", "up arrow", "a", "..."]} + ], + "example_standalone": "", + "example_with_other_blocks": [ + { + "script": "forever\n if then\n broadcast [shoot v]\n end\nend", + "explanation": "This script continuously checks if the space key is pressed and, if so, sends a 'shoot' broadcast." + } + ] + }, + { + "block_name": "", + "block_type": "Sensing", + "op_code": "sensing_mousedown", + "block_shape": "Boolean Block", + "functionality": "Checks if the computer mouse's primary button is being clicked while the cursor is over the stage.", + "inputs": null, + "example_standalone": "", + "example_with_other_blocks": [ + { + "script": "if then\n go to mouse-pointer\nend", + "explanation": "This script makes the sprite follow the mouse pointer only when the mouse button is held down." + } + ] + }, + { + "block_name": "<[my list v] contains ()?>", + "block_type": "Data", + "op_code": "data_listcontainsitem", + "block_shape": "Boolean Block", + "functionality": "Checks if a list includes a specific item.", + "inputs": [ + {"name": "LIST", "type": "dropdown"}, + {"name": "ITEM", "type": "any"} + ], + "example_standalone": "<[inventory v] contains [key]?>", + "example_with_other_blocks": [ + { + "script": "if <[inventory v] contains [key]?> then\n say [You have the key!]", + "explanation": "This script makes the sprite say 'You have the key!' if the 'inventory' list contains the item 'key'." + } + ] + } + ] +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/c_blocks.json b/scratch_VLM/scratch_agent/blocks/c_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..f993bec6b5cf5f88e132a2f3f70c484dee82e656 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/c_blocks.json @@ -0,0 +1,101 @@ +{ + "block_category": "C Blocks", + "description": "C blocks are shaped like the letter 'C'. They are used to loop or conditionally execute blocks that are placed within their opening, managing the flow of scripts.", + "blocks": [ + { + "block_name": "repeat ()", + "block_type": "Control", + "block_shape": "C-Block", + "op_code": "control_repeat", + "functionality": "Repeats the blocks inside it a specified number of times.", + "inputs": [ + { + "name": "times", + "type": "number" + } + ], + "example_standalone": "repeat (10)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n repeat (5)\n move (10) steps\n wait (0.1) seconds\n end", + "explanation": "This script makes the sprite move 10 steps five times, with a short pause after each movement." + } + ] + }, + { + "block_name": "forever", + "block_type": "Control", + "block_shape": "C-Block", + "op_code": "control_forever", + "functionality": "Continuously runs the blocks inside it.", + "inputs": null, + "example_standalone": "forever", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n forever\n move (5) steps\n if on edge, bounce\n end", + "explanation": "This script makes the sprite move endlessly and bounce off the edges of the stage, creating continuous motion." + } + ] + }, + { + "block_name": "if () then", + "block_type": "Control", + "block_shape": "C-Block", + "op_code": "control_if", + "functionality": "Executes the blocks inside it only if the specified boolean condition is true.", + "inputs": [ + { + "name": "condition", + "type": "boolean" + } + ], + "example_standalone": "if then", + "example_with_other_blocks": [ + { + "script": "forever\n if then\n stop [this script v]\n end", + "explanation": "This script continuously checks if the sprite is touching a red color, and if so, it stops the current script." + } + ] + }, + { + "block_name": "if () then else", + "block_type": "Control", + "block_shape": "C-Block", + "op_code": "control_if_else", + "functionality": "Executes one set of blocks if the specified boolean condition is true, and a different set of blocks if the condition is false.", + "inputs": [ + { + "name": "condition", + "type": "boolean" + } + ], + "example_standalone": "if (10)> then else", + "example_with_other_blocks": [ + { + "script": "if (10)> then\n say [You win!] for (2) seconds\nelse\n say [Keep trying!] for (2) seconds\nend", + "explanation": "This script checks the 'score'. If the score is greater than 10, it says 'You win!'; otherwise, it says 'Keep trying!'." + } + ] + }, + { + "block_name": "repeat until ()", + "block_type": "Control", + "block_shape": "C-Block", + "op_code": "control_repeat_until", + "functionality": "Repeats the blocks inside it until the specified boolean condition becomes true.", + "inputs": [ + { + "name": "condition", + "type": "boolean" + } + ], + "example_standalone": "repeat until ", + "example_with_other_blocks": [ + { + "script": "repeat until \n move (5) steps\nend", + "explanation": "This script makes the sprite move 5 steps repeatedly until it touches the edge of the stage." + } + ] + } + ] +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/cap_blocks.json b/scratch_VLM/scratch_agent/blocks/cap_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..bed1e81e1aeee80e3bcaa94581b2247b34481258 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/cap_blocks.json @@ -0,0 +1,53 @@ +{ + "block_category": "Cap Blocks", + "description": "Cap blocks have a notch at the top and a flat bottom. They signify the end of a script, preventing any further blocks from being placed below them, and are used to terminate scripts or specific actions.", + "blocks": [ + { + "block_name": "stop [v]", + "block_type": "Control", + "block_shape": "Cap Block (dynamic: can be Stack)", + "op_code": "control_stop", + "functionality": "Halts all scripts, only the current script, or other scripts within the same sprite. Its shape can dynamically change based on the selected option.", + "inputs": [ + {"name": "option", "type": "dropdown", "options": ["all"]} + ], + "example_standalone": "stop [all v]", + "example_with_other_blocks": [ + { + "script": "if <(health) = (0)> then\n stop [all v]\nend", + "explanation": "This script stops all running scripts in the project if the 'health' variable reaches 0, typically signifying a game over condition. [9, 15]" + } + ] + }, + { + "block_name": "delete this clone", + "block_type": "Control", + "block_shape": "Cap Block", + "op_code": "control_delete_this_clone", + "functionality": "Removes the clone that is executing it from the stage.", + "inputs":null, + "example_standalone": "delete this clone", + "example_with_other_blocks": [ + { + "script": "when I start as a clone\n wait until \n delete this clone", + "explanation": "This script, run by a clone, causes the clone to disappear from the stage once it touches the edge. [1]" + } + ] + }, + { + "block_name": "forever", + "block_type": "Control", + "block_shape": "Cap Block", + "op_code": "control_forever", + "functionality": "Continuously runs the blocks inside it.", + "inputs": null, + "example_standalone": "forever", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n forever\n move (5) steps\n if on edge, bounce\n end", + "explanation": "This script makes the sprite move endlessly and bounce off the edges of the stage, creating continuous motion." + } + ] + } + ] +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/excel_contetn/hat_block.xlsx b/scratch_VLM/scratch_agent/blocks/excel_contetn/hat_block.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..f009f687b0cf267472c9865a350d3d17773c0b31 Binary files /dev/null and b/scratch_VLM/scratch_agent/blocks/excel_contetn/hat_block.xlsx differ diff --git a/scratch_VLM/scratch_agent/blocks/excel_contetn/stack_block.xlsx b/scratch_VLM/scratch_agent/blocks/excel_contetn/stack_block.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..982959de2701e18571537b12bfbccd2aaa42c3a6 Binary files /dev/null and b/scratch_VLM/scratch_agent/blocks/excel_contetn/stack_block.xlsx differ diff --git a/scratch_VLM/scratch_agent/blocks/hat_blocks.json b/scratch_VLM/scratch_agent/blocks/hat_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..d2c54522059ed948a298c912295b2fb598c43cef --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/hat_blocks.json @@ -0,0 +1,205 @@ +{ + "block_category": "Hat Blocks", + "description": "Hat blocks are characterized by a rounded top and a bump at the bottom. They initiate scripts, meaning they are the starting point for a sequence of interconnected blocks.", + "blocks": [ + { + "block_name": "when green flag pressed", + "block_type": "Events", + "op_code": "event_whenflagclicked", + "block_shape": "Hat Block", + "functionality": "This Hat block initiates the script when the green flag is clicked, serving as the common starting point for most Scratch projects.", + "inputs": null, + "example_standalone": "when green flag clicked", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n go to x: (0) y: (0)\n say [Hello!] for (2) seconds", + "explanation": "This script makes the sprite go to the center of the stage and then say 'Hello!' for 2 seconds when the green flag is clicked." + } + ] + }, + { + "block_name": "when () key pressed", + "block_type": "Events", + "op_code": "event_whenkeypressed", + "block_shape": "Hat Block", + "functionality": "This Hat block initiates the script when a specified keyboard key is pressed.", + "inputs": [ + { + "name": "key", + "type": "dropdown", + "options": [ + "space", + "up arrow", + "down arrow", + "right arrow", + "left arrow", + "any", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9" + ] + } + ], + "example_standalone": "when [space v] key pressed", + "example_with_other_blocks": [ + { + "script": "when [right arrow v] key pressed\n point in direction (90)\n move (10) steps", + "explanation": "This script moves the sprite right when the right arrow key is pressed." + } + ] + }, + { + "block_name": "when this sprite clicked", + "block_type": "Events", + "op_code": "event_whenthisspriteclicked", + "block_shape": "Hat Block", + "functionality": "This Hat block starts the script when the sprite itself is clicked.", + "inputs": null, + "example_standalone": "when this sprite clicked", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked\n say [Ouch!] for (1) seconds\n change [score v] by (-1)", + "explanation": "This script makes the sprite say 'Ouch!' and decreases the score by 1 when the sprite is clicked." + } + ] + }, + { + "block_name": "when backdrop switches to ()", + "block_type": "Events", + "op_code": "event_whenbackdropswitchesto", + "block_shape": "Hat Block", + "functionality": "This Hat block triggers the script when the stage backdrop changes to a specified backdrop.", + "inputs": [ + { + "name": "backdrop name", + "type": "dropdown", + "options": ["backdrop1", "backdrop2", "..."] + } + ], + "example_standalone": "when backdrop switches to [game over v]", + "example_with_other_blocks": [ + { + "script": "when backdrop switches to [game over v]\n stop [all v]", + "explanation": "This script stops all running processes when the backdrop changes to 'game over'." + } + ] + }, + { + "block_name": "when () > ()", + "block_type": "Events", + "op_code": "event_whengreaterthan", + "block_shape": "Hat Block", + "functionality": "This Hat block starts the script when a certain value (e.g., loudness from a microphone, or the timer) exceeds a defined threshold.", + "inputs": [ + { + "name": "value type", + "type": "dropdown", + "options": [ + "loudness", + "timer" + ] + }, + { + "name": "threshold", + "type": "number" + } + ], + "example_standalone": "when [loudness v] > (70)", + "example_with_other_blocks": [ + { + "script": "when [loudness v] > (70)\n start sound [scream v]", + "explanation": "This script starts a 'scream' sound when the microphone loudness exceeds 70." + } + ] + }, + { + "block_name": "when I receive ()", + "block_type": "Events", + "op_code": "event_whenbroadcastreceived", + "block_shape": "Hat Block", + "functionality": "This Hat block initiates the script upon the reception of a specific broadcast message. This mechanism facilitates indirect communication between sprites or the stage.", + "inputs": [ + { + "name": "message name", + "type": "dropdown", + "options": ["message1", "message2", "new message..."] + } + ], + "example_standalone": "when I receive [start game v]", + "example_with_other_blocks": [ + { + "script": "when I receive [start game v]\n show\n go to x: (0) y: (0)", + "explanation": "This script makes the sprite visible and moves it to the center of the stage when it receives the 'start game' broadcast." + } + ] + }, + { + "block_name": "When I Start as a Clone", + "block_type": "Control", + "op_code": "control_start_as_clone", + "block_shape": "Hat Block", + "functionality": "This Hat block initiates the script when a clone of the sprite is created. It defines the behavior of individual clones.", + "inputs": null, + "example_standalone": "When I Start as a Clone", + "example_with_other_blocks": [ + { + "script": "when I start as a clone\n go to x: (pick random -240 to 240) y: (pick random -180 to 180)\n show\n forever\n move (10) steps\n if on edge, bounce", + "explanation": "This script makes a newly created clone appear at a random position, become visible, and then continuously move 10 steps, bouncing if it hits an edge." + } + ] + }, + { + "block_name": "define [my custom block]", + "block_type": "My Blocks", + "op_code": "procedures_definition", + "block_shape": "Hat Block", + "functionality": "This Hat block serves as the definition header for a custom block's script. It allows users to define reusable sequences of code by specifying the block's name and any input parameters it will accept. This promotes modularity and abstraction in projects.", + "inputs": [ + { + "name": "PROCCONTAINER", + "type": "block_prototype" + } + ], + "example_standalone": "define jump (height)", + "example_with_other_blocks": [ + { + "script": "define jump (height)\n change y by (height)\n wait (0.5) seconds\n change y by (0 - (height))\n\nwhen green flag clicked\n jump (50)", + "explanation": "This script first defines a custom block named 'jump' that takes a numerical input 'height'. The definition outlines the actions for jumping up and then down. Later, 'jump (50)' is called to make the sprite jump 50 units." + } + ] + } + ] +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/reporter_blocks.json b/scratch_VLM/scratch_agent/blocks/reporter_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..491dc3de5ebb5fa543e078426a68e331241e3662 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/reporter_blocks.json @@ -0,0 +1,709 @@ +{ + "block_category": "Reporter Blocks", + "description": "Reporter blocks have rounded edges. Their purpose is to report values, which can be numbers or strings, and are designed to fit into input slots of other blocks.", + "blocks": [ + { + "block_name": "x position", + "block_type": "Motion", + "op_code": "motion_xposition", + "block_shape": "Reporter Block", + "functionality": "Reports the current X-coordinate of the sprite.", + "inputs": null, + "example_standalone": "x position", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n say (x position) for (2) seconds", + "explanation": "This script makes the sprite say its current X-coordinate for 2 seconds." + } + ] + }, + { + "block_name": "y position", + "block_type": "Motion", + "op_code": "motion_yposition", + "block_shape": "Reporter Block", + "functionality": "Reports the current Y coordinate of the sprite on the stage.", + "inputs": null, + "example_standalone": "y position", + "example_with_other_blocks": [ + { + "script": "set [worms v] to (y position)", + "explanation": "This script assigns the sprite's current Y position to the 'worms' variable." + } + ] + }, + { + "block_name": "direction", + "block_type": "Motion", + "op_code": "motion_direction", + "block_shape": "Reporter Block", + "functionality": "Reports the current direction of the sprite in degrees (0 = up, 90 = right, 180 = down, -90 = left).", + "inputs": null, + "example_standalone": "direction", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n say (direction) for (2) seconds", + "explanation": "This script makes the sprite say its current direction in degrees for 2 seconds." + } + ] + }, + { + "block_name": "costume ()", + "block_type": "Looks", + "op_code": "looks_costumenumbername", + "block_shape": "Reporter Block", + "functionality": "Reports the current costume's number or name.", + "inputs": [ + { + "name": "NUMBER_NAME", + "type": "dropdown", + "options": [ + "number", + "name" + ] + } + ], + "example_standalone": "costume [number v]", + "example_with_other_blocks": [ + { + "script": "say join [I am costume ] (costume [name v])", + "explanation": "This script makes the sprite display its current costume name in a speech bubble." + } + ] + }, + { + "block_name": "size", + "block_type": "Looks", + "op_code": "looks_size", + "block_shape": "Reporter Block", + "functionality": "Reports the current size of the sprite as a percentage.", + "inputs": null, + "example_standalone": "size", + "example_with_other_blocks": [ + { + "script": "set size to ( (size) + (10) )", + "explanation": "This script increases the sprite's size by 10% from its current size." + } + ] + }, + { + "block_name": "backdrop ()", + "block_type": "Looks", + "op_code": "looks_backdropnumbername", + "block_shape": "Reporter Block", + "functionality": "Reports the current backdrop's number or name.", + "inputs": [ + { + "name": "NUMBER_NAME", + "type": "dropdown", + "options": [ + "number", + "name" + ] + } + ], + "example_standalone": "backdrop [number v]", + "example_with_other_blocks": [ + { + "script": "say join [Current backdrop: ] (backdrop [name v]) for (2) seconds", + "explanation": "This script makes the sprite say the name of the current stage backdrop for 2 seconds." + } + ] + }, + { + "block_name": "volume", + "block_type": "Sound", + "op_code": "sound_volume", + "block_shape": "Reporter Block", + "functionality": "Reports the current volume level of the sprite.", + "inputs": null, + "example_standalone": "volume", + "example_with_other_blocks": [ + { + "script": "say join [Current volume: ] (volume)", + "explanation": "This script makes the sprite display its current volume level in a speech bubble." + } + ] + }, + { + "block_name": "distance to ()", + "block_type": "Sensing", + "op_code": "sensing_distanceto", + "block_shape": "Reporter Block", + "functionality": "Reports the distance from the current sprite to the mouse-pointer or another specified sprite.", + "inputs": [ + { + "name": "target", + "type": "dropdown", + "options": ["mouse-pointer", "Sprite1", "Sprite2", "...", "_edge_"] + } + ], + "example_standalone": "distance to [mouse-pointer v]", + "example_with_other_blocks": [ + { + "script": "if <(distance to [Sprite2 v]) < (50)> then\n say [Too close!]", + "explanation": "This script makes the sprite say 'Too close!' if it is less than 50 steps away from 'Sprite2'." + } + ] + }, + { + "block_name": "answer", + "block_type": "Sensing", + "op_code": "sensing_answer", + "block_shape": "Reporter Block", + "functionality": "Holds the most recent text inputted using the 'Ask () and Wait' block.", + "inputs": null, + "example_standalone": "answer", + "example_with_other_blocks": [ + { + "script": "ask [What is your name?] and wait\n say join [Hello ] (answer)", + "explanation": "This script prompts the user for their name and then uses the 'answer' block to incorporate their input into a greeting." + } + ] + }, + { + "block_name": "mouse x", + "block_type": "Sensing", + "op_code": "sensing_mousex", + "block_shape": "Reporter Block", + "functionality": "Reports the mouse-pointer’s current X position on the stage.", + "inputs": null, + "example_standalone": "mouse x", + "example_with_other_blocks": [ + { + "script": "go to x: (mouse x) y: (mouse y)", + "explanation": "This script makes the sprite follow the mouse pointer's X and Y coordinates." + } + ] + }, + { + "block_name": "mouse y", + "block_type": "Sensing", + "op_code": "sensing_mousey", + "block_shape": "Reporter Block", + "functionality": "Reports the mouse-pointer’s current Y position on the stage.", + "inputs": null, + "example_standalone": "mouse y", + "example_with_other_blocks": [ + { + "script": "if <(mouse y) < (0)> then\n say [Below center]", + "explanation": "This script makes the sprite say 'Below center' if the mouse pointer is in the lower half of the stage." + } + ] + }, + { + "block_name": "loudness", + "block_type": "Sensing", + "op_code": "sensing_loudness", + "block_shape": "Reporter Block", + "functionality": "Reports the loudness of noise received by a microphone on a scale of 0 to 100.", + "inputs": null, + "example_standalone": "loudness", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n forever\n if <(loudness) > (30)> then\n start sound [pop v]", + "explanation": "This script continuously checks the microphone loudness and plays a 'pop' sound if it exceeds 30." + } + ] + }, + { + "block_name": "timer", + "block_type": "Sensing", + "op_code": "sensing_timer", + "block_shape": "Reporter Block", + "functionality": "Reports the elapsed time since Scratch was launched or the timer was reset, increasing by 1 every second.", + "inputs": null, + "example_standalone": "timer", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n reset timer\n wait (5) seconds\n say join [Time elapsed: ] (timer)", + "explanation": "This script resets the timer when the green flag is clicked, waits for 5 seconds, and then reports the elapsed time." + } + ] + }, + { + "block_name": "() of ()", + "block_type": "Sensing", + "op_code": "sensing_of", + "block_shape": "Reporter Block", + "functionality": "Reports a specified value (e.g., x position, direction, costume number) of a specified sprite or the Stage.", + "inputs": [ + { + "name": "value to report", + "type": "dropdown", + "options": [ + "x position", + "y position", + "direction", + "costume #", + "costume name", + "size", + "volume", + "backdrop #", + "backdrop name" + ] + }, + { + "name": "sprite/stage", + "type": "dropdown", + "options": ["Stage", "Sprite1", "Sprite2", "...", "_edge_"] + } + ], + "example_standalone": "x position of [Sprite1 v]", + "example_with_other_blocks": [ + { + "script": "set [other sprite X v] to ( (x position) of [Sprite2 v] )", + "explanation": "This script sets the 'other sprite X' variable to the current X-position of 'Sprite2'." + } + ] + }, + { + "block_name": "current ()", + "block_type": "Sensing", + "op_code": "sensing_current", + "block_shape": "Reporter Block", + "functionality": "Reports the current local year, month, date, day of the week, hour, minutes, or seconds.", + "inputs": [ + { + "name": "time unit", + "type": "dropdown", + "options": [ + "year", + "month", + "date", + "day of week", + "hour", + "minute", + "second" + ] + } + ], + "example_standalone": "current [hour v]", + "example_with_other_blocks": [ + { + "script": "say join [The current hour is ] (current [hour v])", + "explanation": "This script makes the sprite say the current hour." + } + ] + }, + { + "block_name": "days since 2000", + "block_type": "Sensing", + "op_code": "sensing_dayssince2000", + "block_shape": "Reporter Block", + "functionality": "Reports the number of days (and fractions of a day) since 00:00:00 UTC on January 1, 2000.", + "inputs": null, + "example_standalone": "days since 2000", + "example_with_other_blocks": [ + { + "script": "say join [Days passed: ] (days since 2000)", + "explanation": "This script makes the sprite display the number of days that have passed since January 1, 2000." + } + ] + }, + { + "block_name": "username", + "block_type": "Sensing", + "op_code": "sensing_username", + "block_shape": "Reporter Block", + "functionality": "Reports the username of the user currently logged into Scratch. If no user is logged in, it reports nothing.", + "inputs": null, + "example_standalone": "username", + "example_with_other_blocks": [ + { + "script": "say join [Hello, ] (username)", + "explanation": "This script makes the sprite greet the user by their Scratch username." + } + ] + }, + { + "block_name": "() + ()", + "block_type": "operator", + "op_code": "operator_add", + "block_shape": "Reporter Block", + "functionality": "Adds two numerical values.", + "inputs": [ + { + "name": "number1", + "type": "number" + }, + { + "name": "number2", + "type": "number" + } + ], + "example_standalone": "(5) + (3)", + "example_with_other_blocks": [ + { + "script": "set [total v] to ( (number 1) + (number 2) )", + "explanation": "This script calculates the sum of 'number 1' and 'number 2' and stores the result in the 'total' variable." + } + ] + }, + { + "block_name": "() - ()", + "block_type": "operator", + "op_code": "operator_subtract", + "block_shape": "Reporter Block", + "functionality": "Subtracts the second numerical value from the first.", + "inputs": [ + { + "name": "number1", + "type": "number" + }, + { + "name": "number2", + "type": "number" + } + ], + "example_standalone": "(10) - (4)", + "example_with_other_blocks": [ + { + "script": "set [difference v] to ( (number 1) - (number 2) )", + "explanation": "This script calculates the subtraction of 'number 2' from 'number 1' and stores the result in the 'difference' variable." + } + ] + }, + { + "block_name": "() * ()", + "block_type": "operator", + "op_code": "operator_multiply", + "block_shape": "Reporter Block", + "functionality": "Multiplies two numerical values.", + "inputs": [ + { + "name": "number1", + "type": "number" + }, + { + "name": "number2", + "type": "number" + } + ], + "example_standalone": "(6) * (7)", + "example_with_other_blocks": [ + { + "script": "set [area v] to ( (length) * (width) )", + "explanation": "This script calculates the area by multiplying 'length' and 'width' variables and stores it in the 'area' variable." + } + ] + }, + { + "block_name": "() / ()", + "block_type": "operator", + "op_code": "operator_divide", + "block_shape": "Reporter Block", + "functionality": "Divides the first numerical value by the second.", + "inputs": [ + { + "name": "number1", + "type": "number" + }, + { + "name": "number2", + "type": "number" + } + ], + "example_standalone": "(20) / (5)", + "example_with_other_blocks": [ + { + "script": "set [average v] to ( (total score) / (number of students) )", + "explanation": "This script calculates the average by dividing 'total score' by 'number of students' and stores it in the 'average' variable." + } + ] + }, + { + "block_name": "pick random () to ()", + "block_type": "operator", + "op_code": "operator_random", + "block_shape": "Reporter Block", + "functionality": "Generates a random integer within a specified inclusive range.", + "inputs": [ + { + "name": "min", + "type": "number" + }, + { + "name": "max", + "type": "number" + } + ], + "example_standalone": "pick random (1) to (10)", + "example_with_other_blocks": [ + { + "script": "go to x: (pick random -240 to 240) y: (pick random -180 to 180)", + "explanation": "This script moves the sprite to a random position on the stage." + } + ] + }, + { + "block_name": "join ()()", + "block_type": "operator", + "op_code": "operator_join", + "block_shape": "Reporter Block", + "functionality": "Concatenates two strings or values into a single string.", + "inputs": [ + { + "name": "string1", + "type": "string/number" + }, + { + "name": "string2", + "type": "string/number" + } + ], + "example_standalone": "join [Hello ][World!]", + "example_with_other_blocks": [ + { + "script": "say join [Hello ][World!]", + "explanation": "This script makes the sprite display 'Hello World!' in a speech bubble by joining two string literals." + } + ] + }, + { + "block_name": "letter () of ()", + "block_type": "operator", + "op_code": "operator_letterof", + "block_shape": "Reporter Block", + "functionality": "Reports the character at a specific numerical position within a string.", + "inputs": [ + { + "name": "index", + "type": "number" + }, + { + "name": "text", + "type": "string" + } + ], + "example_standalone": "letter (1) of [apple]", + "example_with_other_blocks": [ + { + "script": "say letter (1) of [apple]", + "explanation": "This script makes the sprite display the first character of the string 'apple', which is 'a'." + } + ] + }, + { + "block_name": "length of ()", + "block_type": "operator", + "op_code": "operator_length", + "block_shape": "Reporter Block", + "functionality": "Reports the total number of characters in a given string.", + "inputs": [ + { + "name": "text", + "type": "string" + } + ], + "example_standalone": "length of [banana]", + "example_with_other_blocks": [ + { + "script": "say length of [banana]", + "explanation": "This script makes the sprite display the length of the string 'banana', which is 6." + } + ] + }, + { + "block_name": "() mod ()", + "block_type": "operator", + "op_code": "operator_mod", + "block_shape": "Reporter Block", + "functionality": "Reports the remainder when the first number is divided by the second.", + "inputs": [ + { + "name": "number1", + "type": "number" + }, + { + "name": "number2", + "type": "number" + } + ], + "example_standalone": "(10) mod (3)", + "example_with_other_blocks": [ + { + "script": "if <(number) mod (2) = (0)> then\n say [Even number]", + "explanation": "This script checks if a 'number' variable is even by checking if its remainder when divided by 2 is 0." + } + ] + }, + { + "block_name": "round ()", + "block_type": "operator", + "op_code": "operator_round", + "block_shape": "Reporter Block", + "functionality": "Rounds a numerical value to the nearest integer.", + "inputs": [ + { + "name": "number", + "type": "number" + } + ], + "example_standalone": "round (3.7)", + "example_with_other_blocks": [ + { + "script": "set [rounded score v] to (round (score))", + "explanation": "This script rounds the 'score' variable to the nearest whole number and stores it in 'rounded score'." + } + ] + }, + { + "block_name": "() of ()", + "block_type": "operator", + "op_code": "operator_mathop", + "block_shape": "Reporter Block", + "functionality": "Performs various mathematical functions (e.g., absolute value, square root, trigonometric functions).", + "inputs": [ + { + "name": "function type", + "type": "dropdown", + "options": [ + "abs", + "floor", + "ceiling", + "sqrt", + "sin", + "cos", + "tan", + "asin", + "acos", + "atan", + "ln", + "log", + "e ^", + "10 ^" + ] + }, + { + "name": "value", + "type": "number" + } + ], + "example_standalone": "([sqrt v] of (25))", + "example_with_other_blocks": [ + { + "script": "set [distance v] to ([sqrt v] of ( ( (x position) * (x position) ) + ( (y position) * (y position) ) ))", + "explanation": "This script calculates the distance from the origin (0,0) using the Pythagorean theorem and stores it in 'distance'." + } + ] + }, + { + "block_name": "variable", + "block_type": "Data", + "op_code": "data_variable", + "block_shape": "Reporter Block", + "functionality": "Provides the current value stored in a variable.", + "inputs": [ + { + "name": "variable name", + "type": "dropdown", + "options": ["my variable", "score", "..."] + } + ], + "example_standalone": "score", + "example_with_other_blocks": [ + { + "script": "say (score) for (2) seconds", + "explanation": "This script makes the sprite say the current value of the 'score' variable for 2 seconds." + } + ] + }, + { + "block_name": "list", + "block_type": "Data", + "op_code": "data_list", + "block_shape": "Reporter Block", + "functionality": "Reports the entire content of a specified list. When clicked in the editor, it displays the list as a monitor.", + "inputs": [ + { + "name": "list name", + "type": "dropdown", + "options": ["my list", "list2", "..."] + } + ], + "example_standalone": "my list", + "example_with_other_blocks": [ + { + "script": "say (my list)", + "explanation": "This script makes the sprite say all the contents of 'my list'." + } + ] + }, + { + "block_name": "item () of ()", + "block_type": "Data", + "op_code": "data_itemoflist", + "block_shape": "Reporter Block", + "functionality": "Reports the item located at a specific position in a list.", + "inputs": [ + { + "name": "index/option", + "type": "number or dropdown", + "options": [ + "last", + "random" + ] + }, + { + "name": "list name", + "type": "dropdown", + "options": ["shopping list", "my list", "..."] + } + ], + "example_standalone": "item (1) of [shopping list v]", + "example_with_other_blocks": [ + { + "script": "say item (1) of [shopping list v]", + "explanation": "This script makes the sprite display the first item from the 'shopping list'." + } + ] + }, + { + "block_name": "length of () (list)", + "block_type": "Data", + "op_code": "data_lengthoflist", + "block_shape": "Reporter Block", + "functionality": "Provides the total number of items contained in a list.", + "inputs": [ + { + "name": "list name", + "type": "dropdown", + "options": ["my list", "shopping list", "..."] + } + ], + "example_standalone": "length of [my list v]", + "example_with_other_blocks": [ + { + "script": "say join (length of [shopping list v]) [ items in the list.]", + "explanation": "This script makes the sprite display the total number of items currently in the 'shopping list'." + } + ] + }, + { + "block_name": "item # of () in ()", + "block_type": "Data", + "op_code": "data_itemnumoflist", + "block_shape": "Reporter Block", + "functionality": "Reports the index number of the first occurrence of a specified item in a list. If the item is not found, it reports 0.", + "inputs": [ + { + "name": "item", + "type": "string/number" + }, + { + "name": "list name", + "type": "dropdown", + "options": ["my list", "shopping list", "..."] + } + ], + "example_standalone": "item # of [apple] in [shopping list v]", + "example_with_other_blocks": [ + { + "script": "if <(item # of [banana] in [my list v]) > (0)> then\n say join [Banana found at position ] (item # of [banana] in [my list v])", + "explanation": "This script checks if 'banana' is in 'my list' and, if so, reports its position." + } + ] + } + ] +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks/stack_blocks.json b/scratch_VLM/scratch_agent/blocks/stack_blocks.json new file mode 100644 index 0000000000000000000000000000000000000000..ffa63ca43bce4270364ebdec57bef4d178027e58 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks/stack_blocks.json @@ -0,0 +1,1316 @@ +{ + "block_category": "Stack Blocks", + "description": "Stack blocks are the most common block shape, featuring a notch at the top and a bump at the bottom. They perform the main commands within a script and can connect both above and below them.", + "blocks": [ + { + "block_name": "move () steps", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_movesteps", + "functionality": "Moves the sprite forward by the specified number of steps in the direction it is currently facing. A positive value moves it forward, and a negative value moves it backward.", + "inputs": [ + { + "name": "STEPS", + "type": "number" + } + ], + "example_standalone": "move () steps", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n go to x: (0) y: (0)\n point in direction (90)\n move (50) steps", + "explanation": "This script first places the sprite at the center of the stage, points it to the right (90 degrees), and then moves it 50 steps in that direction." + } + ] + }, + { + "block_name": "turn right () degrees", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_turnright", + "functionality": "Turns the sprite clockwise by the specified number of degrees.", + "inputs": [ + { + "name": "DEGREES", + "type": "number" + } + ], + "example_standalone": "turn (clockwise icon) (15) degrees", + "example_with_other_blocks": [ + { + "script": "when [right arrow v] key pressed\n turn (clockwise icon) (15) degrees", + "explanation": "This script makes the sprite turn clockwise by 15 degrees every time the right arrow key is pressed." + }, + { + "script": "when green flag clicked\n forever\n turn (clockwise icon) (15) degrees\n wait (0.5) seconds\n end", + "explanation": "This script makes the sprite continuously spin clockwise by 15 degrees every half second." + } + ] + }, + { + "block_name": "turn left () degrees", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_turnleft", + "functionality": "Turns the sprite counter-clockwise by the specified number of degrees.", + "inputs": [ + { + "name": "DEGREES", + "type": "number" + } + ], + "example_standalone": "turn (counter-clockwise icon) (15) degrees", + "example_with_other_blocks": [ + { + "script": "when [left arrow v] key pressed\n turn (counter-clockwise icon) (15) degrees", + "explanation": "This script makes the sprite turn counter-clockwise by 15 degrees every time the left arrow key is pressed." + }, + { + "script": "when green flag clicked\n forever\n turn (counter-clockwise icon) (15) degrees\n wait (0.5) seconds\n end", + "explanation": "This script makes the sprite continuously spin counter-clockwise by 15 degrees every half second." + } + ] + }, + { + "block_name": "go to ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_goto", + "functionality": "Moves the sprite to a specified location, which can be a random position, the mouse pointer, or another sprite.", + "inputs": [ + { + "name": "TO", + "type": "dropdown", + "options": [ + "random position", + "mouse-pointer", + "sprite1", + "..." + ] + } + ], + "example_standalone": "go to [random position v]", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked\n go to [mouse-pointer v]", + "explanation": "This script moves the sprite to the current position of the mouse pointer whenever the sprite is clicked." + } + ] + }, + { + "block_name": "go to x: () y: ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_gotoxy", + "functionality": "Moves the sprite to the specified X and Y coordinates on the stage.", + "inputs": [ + { + "name": "X", + "type": "number" + }, + { + "name": "Y", + "type": "number" + } + ], + "example_standalone": "go to x: (0) y: (0)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n go to x: (0) y: (0)\n say [Ready to start!] for (1) seconds", + "explanation": "This script positions the sprite at the center of the stage at the beginning of the project and then makes it say 'Ready to start!'." + } + ] + }, + { + "block_name": "glide () secs to ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_glideto", + "functionality": "Glides the sprite smoothly to a specified location (random position, mouse pointer, or another sprite) over a given number of seconds.", + "inputs": [ + { + "name": "SECS", + "type": "number" + }, + { + "name": "TO", + "type": "dropdown", + "options": [ + "random position", + "mouse-pointer", + "sprite1", + "..." + ] + } + ], + "example_standalone": "glide (1) secs to [random position v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n glide (1) secs to [mouse-pointer v]", + "explanation": "This script makes the sprite glide smoothly to the mouse pointer's position over 1 second when the green flag is clicked." + } + ] + }, + { + "block_name": "glide () secs to x: () y: ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_glidesecstoxy", + "functionality": "Glides the sprite smoothly to the specified X and Y coordinates over a given number of seconds.", + "inputs": [ + { + "name": "SECS", + "type": "number" + }, + { + "name": "X", + "type": "number" + }, + { + "name": "Y", + "type": "number" + } + ], + "example_standalone": "glide (1) secs to x: (100) y: (50)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n glide (2) secs to x: (150) y: (-100)\n glide (2) secs to x: (-150) y: (100)", + "explanation": "This script makes the sprite glide to two different points on the stage, taking 2 seconds for each movement." + } + ] + }, + { + "block_name": "point in direction ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_pointindirection", + "functionality": "Sets the sprite's direction to a specified angle in degrees (0 = up, 90 = right, 180 = down, -90 = left).", + "inputs": [ + { + "name": "DIRECTION", + "type": "number" + } + ], + "example_standalone": "point in direction (90)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n point in direction (0)\n move (100) steps", + "explanation": "This script makes the sprite point upwards (0 degrees) and then move 100 steps in that direction." + } + ] + }, + { + "block_name": "point towards ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_pointtowards", + "functionality": "Points the sprite towards the mouse pointer or another specified sprite.", + "inputs": [ + { + "name": "TOWARDS", + "type": "dropdown", + "options": [ + "mouse-pointer", + "sprite1", + "..." + ] + } + ], + "example_standalone": "point towards [mouse-pointer v]", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked\n point towards [mouse-pointer v]\n move (10) steps", + "explanation": "When the sprite is clicked, it will point towards the mouse pointer and then move 10 steps in that direction." + }, + { + "script": "when green flag clicked\n forever\n point towards [mouse-pointer v]\n move (5) steps\n end", + "explanation": "This script makes the sprite continuously follow the mouse pointer." + } + ] + }, + { + "block_name": "change x by ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_changexby", + "functionality": "Changes the sprite's X-coordinate by the specified amount, moving it horizontally.", + "inputs": [ + { + "name": "DX", + "type": "number" + } + ], + "example_standalone": "change x by (10)", + "example_with_other_blocks": [ + { + "script": "when [right arrow v] key pressed\n change x by (10)", + "explanation": "This script moves the sprite 10 steps to the right when the right arrow key is pressed." + } + ] + }, + { + "block_name": "set x to ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_setx", + "functionality": "Sets the sprite's X-coordinate to a specific value, placing it at a precise horizontal position.", + "inputs": [ + { + "name": "X", + "type": "number" + } + ], + "example_standalone": "set x to (0)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set x to (0)\n set y to (0)", + "explanation": "This script centers the sprite horizontally at the start of the project." + } + ] + }, + { + "block_name": "change y by ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_changeyby", + "functionality": "Changes the sprite's Y-coordinate by the specified amount, moving it vertically.", + "inputs": [ + { + "name": "DY", + "type": "number" + } + ], + "example_standalone": "change y by (10)", + "example_with_other_blocks": [ + { + "script": "when [up arrow v] key pressed\n change y by (10)", + "explanation": "This script moves the sprite 10 steps up when the up arrow key is pressed." + } + ] + }, + { + "block_name": "set y to ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_sety", + "functionality": "Sets the sprite's Y-coordinate to a specific value, placing it at a precise vertical position.", + "inputs": [ + { + "name": "Y", + "type": "number" + } + ], + "example_standalone": "set y to (0)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set x to (0)\n set y to (0)", + "explanation": "This script centers the sprite vertically at the start of the project." + } + ] + }, + { + "block_name": "if on edge, bounce", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_ifonedgebounce", + "functionality": "Reverses the sprite's direction if it touches the edge of the stage.", + "inputs": null, + "example_standalone": "if on edge, bounce", + "example_with_other_blocks": [ + { + "script": "when I receive [start moving v]\n repeat (50)\n move (5) steps\n if on edge, bounce\n end", + "explanation": "Upon receiving the 'start moving' broadcast, the sprite will move 5 steps repeatedly for 50 times, bouncing off edges if it touches them." + }, + { + "script": "when green flag clicked\n forever\n move (10) steps\n if on edge, bounce\n end", + "explanation": "This script makes the sprite move continuously and bounce off the edges of the stage." + } + ] + }, + { + "block_name": "set rotation style ()", + "block_type": "Motion", + "block_shape": "Stack Block", + "op_code": "motion_setrotationstyle", + "functionality": "Determines how the sprite rotates: 'left-right' (flips horizontally), 'don't rotate' (stays facing one direction), or 'all around' (rotates freely).", + "inputs": [ + { + "name": "STYLE", + "type": "dropdown", + "options": [ + "left-right", + "don't rotate", + "all around" + ] + } + ], + "example_standalone": "set rotation style [left-right v]", + "example_with_other_blocks": [ + { + "script": "when backdrop switches to [game level 1 v]\n set rotation style [all around v]", + "explanation": "When the backdrop changes to 'game level 1', the sprite's rotation style will be set to 'all around', allowing it to rotate freely." + }, + { + "script": "when green flag clicked\n set rotation style [left-right v]\n forever\n move (10) steps\n if on edge, bounce\n end", + "explanation": "This script makes the sprite move horizontally and flip its costume when it hits an edge, instead of rotating." + } + ] + }, + { + "block_name": "say () for () seconds", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_sayforsecs", + "functionality": "Displays a speech bubble containing specified text for a set duration.", + "inputs": [ + { + "name": "MESSAGE", + "type": "string" + }, + { + "name": "SECS", + "type": "number" + } + ], + "example_standalone": "say [Hello!] for (2) seconds", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n say [Grr] for (3) seconds\n say [Have you seen my honey?] for (3) seconds", + "explanation": "This script makes the sprite display two sequential speech bubbles with different messages and durations. First, it says 'Grr' for 3 seconds, then 'Have you seen my honey?' for another 3 seconds." + } + ] + }, + { + "block_name": "say ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_say", + "functionality": "Displays a speech bubble with the specified text indefinitely until another 'say' or 'think' block is activated.", + "inputs": [ + { + "name": "MESSAGE", + "type": "string" + } + ], + "example_standalone": "say [Hello!]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n say [Welcome to my game!]\n wait (2) seconds\n say []", + "explanation": "This script makes the sprite say 'Welcome to my game!' for 2 seconds, then clears the speech bubble." + } + ] + }, + { + "block_name": "think () for () seconds", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_thinkforsecs", + "functionality": "Displays a thought bubble containing specified text for a set duration.", + "inputs": [ + { + "name": "MESSAGE", + "type": "string" + }, + { + "name": "SECS", + "type": "number" + } + ], + "example_standalone": "think [Hmm...] for (2) seconds", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked\n think [What should I do?] for (2) seconds", + "explanation": "This script makes the sprite display a thought bubble saying 'What should I do?' for 2 seconds when clicked." + } + ] + }, + { + "block_name": "think ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_think", + "functionality": "Displays a thought bubble with the specified text indefinitely until another 'say' or 'think' block is activated.", + "inputs": [ + { + "name": "MESSAGE", + "type": "string" + } + ], + "example_standalone": "think [Got it!]", + "example_with_other_blocks": [ + { + "script": "when I receive [correct answer v]\n think [That's right!]\n wait (1) seconds\n think []", + "explanation": "This script makes the sprite think 'That's right!' for 1 second when a 'correct answer' broadcast is received, then clears the thought bubble." + } + ] + }, + { + "block_name": "switch costume to ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_switchcostumeto", + "functionality": "Alters the sprite's appearance to a designated costume.", + "inputs": [ + { + "name": "COSTUME", + "type": "dropdown/number" + } + ], + "example_standalone": "switch costume to [costume1 v]", + "example_with_other_blocks": [ + { + "script": "when I receive [explosion v]\n repeat (5)\n next costume\n end\n hide", + "explanation": "This script animates an explosion by rapidly switching costumes, then hides the sprite. [3]" + } + ] + }, + { + "block_name": "next costume", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_nextcostume", + "functionality": "Switches the sprite's costume to the next one in its costume list. If it's the last costume, it cycles back to the first.", + "inputs": null, + "example_standalone": "next costume", + "example_with_other_blocks": [ + { + "script": "when [space v] key pressed\n repeat (3)\n next costume\n wait (0.1) seconds\n end", + "explanation": "When the space key is pressed, the sprite will cycle through its next three costumes with a short delay between each change." + }, + { + "script": "when green flag clicked\n forever\n next costume\n wait (0.2) seconds\n end", + "explanation": "This script continuously animates the sprite by switching to the next costume every 0.2 seconds, creating a walking or flying effect." + } + ] + }, + { + "block_name": "switch backdrop to ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_switchbackdropto", + "functionality": "Changes the stage's backdrop to a specified backdrop.", + "inputs": [ + { + "name": "BACKDROP", + "type": "dropdown/number" + } + ], + "example_standalone": "switch backdrop to [backdrop1 v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n switch backdrop to [start screen v]", + "explanation": "This script sets the stage to a 'start screen' backdrop when the project begins." + } + ] + }, + { + "block_name": "switch backdrop to () and wait", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_switchbackdroptowait", + "functionality": "Changes the stage's backdrop to a specified backdrop and pauses the script until any 'When backdrop switches to' scripts for that backdrop have finished.", + "inputs": [ + { + "name": "BACKDROP", + "type": "dropdown/number" + } + ], + "example_standalone": "switch backdrop to [game over v] and wait", + "example_with_other_blocks": [ + { + "script": "broadcast [game over v]\n switch backdrop to [game over v] and wait\n stop [all v]", + "explanation": "This script broadcasts a 'game over' message, then changes the backdrop to 'game over' and waits for any associated scripts to finish before stopping all processes." + } + ] + }, + { + "block_name": "next backdrop", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_nextbackdrop", + "functionality": "Switches the stage's backdrop to the next one in its backdrop list. If it's the last backdrop, it cycles back to the first.", + "inputs": null, + "example_standalone": "next backdrop", + "example_with_other_blocks": [ + { + "script": "when [space v] key pressed\n next backdrop", + "explanation": "This script changes the stage to the next backdrop in the list each time the space key is pressed." + } + ] + }, + { + "block_name": "change size by ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_changesizeby", + "functionality": "Changes the sprite's size by a specified percentage. Positive values make it larger, negative values make it smaller.", + "inputs": [ + { + "name": "CHANGE", + "type": "number" + } + ], + "example_standalone": "change size by (10)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n repeat (10)\n change size by (5)\n wait (0.1) seconds\n end", + "explanation": "This script makes the sprite gradually grow larger over 10 steps, with a short pause between each size change." + } + ] + }, + { + "block_name": "set size to () %", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_setsizeto", + "functionality": "Sets the sprite's size to a specific percentage of its original size.", + "inputs": [ + { + "name": "SIZE", + "type": "number" + } + ], + "example_standalone": "set size to (100) %", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set size to (50) %\n wait (1) seconds\n set size to (100) %", + "explanation": "This script makes the sprite shrink to half its original size at the start, waits for 1 second, then returns to its original size." + } + ] + }, + { + "block_name": "change () effect by ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_changeeffectby", + "functionality": "Changes a visual effect on the sprite by a specified amount (e.g., color, fisheye, whirl, pixelate, mosaic, brightness, ghost).", + "inputs": [ + { + "name": "EFFECT", + "type": "dropdown", + "options": [ + "color", + "fisheye", + "whirl", + "pixelate", + "mosaic", + "brightness", + "ghost" + ] + }, + { + "name": "CHANGE", + "type": "number" + } + ], + "example_standalone": "change [color v] effect by (25)", + "example_with_other_blocks": [ + { + "script": "when loudness > (10)\n change [fisheye v] effect by (5)", + "explanation": "When the loudness detected by the microphone is greater than 10, the sprite's fisheye effect will increase by 5." + }, + { + "script": "when green flag clicked\n forever\n change [color v] effect by (5)\n wait (0.1) seconds\n end", + "explanation": "This script makes the sprite continuously cycle through different colors." + } + ] + }, + { + "block_name": "set () effect to ()", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_seteffectto", + "functionality": "Sets a visual effect on the sprite to a specific value.", + "inputs": [ + { + "name": "EFFECT", + "type": "dropdown", + "options": [ + "color", + "fisheye", + "whirl", + "pixelate", + "mosaic", + "brightness", + "ghost" + ] + }, + { + "name": "VALUE", + "type": "number" + } + ], + "example_standalone": "set [ghost v] effect to (50)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set [ghost v] effect to (75)", + "explanation": "This script makes the sprite 75% transparent at the start of the project." + } + ] + }, + { + "block_name": "clear graphic effects", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_cleargraphiceffects", + "functionality": "Removes all visual effects applied to the sprite.", + "inputs": null, + "example_standalone": "clear graphic effects", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n change [color v] effect by (50)\n wait (2) seconds\n clear graphic effects", + "explanation": "This script changes the sprite's color effect, waits 2 seconds, then resets all graphic effects." + } + ] + }, + { + "block_name": "show", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_show", + "functionality": "Makes the sprite visible on the stage.", + "inputs": null, + "example_standalone": "show", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n hide\nwhen I receive [start game v]\n show", + "explanation": "This script hides the sprite at the beginning of the project and makes it visible when a 'start game' broadcast is received." + } + ] + }, + { + "block_name": "hide", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_hide", + "functionality": "Makes the sprite invisible on the stage.", + "inputs": null, + "example_standalone": "hide", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n hide", + "explanation": "This script hides the sprite from the stage when the green flag is clicked." + } + ] + }, + { + "block_name": "go to () layer", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_gotofrontback", + "functionality": "Moves the sprite to the front-most or back-most layer of other sprites on the stage.", + "inputs": [ + { + "name": "FRONT_BACK", + "type": "dropdown", + "options": [ + "front", + "back" + ] + } + ], + "example_standalone": "go to [front v] layer", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n go to [front v] layer", + "explanation": "This script ensures the sprite is always visible on top of other sprites at the start of the project." + } + ] + }, + { + "block_name": "go () layers", + "block_type": "Looks", + "block_shape": "Stack Block", + "op_code": "looks_goforwardbackwardlayers", + "functionality": "Moves the sprite forward or backward a specified number of layers in relation to other sprites.", + "inputs": [ + { + "name": "FORWARD_BACKWARD", + "type": "dropdown", + "options": [ + "forward", + "backward" + ] + }, + { + "name": "NUM", + "type": "number" + } + ], + "example_standalone": "go [forward v] (1) layers", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked go [forward v] (1) layers", + "explanation": "This script brings the clicked sprite one layer closer to the front." + } + ] + }, + { + "block_name": "play sound () until done", + "block_type": "Sound", + "block_shape": "Stack Block", + "op_code": "sound_playuntildone", + "functionality": "Plays a specified sound and pauses the script's execution until the sound has completed.", + "inputs": [ + { + "name": "sound name", + "type": "dropdown" + } + ], + "example_standalone": "play sound [Meow v] until done", + "example_with_other_blocks": [ + { + "script": "when backdrop switches to [winning screen v]\n play sound [fanfare v] until done\n say [You won!] for (2) seconds", + "explanation": "When the backdrop changes to the 'winning screen', a 'fanfare' sound will play until it finishes, and then the sprite will say 'You won!' for 2 seconds." + }, + { + "script": "forever\n play sound [Music v] until done", + "explanation": "This script creates a continuous loop for background music, playing the 'Music' sound repeatedly." + } + ] + }, + { + "block_name": "start sound ()", + "block_type": "Sound", + "block_shape": "Stack Block", + "op_code": "sound_start", + "functionality": "Initiates playback of a specified sound without pausing the script, allowing other actions to proceed concurrently.", + "inputs": [ + { + "name": "sound name", + "type": "dropdown" + } + ], + "example_standalone": "start sound [Pop v]", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked\n start sound [Pop v]\n change [score v] by (1)", + "explanation": "This script plays a 'Pop' sound and increments the score simultaneously when the sprite is clicked." + } + ] + }, + { + "block_name": "stop all sounds", + "block_type": "Sound", + "block_shape": "Stack Block", + "op_code": "sound_stopallsounds", + "functionality": "Stops all currently playing sounds.", + "inputs": null, + "example_standalone": "stop all sounds", + "example_with_other_blocks": [ + { + "script": "when I receive [game over v]\n stop all sounds", + "explanation": "This script stops any sounds currently playing when the 'game over' broadcast is received." + } + ] + }, + { + "block_name": "change volume by ()", + "block_type": "Sound", + "block_shape": "Stack Block", + "op_code": "sound_changevolumeby", + "functionality": "Changes the project's sound volume by a specified amount.", + "inputs": [ + { + "name": "change", + "type": "number" + } + ], + "example_standalone": "change volume by (-10)", + "example_with_other_blocks": [ + { + "script": "when [down arrow v] key pressed\n change volume by (-5)", + "explanation": "This script decreases the project's volume by 5 when the down arrow key is pressed." + } + ] + }, + { + "block_name": "set volume to () %", + "block_type": "Sound", + "block_shape": "Stack Block", + "op_code": "sound_setvolumeto", + "functionality": "Sets the sound volume to a specific percentage (0-100).", + "inputs": [ + { + "name": "percentage", + "type": "number" + } + ], + "example_standalone": "set volume to (100) %", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set volume to (50) %", + "explanation": "This script sets the project's volume to 50% when the green flag is clicked." + } + ] + }, + { + "block_name": "broadcast ()", + "block_type": "Events", + "block_shape": "Stack Block", + "op_code": "event_broadcast", + "functionality": "Sends a broadcast message throughout the Scratch program, activating any 'when I receive ()' blocks that are set to listen for that message, enabling indirect communication.", + "inputs": [ + { + "name": "message name", + "type": "string/dropdown" + } + ], + "example_standalone": "broadcast [start game v]", + "example_with_other_blocks": [ + { + "script": "if then\n broadcast [jump v]\nend", + "explanation": "This script sends a 'jump' message to other scripts or sprites when the space key is pressed." + } + ] + }, + { + "block_name": "broadcast () and wait", + "block_type": "Events", + "block_shape": "Stack Block", + "op_code": "event_broadcastandwait", + "functionality": "Sends a broadcast message and pauses the current script until all other scripts activated by that broadcast have completed their execution, ensuring sequential coordination.", + "inputs": [ + { + "name": "message name", + "type": "string/dropdown" + } + ], + "example_standalone": "broadcast [initialize sprites v] and wait", + "example_with_other_blocks": [ + { + "script": "broadcast [initialize sprites v] and wait\n say [Game Started!] for (2) seconds", + "explanation": "This script ensures all sprite initialization routines complete before displaying 'Game Started!' for 2 seconds." + } + ] + }, + { + "block_name": "wait () seconds", + "block_type": "Control", + "block_shape": "Stack Block", + "op_code": "control_wait", + "functionality": "Pauses the script for a specified duration.", + "inputs": [ + { + "name": "seconds", + "type": "number" + } + ], + "example_standalone": "wait (1) seconds", + "example_with_other_blocks": [ + { + "script": "say [Hello!] for (1) seconds\n wait (0.5) seconds\n say [Goodbye!] for (1) seconds", + "explanation": "This script creates a timed dialogue sequence, pausing for 0.5 seconds between two speech bubbles." + } + ] + }, + { + "block_name": "wait until ()", + "block_type": "Control", + "block_shape": "Stack Block", + "op_code": "control_wait_until", + "functionality": "Pauses the script until the specified boolean condition becomes true.", + "inputs": [ + { + "name": "condition", + "type": "boolean" + } + ], + "example_standalone": "wait until ", + "example_with_other_blocks": [ + { + "script": "wait until \n start sound [pop v]", + "explanation": "This script pauses until the space key is pressed, then plays a 'pop' sound." + } + ] + }, + { + "block_name": "stop ()", + "block_type": "Control", + "block_shape": "Stack Block", + "op_code": "control_stop", + "functionality": "Stops all scripts, this script, or other scripts in the sprite. Becomes a Cap Block if 'all' or 'this script' is selected in the dropdown menu.", + "inputs": [ + { + "name": "option", + "type": "dropdown", + "options": [ + "all", + "this script", + "other scripts in sprite" + ] + } + ], + "example_standalone": "stop [all v]", + "example_with_other_blocks": [ + { + "script": "if then\n stop [all v]\nend", + "explanation": "This script stops the entire project if the 'score' variable becomes 0." + } + ] + }, + { + "block_name": "create clone of ()", + "block_type": "Control", + "block_shape": "Stack Block", + "op_code": "control_create_clone_of", + "functionality": "Generates a copy, or clone, of a specified sprite (or 'myself' for the current sprite).", + "inputs": [ + { + "name": "sprite_name", + "type": "dropdown", + "options": [ + "myself", + "sprite1", + "..." + ] + } + ], + "example_standalone": "create clone of [myself v]", + "example_with_other_blocks": [ + { + "script": "when I start as a clone\n show\n go to random position\n wait (2) seconds\n delete this clone", + "explanation": "When a clone is created, it will show itself, go to a random position, wait for 2 seconds, and then delete itself." + } + ] + }, + { + "block_name": "delete this clone", + "block_type": "Control", + "block_shape": "Stack Block", + "op_code": "control_delete_this_clone", + "functionality": "Deletes the clone that is currently running the script.", + "inputs": null, + "example_standalone": "delete this clone", + "example_with_other_blocks": [ + { + "script": "when I start as a clone\n wait (5) seconds\n delete this clone", + "explanation": "This script makes each clone wait for 5 seconds after it's created, then deletes itself." + } + ] + }, + { + "block_name": "set [my variable v] to ()", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_setvariableto", + "functionality": "Assigns a specific value (number, string, or boolean) to a variable.", + "inputs": [ + { + "name": "variable name", + "type": "dropdown" + }, + { + "name": "value", + "type": "any" + } + ], + "example_standalone": "set [score v] to (0)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set [score v] to (0)\n set [player name v] to [Guest]", + "explanation": "This script initializes the 'score' variable to 0 and the 'player name' variable to 'Guest' when the project starts." + } + ] + }, + { + "block_name": "change [my variable v] by ()", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_changevariableby", + "functionality": "Increases or decreases a variable's numerical value by a specified amount.", + "inputs": [ + { + "name": "variable name", + "type": "dropdown" + }, + { + "name": "value", + "type": "number" + } + ], + "example_standalone": "change [score v] by (1)", + "example_with_other_blocks": [ + { + "script": "when this sprite clicked\n change [score v] by (1)", + "explanation": "This script increments the 'score' variable by 1 each time the sprite is clicked." + } + ] + }, + { + "block_name": "add () to [my list v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_addtolist", + "functionality": "Appends an item to the end of a list.", + "inputs": [ + { + "name": "item", + "type": "any" + }, + { + "name": "list name", + "type": "dropdown" + } + ], + "example_standalone": "add [apple] to [shopping list v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n add [apple] to [shopping list v]\n add [banana] to [shopping list v]", + "explanation": "This script adds 'apple' and 'banana' as new items to the 'shopping list' when the project starts." + } + ] + }, + { + "block_name": "delete () of [my list v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_deleteoflist", + "functionality": "Removes an item from a list by its index or by selecting 'all' items.", + "inputs": [ + { + "name": "index/option", + "type": "number/dropdown", + "options": [ + "all", + "last", + "random" + ] + }, + { + "name": "list name", + "type": "dropdown" + } + ], + "example_standalone": "delete (1) of [my list v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n delete (all) of [my list v]", + "explanation": "This script clears all items from 'my list' when the green flag is clicked." + } + ] + }, + { + "block_name": "insert () at () of [my list v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_insertatlist", + "functionality": "Inserts an item at a specific position within a list.", + "inputs": [ + { + "name": "item", + "type": "any" + }, + { + "name": "index", + "type": "number" + }, + { + "name": "list name", + "type": "dropdown" + } + ], + "example_standalone": "insert [orange] at (2) of [fruits v]", + "example_with_other_blocks": [ + { + "script": "insert [orange] at (2) of [fruits v]", + "explanation": "This script inserts 'orange' as the second item in the 'fruits' list, shifting subsequent items." + } + ] + }, + { + "block_name": "replace item () of [my list v] with ()", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_replaceitemoflist", + "functionality": "Replaces an item at a specific position in a list with a new value.", + "inputs": [ + { + "name": "index", + "type": "number" + }, + { + "name": "list name", + "type": "dropdown" + }, + { + "name": "new item", + "type": "any" + } + ], + "example_standalone": "replace item (1) of [colors v] with [blue]", + "example_with_other_blocks": [ + { + "script": "replace item (1) of [colors v] with [blue]", + "explanation": "This script changes the first item in the 'colors' list to 'blue'." + } + ] + }, + { + "block_name": "show variable [my variable v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_showvariable", + "functionality": "Makes a variable's monitor visible on the stage.", + "inputs": [ + { + "name": "variable name", + "type": "dropdown" + } + ], + "example_standalone": "show variable [score v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n show variable [score v]", + "explanation": "This script displays the 'score' variable on the stage when the project starts." + } + ] + }, + { + "block_name": "hide variable [my variable v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_hidevariable", + "functionality": "Hides a variable's monitor from the stage.", + "inputs": [ + { + "name": "variable name", + "type": "dropdown" + } + ], + "example_standalone": "hide variable [score v]", + "example_with_other_blocks": [ + { + "script": "when I receive [game over v]\n hide variable [score v]", + "explanation": "This script hides the 'score' variable when the 'game over' broadcast is received." + } + ] + }, + { + "block_name": "show list [my list v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_showlist", + "functionality": "Makes a list's monitor visible on the stage.", + "inputs": [ + { + "name": "list name", + "type": "dropdown" + } + ], + "example_standalone": "show list [shopping list v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n show list [shopping list v]", + "explanation": "This script displays the 'shopping list' on the stage when the project starts." + } + ] + }, + { + "block_name": "hide list [my list v]", + "block_type": "Data", + "block_shape": "Stack Block", + "op_code": "data_hidelist", + "functionality": "Hides a list's monitor from the stage.", + "inputs": [ + { + "name": "list name", + "type": "dropdown" + } + ], + "example_standalone": "hide list [shopping list v]", + "example_with_other_blocks": [ + { + "script": "when I receive [game over v]\n hide list [shopping list v]", + "explanation": "This script hides the 'shopping list' when the 'game over' broadcast is received." + } + ] + }, + { + "block_name": "Ask () and Wait", + "block_type": "Sensing", + "block_shape": "Stack Block", + "op_code": "sensing_askandwait", + "functionality": "Displays an input box with specified text at the bottom of the screen, allowing users to input text, which is stored in the 'Answer' block.", + "inputs": [ + { + "name": "question", + "type": "text" + } + ], + "example_standalone": "ask [What is your name?] and wait", + "example_with_other_blocks": [ + { + "script": "ask [What is your name?] and wait\n say join [Hello ] (answer) for (2) seconds", + "explanation": "This script prompts the user for their name, waits for input, then greets them using the provided answer." + } + ] + }, + { + "block_name": "Reset Timer", + "block_type": "Sensing", + "block_shape": "Stack Block", + "op_code": "sensing_resettimer", + "functionality": "Sets the timer’s value back to 0.0.", + "inputs": null, + "example_standalone": "reset timer", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n reset timer\n wait (5) seconds\n say timer for (2) seconds", + "explanation": "This script resets the timer at the start, waits for 5 seconds, then says the current timer value." + } + ] + }, + { + "block_name": "set drag mode [draggable v]", + "block_type": "Sensing", + "block_shape": "Stack Block", + "op_code": "sensing_setdragmode", + "functionality": "Sets whether the sprite can be dragged by the mouse on the stage.", + "inputs": null, + "fields": { + "DRAG_MODE": { + "type": "dropdown", + "options": [ + "draggable", + "not draggable" + ] + } + }, + "example_standalone": "set drag mode [draggable v]", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n set drag mode [not draggable v]\nwhen this sprite clicked\n set drag mode [draggable v]", + "explanation": "This script makes the sprite not draggable when the project starts, but allows it to be dragged once it's clicked." + } + ] + }, + { + "block_name": "[my custom block]", + "block_type": "My Blocks", + "block_shape": "Stack Block", + "op_code": "procedures_call", + "functionality": "Executes the script defined by a corresponding 'define' Hat block. This block allows users to call and reuse custom code sequences by simply dragging and dropping it into their scripts, optionally providing required input values.", + "inputs": [ + { + "name": "argument_name_1", + "type": "any" + }, + { + "name": "argument_name_2", + "type": "any" + } + ], + "example_standalone": "jump (50)", + "example_with_other_blocks": [ + { + "script": "when green flag clicked\n go to x: (0) y: (0)\n jump (50)\n wait (1) seconds\n say [I jumped!] for (2) seconds", + "explanation": "This script moves the sprite to a starting position, then calls the 'jump' custom block with an input of 50 (assuming 'jump' is a custom block that moves the sprite up and down). After the jump, the sprite says 'I jumped!'." + }, + { + "script": "when green flag clicked\n hide\n forever\n create clone of [myself v]\n wait (1) seconds\n end", + "explanation": "This script continuously creates new clones of the current sprite every second after the original sprite hides itself." + } + ] + } + ] +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks_content.json b/scratch_VLM/scratch_agent/blocks_content.json new file mode 100644 index 0000000000000000000000000000000000000000..b3a39fb7ed163e1462921aa1e693f1b8f41971c1 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks_content.json @@ -0,0 +1,1367 @@ +{ + "motion_movesteps": { + "opcode": "motion_movesteps", + "next": null, + "parent": null, + "inputs": { + "STEPS": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 464, + "y": -416 + }, + "motion_turnright": { + "opcode": "motion_turnright", + "next": null, + "parent": null, + "inputs": { + "DEGREES": [ + 1, + [ + 4, + "15" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 467, + "y": -316 + }, + "motion_turnleft": { + "opcode": "motion_turnleft", + "next": null, + "parent": null, + "inputs": { + "DEGREES": [ + 1, + [ + 4, + "15" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 464, + "y": -210 + }, + "motion_goto": { + "opcode": "motion_goto", + "next": null, + "parent": null, + "inputs": { + "TO": [ + 1, + "@iM=Z?~GCbpC}gT7KAKY" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 465, + "y": -95 + }, + "motion_goto_menu": { + "opcode": "motion_goto_menu", + "next": null, + "parent": "d|J?C902/xy6tD5,|dmB", + "inputs": {}, + "fields": { + "TO": [ + "_random_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "motion_gotoxy": { + "opcode": "motion_gotoxy", + "next": null, + "parent": null, + "inputs": { + "X": [ + 1, + [ + 4, + "0" + ] + ], + "Y": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 468, + "y": 12 + }, + "motion_glideto": { + "opcode": "motion_glideto", + "next": null, + "parent": null, + "inputs": { + "SECS": [ + 1, + [ + 4, + "1" + ] + ], + "TO": [ + 1, + "{id to destination position}" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 470, + "y": 129 + }, + "motion_glideto_menu": { + "opcode": "motion_glideto_menu", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "TO": [ + "_random_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "motion_glidesecstoxy": { + "opcode": "motion_glidesecstoxy", + "next": null, + "parent": null, + "inputs": { + "SECS": [ + 1, + [ + 4, + "1" + ] + ], + "X": [ + 1, + [ + 4, + "0" + ] + ], + "Y": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 476, + "y": 239 + }, + "motion_pointindirection": { + "opcode": "motion_pointindirection", + "next": null, + "parent": null, + "inputs": { + "DIRECTION": [ + 1, + [ + 8, + "90" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 493, + "y": 361 + }, + "motion_pointtowards": { + "opcode": "motion_pointtowards", + "next": null, + "parent": null, + "inputs": { + "TOWARDS": [ + 1, + "6xQl1pPk%9E~Znhm*:ng" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 492, + "y": 463 + }, + "motion_pointtowards_menu": { + "opcode": "motion_pointtowards_menu", + "next": null, + "parent": "Ucm$YBs*^9GFTGXCbal@", + "inputs": {}, + "fields": { + "TOWARDS": [ + "_mouse_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "motion_changexby": { + "opcode": "motion_changexby", + "next": null, + "parent": null, + "inputs": { + "DX": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 851, + "y": -409 + }, + "motion_setx": { + "opcode": "motion_setx", + "next": null, + "parent": null, + "inputs": { + "X": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 864, + "y": -194 + }, + "motion_changeyby": { + "opcode": "motion_changeyby", + "next": null, + "parent": null, + "inputs": { + "DY": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 861, + "y": -61 + }, + "motion_sety": { + "opcode": "motion_sety", + "next": null, + "parent": null, + "inputs": { + "Y": [ + 1, + [ + 4, + "0" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 864, + "y": 66 + }, + "motion_ifonedgebounce": { + "opcode": "motion_ifonedgebounce", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1131, + "y": -397 + }, + "motion_setrotationstyle": { + "opcode": "motion_setrotationstyle", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "STYLE": [ + "left-right", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 1128, + "y": -287 + }, + "motion_xposition": { + "opcode": "motion_xposition", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1193, + "y": -136 + }, + "motion_yposition": { + "opcode": "motion_yposition", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1181, + "y": -64 + }, + "motion_direction": { + "opcode": "motion_direction", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 1188, + "y": 21 + }, + "control_wait": { + "opcode": "control_wait", + "next": null, + "parent": null, + "inputs": { + "DURATION": [ + 1, + [ + 5, + "1" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 337, + "y": 129 + }, + "control_repeat": { + "opcode": "control_repeat", + "next": null, + "parent": null, + "inputs": { + "TIMES": [ + 1, + [ + 6, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 265 + }, + "control_forever": { + "opcode": "control_forever", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 334, + "y": 439 + }, + "control_if": { + "opcode": "control_if", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 331, + "y": 597 + }, + "control_if_else": { + "opcode": "control_if_else", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 335, + "y": 779 + }, + "control_wait_until": { + "opcode": "control_wait_until", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 676, + "y": 285 + }, + "control_repeat_until": { + "opcode": "control_repeat_until", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 692, + "y": 381 + }, + "control_stop": { + "opcode": "control_stop", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "STOP_OPTION": [ + "all", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 708, + "y": 545, + "mutation": { + "tagName": "mutation", + "children": [], + "hasnext": "false" + } + }, + "control_start_as_clone": { + "opcode": "control_start_as_clone", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 665, + "y": 672 + }, + "control_create_clone_of": { + "opcode": "control_create_clone_of", + "next": null, + "parent": null, + "inputs": { + "CLONE_OPTION": [ + 1, + "t))DW9(QSKB]3C/3Ou+J" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 648, + "y": 797 + }, + "control_create_clone_of_menu": { + "opcode": "control_create_clone_of_menu", + "next": null, + "parent": "80yo/}Cw++Z.;x[ohh|7", + "inputs": {}, + "fields": { + "CLONE_OPTION": [ + "_myself_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "control_delete_this_clone": { + "opcode": "control_delete_this_clone", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 642, + "y": 914 + }, + "event_whenflagclicked": { + "opcode": "event_whenflagclicked", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 166, + "y": -422 + }, + "event_whenkeypressed": { + "opcode": "event_whenkeypressed", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "space", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 151, + "y": -329 + }, + "event_whenthisspriteclicked": { + "opcode": "event_whenthisspriteclicked", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 156, + "y": -223 + }, + "event_whenbackdropswitchesto": { + "opcode": "event_whenbackdropswitchesto", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "BACKDROP": [ + "backdrop1", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 148, + "y": -101 + }, + "event_whengreaterthan": { + "opcode": "event_whengreaterthan", + "next": null, + "parent": null, + "inputs": { + "VALUE": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": { + "WHENGREATERTHANMENU": [ + "LOUDNESS", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 150, + "y": 10 + }, + "event_whenbroadcastreceived": { + "opcode": "event_whenbroadcastreceived", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "BROADCAST_OPTION": [ + "message1", + "5O!nei;S$!c!=hCT}0:a" + ] + }, + "shadow": false, + "topLevel": true, + "x": 141, + "y": 118 + }, + "event_broadcast": { + "opcode": "event_broadcast", + "next": null, + "parent": null, + "inputs": { + "BROADCAST_INPUT": [ + 1, + [ + 11, + "message1", + "5O!nei;S$!c!=hCT}0:a" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 151, + "y": 229 + }, + "event_broadcastandwait": { + "opcode": "event_broadcastandwait", + "next": null, + "parent": null, + "inputs": { + "BROADCAST_INPUT": [ + 1, + [ + 11, + "message1", + "5O!nei;S$!c!=hCT}0:a" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 157, + "y": 340 + }, + "sensing_touchingobject": { + "opcode": "sensing_touchingobject", + "next": null, + "parent": null, + "inputs": { + "TOUCHINGOBJECTMENU": [ + 1, + "xSKW9a+wTnM~h~So8Jc]" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 359, + "y": 116 + }, + "sensing_touchingobjectmenu": { + "opcode": "sensing_touchingobjectmenu", + "next": null, + "parent": "Y(n,F@BYzwd4CiN|Bh[P", + "inputs": {}, + "fields": { + "TOUCHINGOBJECTMENU": [ + "_mouse_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sensing_touchingcolor": { + "opcode": "sensing_touchingcolor", + "next": null, + "parent": null, + "inputs": { + "COLOR": [ + 1, + [ + 9, + "#55b888" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 360, + "y": 188 + }, + "sensing_coloristouchingcolor": { + "opcode": "sensing_coloristouchingcolor", + "next": null, + "parent": null, + "inputs": { + "COLOR": [ + 1, + [ + 9, + "#d019f2" + ] + ], + "COLOR2": [ + 1, + [ + 9, + "#2b0de3" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 277 + }, + "sensing_askandwait": { + "opcode": "sensing_askandwait", + "next": null, + "parent": null, + "inputs": { + "QUESTION": [ + 1, + [ + 10, + "What's your name?" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 338, + "y": 354 + }, + "sensing_answer": { + "opcode": "sensing_answer", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 782, + "y": 111 + }, + "sensing_keypressed": { + "opcode": "sensing_keypressed", + "next": null, + "parent": null, + "inputs": { + "KEY_OPTION": [ + 1, + "SNlf@Im$sv%.6ULi-f3i" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 762, + "y": 207 + }, + "sensing_keyoptions": { + "opcode": "sensing_keyoptions", + "next": null, + "parent": "7$xEUO.2hH2R6vh!$(Uj", + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "space", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sensing_mousedown": { + "opcode": "sensing_mousedown", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 822, + "y": 422 + }, + "sensing_mousex": { + "opcode": "sensing_mousex", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 302, + "y": 528 + }, + "sensing_mousey": { + "opcode": "sensing_mousey", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 668, + "y": 547 + }, + "sensing_setdragmode": { + "opcode": "sensing_setdragmode", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "DRAG_MODE": [ + "draggable", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 950, + "y": 574 + }, + "sensing_loudness": { + "opcode": "sensing_loudness", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 658, + "y": 703 + }, + "sensing_timer": { + "opcode": "sensing_timer", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 459, + "y": 671 + }, + "sensing_resettimer": { + "opcode": "sensing_resettimer", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 462, + "y": 781 + }, + "sensing_of": { + "opcode": "sensing_of", + "next": null, + "parent": null, + "inputs": { + "OBJECT": [ + 1, + "t+o*y;iz,!O#aT|qM_+O" + ] + }, + "fields": { + "PROPERTY": [ + "backdrop #", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 997, + "y": 754 + }, + "sensing_of_object_menu": { + "opcode": "sensing_of_object_menu", + "next": null, + "parent": "[4I2wIG/tNc@LQ-;FbsB", + "inputs": {}, + "fields": { + "OBJECT": [ + "_stage_", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "sensing_current": { + "opcode": "sensing_current", + "next": null, + "parent": null, + "inputs": {}, + "fields": { + "CURRENTMENU": [ + "YEAR", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 627, + "y": 884 + }, + "sensing_dayssince2000": { + "opcode": "sensing_dayssince2000", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 959, + "y": 903 + }, + "sensing_username": { + "opcode": "sensing_username", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 833, + "y": 757 + }, + "operator_add": { + "opcode": "operator_add", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 128, + "y": 153 + }, + "operator_subtract": { + "opcode": "operator_subtract", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 134, + "y": 214 + }, + "operator_multiply": { + "opcode": "operator_multiply", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 134, + "y": 278 + }, + "operator_divide": { + "opcode": "operator_divide", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 138, + "y": 359 + }, + "operator_random": { + "opcode": "operator_random", + "next": null, + "parent": null, + "inputs": { + "FROM": [ + 1, + [ + 4, + "1" + ] + ], + "TO": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 311, + "y": 157 + }, + "operator_gt": { + "opcode": "operator_gt", + "next": null, + "parent": null, + "inputs": { + "OPERAND1": [ + 1, + [ + 10, + "" + ] + ], + "OPERAND2": [ + 1, + [ + 10, + "50" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 348, + "y": 217 + }, + "operator_lt": { + "opcode": "operator_lt", + "next": null, + "parent": null, + "inputs": { + "OPERAND1": [ + 1, + [ + 10, + "" + ] + ], + "OPERAND2": [ + 1, + [ + 10, + "50" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 345, + "y": 286 + }, + "operator_equals": { + "opcode": "operator_equals", + "next": null, + "parent": null, + "inputs": { + "OPERAND1": [ + 1, + [ + 10, + "" + ] + ], + "OPERAND2": [ + 1, + [ + 10, + "50" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 345, + "y": 372 + }, + "operator_and": { + "opcode": "operator_and", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 701, + "y": 158 + }, + "operator_or": { + "opcode": "operator_or", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 705, + "y": 222 + }, + "operator_not": { + "opcode": "operator_not", + "next": null, + "parent": null, + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 734, + "y": 283 + }, + "operator_join": { + "opcode": "operator_join", + "next": null, + "parent": null, + "inputs": { + "STRING1": [ + 1, + [ + 10, + "apple " + ] + ], + "STRING2": [ + 1, + [ + 10, + "banana" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 663, + "y": 378 + }, + "operator_letter_of": { + "opcode": "operator_letter_of", + "next": null, + "parent": null, + "inputs": { + "LETTER": [ + 1, + [ + 6, + "1" + ] + ], + "STRING": [ + 1, + [ + 10, + "apple" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 664, + "y": 445 + }, + "operator_length": { + "opcode": "operator_length", + "next": null, + "parent": null, + "inputs": { + "STRING": [ + 1, + [ + 10, + "apple" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 664, + "y": 521 + }, + "operator_contains": { + "opcode": "operator_contains", + "next": null, + "parent": null, + "inputs": { + "STRING1": [ + 1, + [ + 10, + "apple" + ] + ], + "STRING2": [ + 1, + [ + 10, + "a" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 634, + "y": 599 + }, + "operator_mod": { + "opcode": "operator_mod", + "next": null, + "parent": null, + "inputs": { + "NUM1": [ + 1, + [ + 4, + "" + ] + ], + "NUM2": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 295, + "y": 594 + }, + "operator_round": { + "opcode": "operator_round", + "next": null, + "parent": null, + "inputs": { + "NUM": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": true, + "x": 307, + "y": 674 + }, + "operator_mathop": { + "opcode": "operator_mathop", + "next": null, + "parent": null, + "inputs": { + "NUM": [ + 1, + [ + 4, + "" + ] + ] + }, + "fields": { + "OPERATOR": [ + "abs", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 280, + "y": 754 + } +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/blocks_defination.py b/scratch_VLM/scratch_agent/blocks_defination.py new file mode 100644 index 0000000000000000000000000000000000000000..6c2c26fdffebadda65870dde738e164b5dd81495 --- /dev/null +++ b/scratch_VLM/scratch_agent/blocks_defination.py @@ -0,0 +1,274 @@ +ALL_SCRATCH_BLOCKS_CATALOG = { + "motion_movesteps": { + "opcode": "motion_movesteps", + "next": null, + "parent": null, + "inputs": { + "STEPS": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": False, + "topLevel": True, + "x": 464, + "y": -416 + }, + "motion_turnright": { + "opcode": "motion_turnright", + "inputs": {"DEGREES": [1, [4, "15"]]}, + "fields": {}, + "shadow": False + }, + "motion_turnleft": { + "opcode": "motion_turnleft", + "inputs": {"DEGREES": [1, [4, "15"]]}, + "fields": {}, + "shadow": False + }, + "motion_goto": { + "opcode": "motion_goto", + "inputs": {"TO": [1, None]}, # TO input will be filled by a menu block ID + "fields": {}, + "shadow": False + }, + "motion_goto_menu": { # Shadow block for motion_goto + "opcode": "motion_goto_menu", + "inputs": {}, + "fields": {"TO": ["_random_", None]}, + "shadow": True + }, + "motion_gotoxy": { + "opcode": "motion_gotoxy", + "inputs": {"X": [1, [4, "0"]], "Y": [1, [4, "0"]]}, + "fields": {}, + "shadow": False + }, + "motion_glideto": { + "opcode": "motion_glideto", + "inputs": {"SECS": [1, [4, "1"]], "TO": [1, None]}, # TO input will be filled by a menu block ID + "fields": {}, + "shadow": False + }, + "motion_glideto_menu": { # Shadow block for motion_glideto + "opcode": "motion_glideto_menu", + "inputs": {}, + "fields": {"TO": ["_random_", None]}, + "shadow": True + }, + "motion_glidesecstoxy": { + "opcode": "motion_glidesecstoxy", + "inputs": {"SECS": [1, [4, "1"]], "X": [1, [4, "0"]], "Y": [1, [4, "0"]]}, + "fields": {}, + "shadow": False + }, + "motion_pointindirection": { + "opcode": "motion_pointindirection", + "inputs": {"DIRECTION": [1, [8, "90"]]}, + "fields": {}, + "shadow": False + }, + "motion_pointtowards": { + "opcode": "motion_pointtowards", + "inputs": {"TOWARDS": [1, None]}, # TOWARDS input will be filled by a menu block ID + "fields": {}, + "shadow": False + }, + "motion_pointtowards_menu": { # Shadow block for motion_pointtowards + "opcode": "motion_pointtowards_menu", + "inputs": {}, + "fields": {"TOWARDS": ["_mouse_", None]}, + "shadow": True + }, + "motion_changexby": { + "opcode": "motion_changexby", + "inputs": {"DX": [1, [4, "10"]]}, + "fields": {}, + "shadow": False + }, + "motion_setx": { + "opcode": "motion_setx", + "inputs": {"X": [1, [4, "0"]]}, + "fields": {}, + "shadow": False + }, + "motion_changeyby": { + "opcode": "motion_changeyby", + "inputs": {"DY": [1, [4, "10"]]}, + "fields": {}, + "shadow": False + }, + "motion_sety": { + "opcode": "motion_sety", + "inputs": {"Y": [1, [4, "0"]]}, + "fields": {}, + "shadow": False + }, + "motion_ifonedgebounce": { + "opcode": "motion_ifonedgebounce", + "next": None, "parent": None, "inputs": {}, "fields": {}, "shadow": False + }, + "motion_setrotationstyle": { + "opcode": "motion_setrotationstyle", + "inputs": {}, + "fields": {"STYLE": ["left-right", None]}, + "shadow": False + }, + "motion_xposition": { + "opcode": "motion_xposition", + "inputs": {}, "fields": {}, "shadow": False + }, + "motion_yposition": { + "opcode": "motion_yposition", + "inputs": {}, "fields": {}, "shadow": False + }, + "motion_direction": { + "opcode": "motion_direction", + "inputs": {}, "fields": {}, "shadow": False + }, + # Control Blocks + "control_wait": { + "opcode": "control_wait", + "inputs": {"DURATION": [1, [5, "1"]]}, + "fields": {}, + "shadow": False + }, + "control_repeat": { + "opcode": "control_repeat", + "inputs": {"TIMES": [1, [6, "10"]], "SUBSTACK": [2, None]}, # SUBSTACK to be filled + "fields": {}, + "shadow": False + }, + "control_forever": { + "opcode": "control_forever", + "inputs": {"SUBSTACK": [2, None]}, # SUBSTACK to be filled + "fields": {}, + "shadow": False + }, + "control_if": { + "opcode": "control_if", + "inputs": {"CONDITION": [2, None], "SUBSTACK": [2, None]}, # CONDITION and SUBSTACK to be filled + "fields": {}, + "shadow": False + }, + "control_if_else": { + "opcode": "control_if_else", + "inputs": {"CONDITION": [2, None], "SUBSTACK": [2, None], "SUBSTACK2": [2, None]}, # All to be filled + "fields": {}, + "shadow": False + }, + "control_wait_until": { + "opcode": "control_wait_until", + "inputs": {"CONDITION": [2, None]}, + "fields": {}, + "shadow": False + }, + "control_repeat_until": { + "opcode": "control_repeat_until", + "inputs": {"CONDITION": [2, None], "SUBSTACK": [2, None]}, + "fields": {}, + "shadow": False + }, + "control_stop": { + "opcode": "control_stop", + "inputs": {}, + "fields": {"STOP_OPTION": ["all", None]}, + "shadow": False, + "mutation": {"tagName": "mutation", "children": [], "hasnext": "false"} + }, + "control_start_as_clone": { + "opcode": "control_start_as_clone", + "inputs": {}, "fields": {}, "shadow": False + }, + "control_create_clone_of": { + "opcode": "control_create_clone_of", + "inputs": {"CLONE_OPTION": [1, None]}, # CLONE_OPTION to be filled by menu block ID + "fields": {}, + "shadow": False + }, + "control_create_clone_of_menu": { # Shadow block for control_create_clone_of + "opcode": "control_create_clone_of_menu", + "inputs": {}, + "fields": {"CLONE_OPTION": ["_myself_", None]}, + "shadow": True + }, + "control_delete_this_clone": { + "opcode": "control_delete_this_clone", + "inputs": {}, "fields": {}, "shadow": False + }, + # Event Blocks + "event_whenflagclicked": { + "opcode": "event_whenflagclicked", + "inputs": {}, "fields": {}, "shadow": False + }, + "event_whenkeypressed": { + "opcode": "event_whenkeypressed", + "inputs": {}, + "fields": {"KEY_OPTION": ["space", None]}, + "shadow": False + }, + "event_whenthisspriteclicked": { + "opcode": "event_whenthisspriteclicked", + "inputs": {}, "fields": {}, "shadow": False + }, + "event_whenbackdropswitchesto": { + "opcode": "event_whenbackdropswitchesto", + "inputs": {}, + "fields": {"BACKDROP": ["backdrop1", None]}, + "shadow": False + }, + "event_whengreaterthan": { + "opcode": "event_whengreaterthan", + "inputs": {"VALUE": [1, [4, "10"]]}, + "fields": {"WHENGREATERTHANMENU": ["LOUDNESS", None]}, + "shadow": False + }, + "event_whenbroadcastreceived": { + "opcode": "event_whenbroadcastreceived", + "inputs": {}, + "fields": {"BROADCAST_OPTION": ["message1", None]}, + "shadow": False + }, + "event_broadcast": { + "opcode": "event_broadcast", + "inputs": {"BROADCAST_INPUT": [1, None]}, # BROADCAST_INPUT to be filled by menu or text block + "fields": {}, + "shadow": False + }, + "event_broadcastandwait": { + "opcode": "event_broadcastandwait", + "inputs": {"BROADCAST_INPUT": [1, None]}, + "fields": {}, + "shadow": False + }, + # Sensing Blocks (example, can be expanded) + "sensing_touchingobject": { + "opcode": "sensing_touchingobject", + "inputs": {"TOUCHINGOBJECTMENU": [1, None]}, # Input for menu block ID + "fields": {}, + "shadow": False + }, + "sensing_touchingobjectmenu": { # Shadow block for sensing_touchingobject + "opcode": "sensing_touchingobjectmenu", + "inputs": {}, + "fields": {"TOUCHINGOBJECTMENU": ["_mouse_", None]}, + "shadow": True + }, + # Operators (example, can be expanded) + "operator_add": { + "opcode": "operator_add", + "inputs": {"NUM1": [1, [4, "0"]], "NUM2": [1, [4, "0"]]}, + "fields": {}, + "shadow": False + }, + "operator_equals": { + "opcode": "operator_equals", + "inputs": {"OPERAND1": [1, [10, ""]], "OPERAND2": [1, [10, ""]]}, + "fields": {}, + "shadow": False + } +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/game_plans.txt b/scratch_VLM/scratch_agent/game_plans.txt new file mode 100644 index 0000000000000000000000000000000000000000..49eda28d59fec70e3f9095f99471b673e3e7e1c6 --- /dev/null +++ b/scratch_VLM/scratch_agent/game_plans.txt @@ -0,0 +1,74 @@ +######################################### Cat jumping over the obstacle game ########################################### + +{ + "action_overall_flow": { + "Sprite1": { + "description": "Main character (cat) actions: jump on Space key and reset position on R key", + "plans": [ + { + "event": "event_whenkeypressed", + "logic": "play 'Meow' sound; switch to next costume; repeat 10 times: change y by 10; wait 0.0001 seconds; repeat 10 times: change y by -10; wait 0.0001 seconds; switch to next costume", + "events": [ + "event_whenkeypressed" + ], + "sound": [ + "sound_play", + "sound_sounds_menu" + ], + "looks": [ + "looks_nextcostume" + ], + "control": [ + "control_repeat", + "control_wait" + ], + "motion": [ + "motion_changeyby" + ] + }, + { + "event": "event_whenkeypressed", + "logic": "go to x: -160 y: -110; switch costume to 'costume1'", + "events": [ + "event_whenkeypressed" + ], + "motion": [ + "motion_gotoxy" + ], + "looks": [ + "looks_switchcostumeto", + "looks_costume" + ] + } + ] + }, + "Soccer Ball": { + "description": "Obstacle (ball) movement on R key and stop on collision", + "plans": [ + { + "event": "event_whenkeypressed", + "logic": "go to x: 240 y: -135; forever: glide 3 seconds to x: -240 y: -135; if touching 'Sprite1' then stop all", + "events": [ + "event_whenkeypressed" + ], + "motion": [ + "motion_gotoxy", + "motion_glidesecstoxy" + ], + "control": [ + "control_forever", + "control_if", + "control_stop" + ], + "sensing": [ + "sensing_touchingobject", + "sensing_touchingobjectmenu" + ], + "operators": [ + "operator_eq" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/helper_function.py b/scratch_VLM/scratch_agent/helper_function.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/scratch_VLM/scratch_agent/langgraph_workflow.png b/scratch_VLM/scratch_agent/langgraph_workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..ba09c2df55c27ce74004170d5bff805aa209454a Binary files /dev/null and b/scratch_VLM/scratch_agent/langgraph_workflow.png differ diff --git a/scratch_VLM/scratch_agent/langgraph_workflow_main.png b/scratch_VLM/scratch_agent/langgraph_workflow_main.png new file mode 100644 index 0000000000000000000000000000000000000000..67fb4b9618beb4547799b48ffbfff6bbcd641201 Binary files /dev/null and b/scratch_VLM/scratch_agent/langgraph_workflow_main.png differ diff --git a/scratch_VLM/scratch_agent/last_updated.txt b/scratch_VLM/scratch_agent/last_updated.txt new file mode 100644 index 0000000000000000000000000000000000000000..133f5b0cc7572d19690ea43e9f55c4f1907cb515 --- /dev/null +++ b/scratch_VLM/scratch_agent/last_updated.txt @@ -0,0 +1,2 @@ +app.py [updated the action plan node and action builder node and also add the behaviour plan and behaviour builder node ] +app.py updated version in sys_scratch_2.py [enhacement in the node of action planner] \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/logs.py b/scratch_VLM/scratch_agent/logs.py new file mode 100644 index 0000000000000000000000000000000000000000..8a173d0bb626f77ce673a35e0b494d9706bb2ab5 --- /dev/null +++ b/scratch_VLM/scratch_agent/logs.py @@ -0,0 +1,1757 @@ +from flask import Flask, request, jsonify, render_template, send_from_directory +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +#from langgraph.graph import draw +from langchain.chat_models import ChatOpenAI +from PIL import Image +import os, json, re +import shutil +import uuid +from langgraph.graph import StateGraph, END +import logging +from typing import Dict, TypedDict, Optional, Any, List + +# --- Configure logging --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +# --- LLM / Vision model setup --- +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + logger.critical("GROQ_API_KEY environment variable is not set or invalid. Please set it to proceed.") + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +llm = ChatGroq( + #model="deepseek-r1-distill-llama-70b", + #model="llama-3.3-70b-versatile", + #model="meta-llama/llama-4-maverick-17b-128e-instruct", + model="meta-llama/llama-4-scout-17b-16e-instruct", + temperature=0.0, +) + +# llm = ChatOpenAI( +# openai_api_key=getenv("OPENROUTER_API_KEY"), +# openai_api_base=getenv("OPENROUTER_BASE_URL"), +# model_name="", +# model_kwargs={ +# "headers": { +# "HTTP-Referer": getenv("YOUR_SITE_URL"), +# "X-Title": getenv("YOUR_SITE_NAME"), +# } +# }, +# ) + +# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema, informed by comprehensive reference materials. When generating or modifying the `blocks` section, pay extremely close attention to the following: + +**Scratch Target and Block Schema Rules:** +1. **Target Structure:** + * Every target (Stage or Sprite) must have an `objName` property. For the Stage, it's typically `"objName": "Stage"`. For sprites, it's their name (e.g., `"objName": "Cat"`). + * `variables` dictionary: This dictionary maps unique variable IDs to arrays `[variable_name, initial_value]`. + Example: `"variables": { "myVarId123": ["score", 0] }` + +2. **Block Structure:** Every block must have `opcode`, `blockType`, `parent`, `next`, `inputs`, `fields`, `shadow`, `topLevel`. + * `opcode`: Unique internal identifier for the block's specific functionality (e.g., `"motion_movesteps"`)[cite: 439, 452]. + * `blockType`: Specifies the visual shape and general behavior. Common types include `"command"` (Stack block), `"reporter"` (Reporter block), `"Boolean"` (Boolean block), and `"hat"` (Hat block)[cite: 453, 454]. + * `parent`: ID of the block it's stacked on (or `null` for top-level). + * `next`: ID of the block directly below it (or `null` for end of stack). + * `topLevel`: `true` if it's a hat block or a standalone block, `false` otherwise. + * `shadow`: `true` if it's a shadow block (e.g., a default number input), `false` otherwise. + +3. **`inputs` vs. `fields`:** This is critical. + * **`inputs`**: Used for blocks or values *plugged into* a block (e.g., condition for an `if` block, target for `go to`). Values are **arrays**. + * `[1, ]`: Represents a Reporter or Boolean block *plugged in* (e.g., `operator_equals`). + * `[1, [, ""]]`: For a shadow block with a literal value. The `type` specifies the data type and can be "num" (number), "str" (string), "bool" (Boolean), "colour", "angle", "date", "dropdown", "iconmenu", or "variable"[cite: 457, 461]. + Example: `[1, ["num", "10"]]` for a number input. + * `[2, ]`: For a C-block's substack (e.g., `SUBSTACK` in `control_if`). + * **`fields`**: Used for dropdown menus or direct internal values *within* a block (e.g., key selected in `when key pressed`, variable name in `set variable`). Values are always **arrays** `["", null]`. + * Example for `event_whenkeypressed`: `"KEY": ["space", null]` + * Example for `data_setvariableto`: `"VARIABLE": ["score", null]` + +4. **Unique IDs:** All block IDs, variable IDs, and list IDs must be unique strings (e.g., "myBlock123", "myVarId456"). Do NOT use placeholder strings like "block_id_here". + +5. **No Nested `blocks` Dictionary:** The `blocks` dictionary should only appear once per `target` (sprite/stage). Do NOT nest a `blocks` dictionary inside an individual block definition. Blocks that are part of a substack (like inside an `if` or `forever` or `repeat` loop and other) are linked via the `SUBSTACK` input, where the input value is `[2, "ID_of_first_block_in_substack"]`. + +6. **Asset Properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY` should be correct for costumes/sounds. + +**General Principles and Important Considerations:** +* **Backward Compatibility:** Adhere strictly to existing Scratch 3.0 opcodes and schema to ensure backward compatibility with older projects[cite: 439, 440, 446]. +* **Forgiving Inputs:** Recognize that Scratch is designed to be "forgiving in its interpretation of inputs"[cite: 442, 459]. While generating valid JSON, understand that the Scratch VM handles potentially "invalid" inputs gracefully (e.g., type coercion, returning default values like zero or empty strings, rather than crashing)[cite: 443, 459]. This implies that precise type matching for inputs might be handled internally by Scratch, allowing for some flexibility in how values are provided, but the agent should aim for the most common and logical type. + +**When responding, you must:** +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, field, and variable declaration strictly follows the Scratch 3.0 schema and the rules above. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" + +agent = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT +) + +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + backdrops = [] + sprites = [] + try: + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + logger.info("Successfully listed static assets.") + except Exception as e: + logger.error(f"Error listing static assets: {e}") + return jsonify({"error": "Failed to list assets"}), 500 + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + logger.warning(f"Could not determine image dimensions for {filename}: {e}. Setting center to 0,0.") + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- New endpoint to fetch project.json --- +@app.route("/get_project/", methods=["GET"]) +def get_project(project_id): + project_folder = os.path.join("generated_projects", project_id) + project_json_path = os.path.join(project_folder, "project.json") + + try: + if os.path.exists(project_json_path): + logger.info(f"Serving project.json for project ID: {project_id}") + return send_from_directory(project_folder, "project.json", as_attachment=True, download_name=f"{project_id}.json") + else: + logger.warning(f"Project JSON not found for ID: {project_id}") + return jsonify({"error": "Project not found"}), 404 + except Exception as e: + logger.error(f"Error serving project.json for ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve project"}), 500 + +# --- New endpoint to fetch assets --- +@app.route("/get_asset//", methods=["GET"]) +def get_asset(project_id, filename): + project_folder = os.path.join("generated_projects", project_id) + asset_path = os.path.join(project_folder, filename) + + try: + if os.path.exists(asset_path): + logger.info(f"Serving asset '{filename}' for project ID: {project_id}") + return send_from_directory(project_folder, filename) + else: + logger.warning(f"Asset '{filename}' not found for project ID: {project_id}") + return jsonify({"error": "Asset not found"}), 404 + except Exception as e: + logger.error(f"Error serving asset '{filename}' for project ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve asset"}), 500 + + +### LangGraph Workflow Definition + +# Define a state for your graph using TypedDict +class GameState(TypedDict): + project_json: dict + description: str + project_id: str + sprite_initial_positions: dict # Add this as well, as it's part of your state + action_plan: Optional[Dict] + #behavior_plan: Optional[Dict] + improvement_plan: Optional[Dict] + needs_improvement: bool + plan_validation_feedback: Optional[Dict] + iteration_count: int # Track the number of iterations for improvements + review_block_feedback: Optional[Dict] # Feedback from the agent on the blocks after verification + +# Helper function to update project JSON with sprite positions +import copy +def update_project_with_sprite_positions(project_json: dict, sprite_positions: dict) -> dict: + """ + Update the 'x' and 'y' coordinates of sprites in the Scratch project JSON. + + Args: + project_json (dict): Original Scratch project JSON. + sprite_positions (dict): Dict mapping sprite names to {'x': int, 'y': int}. + + Returns: + dict: Updated project JSON with new sprite positions. + """ + updated_project = copy.deepcopy(project_json) + + for target in updated_project.get("targets", []): + if not target.get("isStage", False): + sprite_name = target.get("name") + if sprite_name in sprite_positions: + pos = sprite_positions[sprite_name] + if "x" in pos and "y" in pos: + target["x"] = pos["x"] + target["y"] = pos["y"] + + return updated_project + +# Helper function to load the block catalog from a JSON file +def _load_block_catalog(file_path: str) -> Dict: + """Loads the Scratch block catalog from a specified JSON file.""" + try: + with open(file_path, 'r') as f: + catalog = json.load(f) + logger.info(f"Successfully loaded block catalog from {file_path}") + return catalog + except FileNotFoundError: + logger.error(f"Error: Block catalog file not found at {file_path}") + # Return an empty dict or raise an error, depending on desired behavior + return {} + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON from {file_path}: {e}") + return {} + except Exception as e: + logger.error(f"An unexpected error occurred while loading {file_path}: {e}") + return {} + +# --- Global variable for the block catalog --- +ALL_SCRATCH_BLOCKS_CATALOG = {} +BLOCK_CATALOG_PATH = r"blocks\blocks.json" # Define the path to your JSON file +HAT_BLOCKS_PATH = r"blocks\hat_blocks.json" # Path to the hat blocks JSON file +STACK_BLOCKS_PATH = r"blocks\stack_blocks.json" # Path to the stack blocks JSON file +REPORTER_BLOCKS_PATH = r"blocks\reporter_blocks.json" # Path to the reporter blocks JSON file +BOOLEAN_BLOCKS_PATH = r"blocks\boolean_blocks.json" # Path to the boolean blocks JSON file +C_BLOCKS_PATH = r"blocks\c_blocks.json" # Path to the C blocks JSON file +CAP_BLOCKS_PATH = r"blocks\cap_blocks.json" # Path to the cap blocks JSON file + +# Load the block catalogs from their respective JSON files +hat_block_data = _load_block_catalog(HAT_BLOCKS_PATH) +hat_description = hat_block_data["description"] +hat_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in hat_block_data["blocks"]]) +print("Hat blocks loaded successfully.", hat_description) +boolean_block_data = _load_block_catalog(BOOLEAN_BLOCKS_PATH) +boolean_description = boolean_block_data["description"] +boolean_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in boolean_block_data["blocks"]]) + +c_block_data = _load_block_catalog(C_BLOCKS_PATH) +c_description = c_block_data["description"] +c_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in c_block_data["blocks"]]) + +cap_block_data = _load_block_catalog(CAP_BLOCKS_PATH) +cap_description = cap_block_data["description"] +cap_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in cap_block_data["blocks"]]) + +reporter_block_data = _load_block_catalog(REPORTER_BLOCKS_PATH) +reporter_description = reporter_block_data["description"] +reporter_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in reporter_block_data["blocks"]]) + +stack_block_data = _load_block_catalog(STACK_BLOCKS_PATH) +stack_description = stack_block_data["description"] +stack_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in stack_block_data["blocks"]]) + +# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally +ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH) + +# Helper function to generate a unique block ID +def generate_block_id(): + """Generates a short, unique ID for a Scratch block.""" + return str(uuid.uuid4())[:10].replace('-', '') # Shorten for readability, ensure uniqueness + +# Placeholder for your extract_json_from_llm_response function +# Helper function to extract JSON from LLM response +def extract_json_from_llm_response(raw_response: str) -> dict: + """ + Extracts a JSON object from an LLM response string, robustly handling + literal newlines inside string values by escaping them. + """ + # 1) Pull out the inner JSON block (```json … ```) + json_block = raw_response + m = re.search(r"```json\s*(\{.*\})\s*```", raw_response, re.DOTALL) + if m: + json_block = m.group(1) + + # 2) Find the feedback field and escape its newlines + # This pattern captures everything between "feedback": " … " + feedback_pattern = r'("feedback"\s*:\s*")(.+?)("\s*,)' + def _escape_feedback(m): + prefix, val, suffix = m.groups() + # replace each literal newline and carriage return with \n + safe_val = val.replace("\r", "\\r").replace("\n", "\\n") + return prefix + safe_val + suffix + + json_sanitized = re.sub(feedback_pattern, _escape_feedback, json_block, flags=re.DOTALL) + + # 3) Now try loading + try: + return json.loads(json_sanitized) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse sanitized JSON: {e!r}\nContent was:\n{json_sanitized}") + raise + +# Node 1: Detailed description generator. +def game_description_node(state: GameState): + """ + Generates a detailed narrative description of the game based on the initial query. + """ + logger.info("--- Running GameDescriptionNode ---") + + initial_description = state.get("description", "A simple game.") + + description_prompt = ( + f"You are an AI assistant tasked with generating a detailed narrative description for a game.\n" + f"The initial high-level description is: '{initial_description}'.\n" + f"Expand on this to create a rich, engaging, and detailed description, including potential gameplay elements, objectives, and overall feel.\n" + f"The output should be a plain text description." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": description_prompt}]}) + detailed_game_description = response["messages"][-1].content + state["description"] = detailed_game_description + logger.info("Detailed game description generated by GameDescriptionNode.") + print(f"Detailed Game Description: {detailed_game_description}") + return state + except Exception as e: + logger.error(f"Error in GameDescriptionNode: {e}") + raise + +# Node 2: Analysis User Query and Initial Positions +def parse_query_and_set_initial_positions(state: GameState): + logger.info("--- Running ParseQueryNode ---") + + llm_query_prompt = ( + f"Based on the user's game description: '{state['description']}', " + f"and the current Scratch project JSON below, " + f"determine the most appropriate initial 'x' and 'y' coordinates for each sprite. " + f"Return ONLY a JSON object with a single key 'sprite_initial_positions' mapping sprite names to their {{'x': int, 'y': int}} coordinates.\n\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Example Json output:\n" + "```json\n" + "{\n" + " \"sprite_initial_positions\": {\n" + " \"Sprite1\": {\"x\": -160, \"y\": -110},\n" + " \"Sprite2\": {\"x\": 240, \"y\": -135}\n" + " }\n" + "}" + "```\n" + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_query_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM:", raw_response) + + updated_data = extract_json_from_llm_response(raw_response) + sprite_positions = updated_data.get("sprite_initial_positions", {}) + + new_project_json = update_project_with_sprite_positions(state["project_json"], sprite_positions) + print("Updated project JSON with sprite positions:", json.dumps(new_project_json, indent=2)) + return {"project_json": new_project_json, "sprite_initial_positions": sprite_positions} + #return {"project_json": new_project_json} + + except Exception as e: + logger.error(f"Error in ParseQueryNode: {e}") + raise + + +# Node 3: Sprite Action Plan Builder +def overall_planner_node(state: GameState): + """ + Generates a comprehensive action plan for sprites, including detailed Scratch block information. + This node acts as an overall planner, leveraging knowledge of all block shapes and categories. + """ + logger.info("--- Running OverallPlannerNode ---") + + description = state.get("description", "") + project_json = state["project_json"] + sprite_names = [t["name"] for t in project_json["targets"] if not t["isStage"]] + + # Get sprite positions from the project_json if available, or set defaults + sprite_positions = {} + for target in project_json["targets"]: + if not target["isStage"]: + sprite_positions[target["name"]] = {"x": target.get("x", 0), "y": target.get("y", 0)} + + planning_prompt = ( + "Generate a detailed action plan for the game's sprites based on the user query and sprite details.\n\n" + f"The game description is: '{description}'.\n\n" + f"Sprites in game: {', '.join(sprite_names)}\n" + f"Sprites current positions: {json.dumps(sprite_positions)}\n\n" + "--- Scratch 3.0 Block Reference ---\n" + "This section provides a comprehensive reference of Scratch 3.0 blocks, categorized by their shape, " + "including their programmatic identifiers (opcodes) and functional descriptions. Use this to accurately " + "identify opcodes and understand block behavior for your plans.\n\n" + + f"### Hat Blocks\n" + f"Description: {hat_description}\n" + f"Blocks:\n{hat_opcodes_functionalities}\n\n" + + f"### Boolean Blocks\n" + f"Description: {boolean_description}\n" + f"Blocks:\n{boolean_opcodes_functionalities}\n\n" + + f"### C Blocks\n" + f"Description: {c_description}\n" + f"Blocks:\n{c_opcodes_functionalities}\n\n" + + f"### Cap Blocks\n" + f"Description: {cap_description}\n" + f"Blocks:\n{cap_opcodes_functionalities}\n\n" + + f"### Reporter Blocks\n" + f"Description: {reporter_description}\n" + f"Blocks:\n{reporter_opcodes_functionalities}\n\n" + + f"### Stack Blocks\n" + f"Description: {stack_description}\n" + f"Blocks:\n{stack_opcodes_functionalities}\n\n" + + "-----------------------------------\n\n" + "Your task is to define the primary actions and movements for each sprite. " + "The output should be a JSON object with a single key 'action_overall_flow'. " + "The value should be a dictionary where keys are sprite names (e.g., 'Player', 'Enemy') " + "and values are dictionaries containing a 'description' and a 'plans' list. " + "Each 'plan' in the list must correspond to a single Scratch Hat Block (event block). " + "For each plan, you must identify and include the following:\n" + "1. The 'event' field: the exact `opcode` of the hat block (e.g., 'event_whenflagclicked', 'event_whenkeypressed', 'control_start_as_clone', 'event_whenbroadcastreceived'). " + "2. The 'logic' field: a natural language description of all sequential actions under that single event. " + " Separate distinct actions within the 'logic' string with a semicolon ';'. " + " Each action should be as granular as possible, describing one Scratch block or a very small, tightly coupled sequence of directly connected blocks. " + " Avoid combining complex control flows or multiple distinct operations into a single action if they can be broken down naturally." + " For example, for a jump, combine: 'change y by N; wait M seconds; change y by -N'." + " For continuous movement, differentiate 'move N steps' from 'glide N seconds to x:Y y:Z'." + " When an action needs to repeat indefinitely (e.g., with a 'forever' block), explicitly state 'forever: ' at the beginning of the logic for that section, followed by the actions inside it." + " Ensure the language used in 'logic' directly translates to Scratch block actions, using verbs like 'move', 'change', 'set', 'say', 'hide', 'show', 'broadcast', 'wait', 'if', 'repeat', etc., consistent with block functionalities." + "3. Explicit lists of Scratch block opcodes by category: `motion`, `control`, `operators`, `sensing`, `looks`, `sound`, `events`, `data`. " + " These lists should contain ALL relevant Scratch block `opcode`s (not just names) that would be used to implement the 'logic' for that specific plan, including shadow blocks for values or dropdowns if they are an integral part of the block's operation (e.g., `math_number` for numerical inputs, `event_whenkeypressed_keymenu` for key options). " + " **Crucially, derive these opcodes from the detailed actions described in the 'logic' string, utilizing the 'Scratch 3.0 Block Reference' provided above.** " + " Include only the categories that have relevant blocks for that plan; omit empty lists." + "Ensure the logic clearly distinguishes between 'move', 'glide', 'go to', and 'change x/y by' as they are distinct block types." + "Refer to sprite names precisely as they appear in `sprite_names` (e.g., 'Sprite1', 'soccer ball')." + "\n\nExample structure for 'action_overall_flow' (follow this precise format, including all relevant opcode lists and ensuring correct opcode identification):\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at staring point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"repeat(10):(change y by value 10; wait 0.1 seconds; change y by value -10);\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_wait\",\"control_repeat\",\"control_wait\"],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement and interaction\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135; forever: glide 2 seconds to x:-240 y:-135; if x position < -235, then set x to 240; if touching Sprite1, then hide;\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\", \"motion_setx\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operators\": [\"operators_lt\"],\n" + " \"sensing\": [\"sensing_istouching\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [\"looks_hide\"],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Based on the provided context, generate the `action_overall_flow`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM [OverallPlannerNode]:", raw_response) # Uncomment for debugging + overall_plan = extract_json_from_llm_response(raw_response) + state["action_plan"] = overall_plan + logger.info("Overall plan generated by OverallPlannerNode.") + return state + except Exception as e: + logger.error(f"Error in OverallPlannerNode: {e}") + raise + +# Helper function to get a block by its opcode from a single catalog +def get_block_by_opcode(catalog_data: dict, opcode: str) -> dict | None: + """ + Search a single catalog (with keys "description" and "blocks": List[dict]) + for a block whose 'op_code' matches the given opcode. + Returns the block dict or None if not found. + """ + for block in catalog_data["blocks"]: + if block.get("op_code") == opcode: + return block + return None + +# Helper function to find a block in all catalogs by opcode +def find_block_in_all(opcode: str, all_catalogs: list[dict]) -> dict | None: + """ + Search across multiple catalogs for a given opcode. + Returns the first matching block dict or None. + """ + for catalog in all_catalogs: + blk = get_block_by_opcode(catalog, opcode) + if blk is not None: + return blk + return None + +# Node 4: Sprite Plan Verification Node +def plan_verification_node(state: GameState): + """ + Validates the generated action plan/blocks, identifies missing logic, + provides feedback, and determines if further improvements are needed. + Also manages the iteration count for the improvement loop. + """ + logger.info(f"--- Running VerificationNode (Iteration: {state.get('iteration_count', 0)}) ---") + + MAX_IMPROVEMENT_ITERATIONS = 2 # Set a sensible limit to prevent infinite loops + current_iteration = state.get("iteration_count", 0) + #project_json = state["project_json"] + action_plan = state.get("action_plan", {}) + #improvement_plan = state.get("improvement_plan", {}) # May contain prior improvement guidance + + # Corrected validation_prompt + validation_prompt = ( + f"You are an AI validator for Scratch project plans and generated blocks. " + f"Your task is to review the current state of the game's action plan and generated blocks. " + f"Critically analyze if there is any 'missing logic' or inconsistencies, and provide concrete feedback for improvement.\n\n" + f"**Game description in detail:**\n" + #f"'{state.get('detailed_game_description', state.get('description', ''))}'.\n\n" # Use detailed_game_description + f"'{state.get('description', '')}'.\n\n" + f"**Current Action Plan (High-Level Logic):**\n" + f"```json\n{json.dumps(action_plan, indent=2)}\n```\n\n" + #f"**Current Project JSON (Generated Blocks):**\n" + #f"```json\n{json.dumps(project_json, indent=2)}\n```\n\n" + f"**Previous Improvement Feedback (if any):**\n" + f"{state.get('plan_validation_feedback', 'None')}\n\n" + f"Based on the above, answer the following in JSON format:\n" + "```json\n" + "{\n" + " \"feedback\": \"Your detailed feedback on missing logic, inconsistencies, or areas for improvement. Be specific. If everything is perfect, state so.\",\n" + " \"needs_improvement\": true/false,\n" + " \"suggested_description_updates\": \"Any suggested phrasing to update the overall game description based on identified gaps and be minimalistic. Empty string if no updates needed.\"\n" + "}\n" + "```\n" + "**Important:** For the 'needs_improvement' field, you MUST output either `true` or `false` (boolean values). Do NOT include any additional text or comments within the JSON structure for this specific field. " + f"Be very strict: if any part of the plan or blocks seems incomplete or deviates from a perfect implementation, set 'needs_improvement' to true." + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": validation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [VerificationNode]: {raw_response[:500]}...") + + validation_result = extract_json_from_llm_response(raw_response) + + # Update state with feedback and improvement flag + state["plan_validation_feedback"] = validation_result.get("feedback", "No specific feedback provided.") + state["needs_improvement"] = validation_result.get("needs_improvement", False) + + suggested_description_updates = validation_result.get("suggested_description_updates", "") + if suggested_description_updates: + # You might want to append or intelligently merge this with the existing detailed_game_description + # For simplicity, let's just append for now or update a specific field + #current_description = state.get("detailed_game_description", state.get("description", "")) + current_description = state.get(state.get("description", "")) + #state["detailed_game_description"] = f"{current_description}\n\nSuggested Update: {suggested_description_updates}"\ + state["description"] = f"{current_description}\n\nSuggested Update: {suggested_description_updates}" + logger.info("Updated detailed game description based on validation feedback.") + + + # Manage iteration count + if state["needs_improvement"]: + state["iteration_count"] = current_iteration + 1 + if state["iteration_count"] >= MAX_IMPROVEMENT_ITERATIONS: + logger.warning(f"Max improvement iterations ({MAX_IMPROVEMENT_ITERATIONS}) reached. Forcing 'needs_improvement' to False.") + state["needs_improvement"] = False + state["plan_validation_feedback"] += "\n(Note: Max iterations reached, stopping further improvements.)" + else: + state["iteration_count"] = 0 # Reset if no more improvement needed + + logger.info(f"Verification completed. Needs Improvement: {state['needs_improvement']}. Feedback: {state['plan_validation_feedback'][:100]}...") + print(f"[updated action_plan after verification] on ({current_iteration}): {json.dumps(state.get("action_plan", {}), indent=2)}") + return state + except Exception as e: + logger.error(f"Error in VerificationNode: {e}") + state["needs_improvement"] = False # Force end loop on error + state["plan_validation_feedback"] = f"Validation error: {e}" + raise + +# Node 5: Refined Planner Node +def refined_planner_node(state: GameState): + """ + Refines the action plan based on validation feedback and game description. + """ + logger.info("--- Running RefinedPlannerNode ---") + + #detailed_game_description = state.get("detailed_game_description", state.get("description", "A game.")) + detailed_game_description = state.get("description","A game.") + current_action_plan = state.get("action_plan", {}) + plan_validation_feedback = state.get("plan_validation_feedback", "No specific feedback provided. Assume general refinement is needed.") + + refinement_prompt = ( + "Refine the existing action plan for the game's sprites based on the detailed game description and specific validation feedback.\n\n" + f"**Detailed Game Description:** '{detailed_game_description}'\n\n" + f"**Current Action Plan (to be refined):**\n" + f"```json\n{json.dumps(current_action_plan, indent=2)}\n```\n\n" + f"**Validation Feedback (from previous verification):**\n" + f"'{plan_validation_feedback}'\n\n" + "--- Scratch 3.0 Block Reference ---\n" + f"### Hat Blocks\nDescription: {hat_description}\nBlocks:\n{hat_opcodes_functionalities}\n\n" + f"### Boolean Blocks\nDescription: {boolean_description}\nBlocks:\n{boolean_opcodes_functionalities}\n\n" + f"### C Blocks\nDescription: {c_description}\nBlocks:\n{c_opcodes_functionalities}\n\n" + f"### Cap Blocks\nDescription: {cap_description}\nBlocks:\n{cap_opcodes_functionalities}\n\n" + f"### Reporter Blocks\nDescription: {reporter_description}\nBlocks:\n{reporter_opcodes_functionalities}\n\n" + f"### Stack Blocks\nDescription: {stack_description}\nBlocks:\n{stack_opcodes_functionalities}\n\n" + "-----------------------------------\n\n" + "Your task is to refined JSON object for 'action_overall_flow' which is as above. " + "Refer to sprite names precisely as they appear in `sprite_names` (e.g., 'Sprite1', 'soccer ball')and make sure you donot change its names. " + "\n\nExample structure for 'action_overall_flow' with update in the section from feedbacks (follow this precise format, including all relevant opcode lists and ensuring correct opcode identification):\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at staring point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"repeat(10):(change y by value 10; wait 0.1 seconds; change y by value -10);\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_wait\",\"control_repeat\",\"control_wait\"],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement and interaction\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135; forever: glide 2 seconds to x:-240 y:-135; if x position < -235, then set x to 240; if touching Sprite1, then hide;\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\", \"motion_setx\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operators\": [\"operators_lt\"],\n" + " \"sensing\": [\"sensing_istouching\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [\"looks_hide\"],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Address the validation feedback by correcting errors, adding missing logic, or improving existing logic. " + "Maintain the exact JSON structure as in the 'Current Action Plan' example. " + "Ensure all 'logic' descriptions are clear, granular, and all listed opcodes are accurate and directly relevant to the refined logic. " + "Only include categories with relevant opcodes. " + "If the feedback suggests a major change, rethink the plan for the affected sprite(s). " + "If the feedback is minimal, make small, precise adjustments." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": refinement_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [RefinedPlannerNode]: {raw_response[:500]}...") + + refined_plan = extract_json_from_llm_response(raw_response) + if refined_plan: + state["action_plan"] = refined_plan.get("action_overall_flow", {}) # Update the main action plan + logger.info("Action plan refined by RefinedPlannerNode.") + else: + logger.warning("RefinedPlannerNode did not return a valid 'action_overall_flow' structure. Keeping previous plan.") + print("[Refined Action Plan]:", json.dumps(state["action_plan"], indent=2)) + print("[current state after refinement]:", json.dumps(state, indent=2)) + return state + except Exception as e: + logger.error(f"Error in RefinedPlannerNode: {e}") + raise + +# Node 4: Overall Block Builder Node +def overall_block_builder_node(state: GameState): + logger.info("--- Running OverallBlockBuilderNode ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + # Pre-load all block-catalog JSONs once + all_catalogs = [ + hat_block_data, + boolean_block_data, + c_block_data, + cap_block_data, + reporter_block_data, + stack_block_data + ] + + + action_plan = state.get("action_plan", {}) + if not action_plan: + logger.warning("No action plan found in state. Skipping OverallBlockBuilderNode.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + for sprite_name, sprite_actions_data in action_plan.get("action_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_actions_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operators_opcodes = plan_entry.get("operators", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + # Create a string representation of the identified opcodes for the prompt + identified_opcodes_str = "" + if motion_opcodes: + identified_opcodes_str += f" Motion Blocks (opcodes): {', '.join(motion_opcodes)}\n" + if control_opcodes: + identified_opcodes_str += f" Control Blocks (opcodes): {', '.join(control_opcodes)}\n" + if operators_opcodes: + identified_opcodes_str += f" Operator Blocks (opcodes): {', '.join(operators_opcodes)}\n" + if sensing_opcodes: + identified_opcodes_str += f" Sensing Blocks (opcodes): {', '.join(sensing_opcodes)}\n" + if looks_opcodes: + identified_opcodes_str += f" Looks Blocks (opcodes): {', '.join(looks_opcodes)}\n" + if sound_opcodes: + identified_opcodes_str += f" Sound Blocks (opcodes): {', '.join(sound_opcodes)}\n" + if events_opcodes: + identified_opcodes_str += f" Event Blocks (opcodes): {', '.join(events_opcodes)}\n" + if data_opcodes: + identified_opcodes_str += f" Data Blocks (opcodes): {', '.join(data_opcodes)}\n" + + needed_opcodes = motion_opcodes + control_opcodes + operators_opcodes + sensing_opcodes + looks_opcodes + sound_opcodes + events_opcodes + data_opcodes + needed_opcodes = list(set(needed_opcodes)) + + + # 2) build filtered runtime catalog (if you still need it) + filtered_catalog = { + op: ALL_SCRATCH_BLOCKS_CATALOG[op] + for op in needed_opcodes + if op in ALL_SCRATCH_BLOCKS_CATALOG + } + + # 3) merge human‑written catalog + runtime entry for each opcode + combined_blocks = {} + for op in needed_opcodes: + catalog_def = find_block_in_all(op, all_catalogs) or {} + runtime_def = ALL_SCRATCH_BLOCKS_CATALOG.get(op, {}) + # merge: catalog fields first, then runtime overrides/adds + merged = {**catalog_def, **runtime_def} + combined_blocks[op] = merged + #print("Combined blocks for this script:", json.dumps(combined_blocks, indent=2)) + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is for the event with opcode '{event_opcode}'.\n" + f"The sequential logic to implement is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Based on the planning, the following Scratch block opcodes are expected to be used to implement this logic. Focus on using these specific opcodes where applicable, and refer to the ALL_SCRATCH_BLOCKS_CATALOG for their full structure and required inputs/fields:**\n" + f"Here is the comprehensive catalog of required Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(combined_blocks, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON (EXTREMELY IMPORTANT - FOLLOW THESE EXAMPLES PRECISELY):**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'myBlockID123').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Handle `inputs` for parameters and substacks (CRITICAL DETAIL - PAY CLOSE ATTENTION TO EXAMPLES):**\n" + f" * The value for any key within the `inputs` dictionary MUST always be an **array** of two elements: `[type_code, value_or_block_id]`.\n" + f" * **STRICTLY FORBIDDEN MISTAKE:** DO NOT put an array like `[\"num\", \"value\"]` or `[\"_edge_\", null]` directly as the `value_or_block_id`. This is the source of past errors.\n" + f" * The `type_code` (first element of the array) indicates the nature of the input:\n" + f" * `1`: For a primitive value or a shadow block ID (e.g., a number, string, boolean, or reference to a shadow block). This is the most common type for direct values.\n" + f" * `2`: For a block ID where another block is directly plugged into this input (e.g., an operator block connected to an `if` condition, or the first block of a C-block's substack).\n" + f" * **Correct Example for Numerical Input (e.g., for `motion_movesteps` STEPS, or `motion_gotoxy` X/Y):**\n" + f" If you need to input the number `10` into a block, you MUST create a separate `math_number` shadow block for it, and then reference its ID.\n" + f" ```json\n" + f" // Main block using the number\n" + f" \"mainBlockID\": {{\n" + f" \"opcode\": \"motion_movesteps\",\n" + f" \"inputs\": {{\n" + f" \"STEPS\": [1, \"shadowNumID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the number '10'\n" + f" \"shadowNumID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"mainBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for Dropdown/Menu Input (e.g., `sensing_touchingobject` with 'edge'):**\n" + f" If you need to select 'edge' for the `touching ()?` block, you MUST create a separate `sensing_touchingobjectmenu` shadow block and reference its ID.\n" + f" ```json\n" + f" // Main block using the dropdown selection\n" + f" \"touchingBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\",\n" + f" \"inputs\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [1, \"shadowEdgeMenuID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the 'edge' menu option\n" + f" \"shadowEdgeMenuID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\",\n" + f" \"fields\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [\"_edge_\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"touchingBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block (e.g., `control_forever`):**\n" + f" The `control_forever` block MUST have a `SUBSTACK` input pointing to the first block inside its loop. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_FOREVER_LOOP_ID\"]`.\n" + f" ```json\n" + f" \"foreverBlockID\": {{\n" + f" \"opcode\": \"control_forever\",\n" + f" \"inputs\": {{\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideForeverID\"]\n" + f" }},\n" + f" \"next\": null,\n" + f" \"parent\": \"blockAboveForeverID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"firstBlockInsideForeverID\": {{\n" + f" // ... definition of the first block inside the forever loop\n" + f" \"parent\": \"foreverBlockID\",\n" + f" \"next\": \"secondBlockInsideForeverID\" // if there's another block\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block with condition (e.g., `control_if`):**\n" + f" The `control_if` block MUST have a `CONDITION` input (typically `type_code: 1` referencing a boolean reporter block) and a `SUBSTACK` input (`type_code: 2` referencing the first block inside the if-body).\n" + f" ```json\n" + f" \"ifBlockID\": {{\n" + f" \"opcode\": \"control_if\",\n" + f" \"inputs\": {{\n" + f" \"CONDITION\": [1, \"conditionBlockID\"],\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideIfID\"]\n" + f" }},\n" + f" \"next\": \"blockAfterIfID\",\n" + f" \"parent\": \"blockAboveIfID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"conditionBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", // Example condition block\n" + f" // ... definition for condition block, parent should be \"ifBlockID\"\n" + f" }},\n" + f" \"firstBlockInsideIfID\": {{\n" + f" // ... definition of the first block inside the if body\n" + f" \"parent\": \"ifBlockID\",\n" + f" \"next\": null // or next block if more\n" + f" }}\n" + f" ```\n" + f"5. **Define ALL Shadow Blocks Separately (THIS IS ESSENTIAL):** Every time a block's input requires a number, string literal, or a selection from a dropdown/menu, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow. Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f"6. **`fields` for direct dropdown values/text:** Use the `fields` dictionary ONLY IF the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., the `KEY_OPTION` field within the `event_whenkeypressed_keymenu` shadow block itself, or the `variable` field on a `data_setvariableto` block). Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Ensure Unique Block IDs:** Every block you generate (main blocks and shadow blocks) must have a unique ID within the entire script's block dictionary.\n" + f"9. **Strictly Use Catalog Opcodes:** You MUST only use `opcode` values that are present in the provided `ALL_SCRATCH_BLOCKS_CATALOG`. Do NOT use unlisted opcodes like `motion_jump`.\n" + f"10. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [OverallBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + + generated_blocks = extract_json_from_llm_response(raw_response) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + + # setting the iteration count for the script + state["iteration_count"] = 0 + + logger.info(f"Action blocks added for sprite '{sprite_name}', script '{event_opcode}' by OverallBlockBuilderNode.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with action nodes.") + print("Updated project JSON with action nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + +def get_block_type(opcode: str) -> str: + """Determines the general type of a Scratch block based on its opcode.""" + if not opcode: + return "unknown" + if opcode.startswith("event_when") or opcode == "control_start_as_clone": + return "hat" + elif opcode.startswith("control_") and ("if" in opcode or "repeat" in opcode or "forever" in opcode): + return "c_block" + elif opcode in ["operators_equals", "operators_gt", "operators_lt", "operators_and", "operators_or", "operators_not"] or \ + (opcode.startswith("sensing_") and ("mousedown" in opcode or "keypressed" in opcode or "touching" in opcode)): + return "boolean" + elif opcode.endswith("menu"): # For dropdown shadow blocks, treat as reporter for type checking + return "reporter" + elif any(s in opcode for s in ["position", "direction", "size", "volume", "costume", "backdrop", "random", "add", "subtract", "multiply", "divide", "length", "item", "of"]): + # A more comprehensive check for reporters + return "reporter" + elif "stop_all" in opcode or "delete_this_clone" in opcode or "procedures_definition" in opcode: + return "cap" + # Default to stack for most command blocks if not explicitly defined + return ALL_SCRATCH_BLOCKS_CATALOG.get(opcode, {}).get("blockType", "stack") + + +def filter_script_blocks(all_blocks: dict, hat_block_id: str) -> dict: + """ + Filters and returns only the blocks that are part of a specific script + starting from the given hat_block_id, including connected reporters/shadows. + """ + script_blocks = {} + q = [hat_block_id] + visited = set() + + while q: + current_block_id = q.pop(0) + if current_block_id in visited: + continue + visited.add(current_block_id) + + block_data = all_blocks.get(current_block_id) + if not block_data: + continue + + script_blocks[current_block_id] = block_data + + # Add next block in sequence + next_id = block_data.get("next") + if next_id and next_id in all_blocks: + q.append(next_id) + + # Add blocks connected via inputs (e.g., reporters, shadow blocks, substacks) + if "inputs" in block_data: + for input_name, input_value in block_data["inputs"].items(): + if isinstance(input_value, list) and len(input_value) >= 2: + value_or_block_id = input_value[1] + if isinstance(value_or_block_id, str) and value_or_block_id in all_blocks: + q.append(value_or_block_id) + # For type code 3 (reporter with default value), the third element might be a connected block + if len(input_value) >= 3 and isinstance(input_value[2], str) and input_value[2] in all_blocks: + q.append(input_value[2]) + + # For C-blocks, add blocks in substacks (if present) + # SUBSTACKs are inputs that have type code 2 and contain the block ID of the first block in the substack + if get_block_type(block_data.get("opcode")) == "c_block": + for input_key, input_val in block_data.get("inputs", {}).items(): + if input_key.startswith("SUBSTACK") and isinstance(input_val, list) and len(input_val) >= 2 and input_val[0] == 2 and isinstance(input_val[1], str) and input_val[1] in all_blocks: + q.append(input_val[1]) + + return script_blocks + + +def analyze_script_structure(script_blocks: dict, hat_block_id: str, sprite_name: str) -> list: + """ + Analyzes the structure of a single Scratch script for common errors. + Returns a list of issue strings. + """ + issues = [] + + # 1. Validate the hat block + hat_block = script_blocks.get(hat_block_id) + if not hat_block: + issues.append(f"Script for sprite '{sprite_name}' (hat ID: {hat_block_id}) has no hat block data.") + return issues # Cannot proceed without a hat block + + if not hat_block.get("topLevel") or hat_block.get("parent") is not None: + issues.append(f"Hat block '{hat_block_id}' for sprite '{sprite_name}' is not marked as topLevel or has a parent.") + + # 2. Check all blocks within the script + for block_id, block_data in script_blocks.items(): + opcode = block_data.get("opcode") + if not opcode: + issues.append(f"Block '{block_id}' for sprite '{sprite_name}' is missing an opcode.") + continue + + # Check if opcode exists in the catalog (simplified check for this example) + if opcode not in ALL_SCRATCH_BLOCKS_CATALOG: + issues.append(f"Block '{block_id}' for sprite '{sprite_name}' has unknown opcode '{opcode}'.") + + # Parent-Child and Next-Previous Linkage + parent_id = block_data.get("parent") + next_id = block_data.get("next") + + if parent_id: + parent_block = script_blocks.get(parent_id) + if not parent_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' references non-existent parent '{parent_id}'.") + else: + parent_block_type = get_block_type(parent_block.get("opcode")) + current_block_type = get_block_type(opcode) + + if parent_block_type in ["stack", "hat"]: + # For stack and hat blocks, the parent's 'next' should point to this block + if parent_block.get("next") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode})'s parent '{parent_id}' does not link back to it via 'next'.") + elif parent_block_type == "c_block": + # For C-blocks, this block should be in a substack input + found_in_substack = False + for input_key, input_val in parent_block.get("inputs", {}).items(): + if isinstance(input_val, list) and len(input_val) >= 2 and input_val[1] == block_id: + if input_val[0] == 2 and input_key.startswith("SUBSTACK"): # Type code 2 for substacks + found_in_substack = True + break + # Shadow blocks and reporter blocks can also be inputs to C-blocks but not as substacks + if not found_in_substack and not block_data.get("shadow") and current_block_type not in ["reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) has parent '{parent_id}' but is not a substack, shadow, or reporter/boolean connection to a C-block.") + elif parent_block_type in ["reporter", "boolean"]: + # Reporter/Boolean blocks can be parents if they are input to another block + found_in_input = False + for input_key, input_val in parent_block.get("inputs", {}).items(): + if isinstance(input_val, list) and len(input_val) >= 2 and input_val[1] == block_id: + found_in_input = True + break + if not found_in_input: + issues.append(f"Block '{block_id}' (opcode: {opcode}) has reporter/boolean parent '{parent_id}' but is not linked via an input.") + else: # e.g., cap blocks cannot be parents to other blocks in a sequence + issues.append(f"Block '{block_id}' (opcode: {opcode}) has an unexpected parent block type '{parent_block_type}' for parent '{parent_id}'.") + + + if next_id: + next_block = script_blocks.get(next_id) + if not next_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' references non-existent next block '{next_id}'.") + elif next_block.get("parent") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode})'s next block '{next_id}' does not link back to it via 'parent'.") + elif block_data.get("shadow"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection.") + elif get_block_type(opcode) == "hat": # Hat blocks should not generally have a 'next' in a linear script sequence + issues.append(f"Hat block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection as a sequential block.") + elif get_block_type(opcode) == "cap": + issues.append(f"Cap block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection as it signifies script end.") + + + # Input validation + if "inputs" in block_data: + for input_name, input_value in block_data["inputs"].items(): + if not isinstance(input_value, list) or len(input_value) < 2: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' has malformed input '{input_name}': {input_value}.") + continue + + type_code = input_value[0] + value_or_block_id = input_value[1] + + # Type code 1: number/string/boolean literal or block ID (reporter/boolean) + # Type code 2: block ID (substack or reporter/boolean plugged in) + # Type code 3: number/string/boolean literal with shadow, or reporter with default value if nothing plugged in + if type_code not in [1, 2, 3]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has invalid type code: {type_code}. Expected 1, 2, or 3.") + + if isinstance(value_or_block_id, str): + # It's a block ID, check if it exists and its parent link is correct + connected_block = script_blocks.get(value_or_block_id) + if not connected_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' references non-existent block ID '{value_or_block_id}'.") + elif connected_block.get("parent") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' connects to '{value_or_block_id}', but '{value_or_block_id}'s parent is not '{block_id}'.") + # Check for type code consistency with connected block type + elif type_code == 2 and get_block_type(connected_block.get("opcode")) not in ["stack", "hat", "c_block", "cap", "reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has type code 2 but connected block '{value_or_block_id}' is not a valid block type for substack/input.") + elif type_code == 1 and get_block_type(connected_block.get("opcode")) not in ["reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has type code 1 but connected block '{value_or_block_id}' is not a reporter or boolean (expected for direct value input).") + + # Specific checks for C-blocks' SUBSTACK + if block_data.get("blockType") == "c_block" and input_name.startswith("SUBSTACK"): # Changed to startswith as SUBSTACK1, SUBSTACK2 might exist + if not (isinstance(value_or_block_id, str) and script_blocks.get(value_or_block_id) and type_code == 2): + issues.append(f"C-block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' has an invalid or missing SUBSTACK input configuration.") + + + # Shadow block specific checks + if block_data.get("shadow"): + if block_data.get("topLevel"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' is incorrectly marked as topLevel.") + if not block_data.get("parent"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' is missing a parent.") + if block_data.get("next"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection.") + # Check fields for math_number and menu blocks + if opcode == "math_number" and not (isinstance(block_data.get("fields", {}).get("NUM"), list) and len(block_data["fields"]["NUM"]) >= 1 and isinstance(block_data["fields"]["NUM"][0], (str, int, float))): + issues.append(f"Math_number shadow block '{block_id}' (opcode: {opcode}) has malformed 'NUM' field.") + # This logic for menu shadow blocks assumes the field name is the opcode in uppercase without _MENU + # Example: for "looks_costumemenu", it expects a field "COSTUME" + if opcode.endswith("menu"): + expected_field = opcode.upper().replace("_MENU", "") + if not (isinstance(block_data.get("fields", {}).get(expected_field), list) and len(block_data["fields"][expected_field]) >= 1 and isinstance(block_data["fields"][expected_field][0], str)): + issues.append(f"Menu shadow block '{block_id}' (opcode: {opcode}) has malformed field '{expected_field}' for its specific menu option.") + + + return issues + +# Node 5: Verification Node +def block_verification_node(state: dict) -> dict: + logger.info(f"--- Running BlockVerificationNode (Iteration: {state.get('iteration_count', 0)}) ---") + + MAX_IMPROVEMENT_ITERATIONS = 3 # Set a sensible limit to prevent infinite loops + current_iteration = state.get("iteration_count", 0) + project_json = state["project_json"] + targets = project_json["targets"] + + # Initialize needs_improvement for the current run + state["needs_improvement"] = False + block_validation_feedback_overall = [] + + improvement_plan = {"sprite_issues": {}} + + for target in targets: + sprite_name = target["name"] + all_blocks_for_sprite = target.get("blocks", {}) + + if not all_blocks_for_sprite: + logger.info(f"Sprite '{sprite_name}' has no blocks. Skipping verification.") + continue + + sprite_issues = [] + hat_block_ids = [ + block_id for block_id, block_data in all_blocks_for_sprite.items() + if block_data.get("topLevel") and get_block_type(block_data.get("opcode")) == "hat" + ] + + processed_script_blocks = set() + + if not hat_block_ids: + sprite_issues.append("No top-level hat blocks found for this sprite. Scripts may not run.") + for block_id, block_data in all_blocks_for_sprite.items(): + if block_data.get("topLevel") and not get_block_type(block_data.get("opcode")) == "hat": + sprite_issues.append(f"Top-level block '{block_id}' (opcode: {block_data.get('opcode')}) is not a hat block, so it will not run automatically.") + if not block_data.get("topLevel") and not block_data.get("parent") and not block_data.get("shadow"): + sprite_issues.append(f"Orphaned block '{block_id}' (opcode: {block_data.get('opcode')}) is not top-level, has no parent, and is not a shadow block.") + + for hat_id in hat_block_ids: + logger.info(f"Verifying script starting with hat block '{hat_id}' for sprite '{sprite_name}'.") + + current_script_blocks = filter_script_blocks(all_blocks_for_sprite, hat_id) + processed_script_blocks.update(current_script_blocks.keys()) + + script_issues = analyze_script_structure(current_script_blocks, hat_id, sprite_name) + if script_issues: + sprite_issues.append(f"Issues in script starting with '{hat_id}':") + sprite_issues.extend([f" - {issue}" for issue in script_issues]) + else: + logger.info(f"Script starting with '{hat_id}' for sprite '{sprite_name}' passed basic verification.") + + orphaned_blocks_overall = { + block_id for block_id in all_blocks_for_sprite.keys() + if block_id not in processed_script_blocks + and not all_blocks_for_sprite[block_id].get("topLevel") + and not all_blocks_for_sprite[block_id].get("parent") + } + + if orphaned_blocks_overall: + sprite_issues.append(f"Found {len(orphaned_blocks_overall)} truly orphaned blocks not connected to any valid script: {', '.join(list(orphaned_blocks_overall)[:5])}{'...' if len(orphaned_blocks_overall) > 5 else ''}.") + + if sprite_issues: + improvement_plan["sprite_issues"][sprite_name] = sprite_issues + logger.warning(f"Verification found issues for sprite '{sprite_name}'.") + block_validation_feedback_overall.append(f"Issues for {sprite_name}:\n" + "\n".join([f"- {issue}" for issue in sprite_issues])) + state["needs_improvement"] = True + + print(f"\n--- Verification Report (Issues Found for {sprite_name}) ---") + print(json.dumps({sprite_name: sprite_issues}, indent=2)) + else: + logger.info(f"Sprite '{sprite_name}' passed all verification checks.") + + # Consolidate feedback for the LLM + state["block_validation_feedback"] = "\n\n".join(block_validation_feedback_overall) + + if state["needs_improvement"]: + + llm_reviewer_prompt = ( + "You are an expert Scratch project reviewer. Your task is to analyze the provided " + "structural issues found in a Scratch project's sprites and suggest improvements or " + "further insights. Focus on clarity, accuracy, and actionable advice.\n\n" + "Here are the detected structural issues:\n" + f"{json.dumps(improvement_plan['sprite_issues'], indent=2)}\n\n" + "Please review these issues and provide a consolidated report with potential causes " + "and recommendations for fixing them. If an issue is minor or expected in certain " + "scenarios (e.g., hidden blocks for backward compatibility), please note that." + "Structure your response as a JSON object with 'review_summary' as the key, " + "containing a dictionary where keys are sprite names and values are lists of suggested improvements." + "Example:\n" + "```json\n" + "{\n" + " \"review_summary\": {\n" + " \"Sprite1\": [\n" + " \"Issue: Hat block 'abc' is not topLevel. Recommendation: Ensure all scripts start with a top-level hat block that has no parent.\",\n" + " \"Issue: Block 'xyz' has unknown opcode 'motion_nonexistent'. Recommendation: Verify the opcode against the Scratch 3.0 block reference. This might be a typo or a deprecated block.\"\n" + " ],\n" + " \"Sprite2\": [\n" + " \"Issue: Found 3 orphaned blocks. Recommendation: Reconnect these blocks to existing scripts or remove them if no longer needed.\"\n" + " ]\n" + " }\n" + "}\n" + "```" + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_reviewer_prompt}]}) + raw_review_response = response["messages"][-1].content + state["review_block_feedback"] = extract_json_from_llm_response(raw_review_response) + logger.info("Agent review feedback added to the state.") + print("\n--- Agent Review Feedback ---") + print(json.dumps(state["review_block_feedback"], indent=2)) + + except Exception as e: + logger.error(f"Error invoking agent for review in BlockVerificationNode: {e}") + state["review_block_feedback"] = {"review_summary": {"Overall": [f"Error during LLM review: {e}"]}} + else: + logger.info("BlockVerificationNode completed: No issues found in any sprite blocks.") + print("\n--- Verification Report (No Issues Found) ---") + state["block_validation_feedback"] = "No issues found in sprite blocks." + state["review_block_feedback"] = {"review_summary": {"Overall": ["No issues found in sprite blocks. All good!"]}} + + # Manage iteration count based on overall needs_improvement flag + if state["needs_improvement"]: + state["iteration_count"] = current_iteration + 1 + if state["iteration_count"] >= MAX_IMPROVEMENT_ITERATIONS: + logger.warning(f"Max improvement iterations ({MAX_IMPROVEMENT_ITERATIONS}) reached for block verification. Forcing 'needs_improvement' to False.") + state["needs_improvement"] = False + state["block_validation_feedback"] += "\n(Note: Max iterations reached for block verification, stopping further improvements.)" + state["improvement_plan"] = improvement_plan + logger.info("BlockVerificationNode found issues and added an improvement plan to the state.") + print("\n--- Overall Verification Report (Issues Found) ---") + print(json.dumps(improvement_plan, indent=2)) + else: + state["iteration_count"] = 0 # Reset if no more improvement needed for blocks + + logger.info(f"Block verification completed. Needs Improvement: {state['needs_improvement']}. Feedback: {state['block_validation_feedback'][:100]}...") + return state + +def improvement_block_builder_node(state: GameState): + logger.info("--- Running ImprovementBlockBuilderNode ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + # Pre-load all block-catalog JSONs once + all_catalogs = [ + hat_block_data, + boolean_block_data, + c_block_data, + cap_block_data, + reporter_block_data, + stack_block_data + ] + + improvement_plan = state.get("improvement_plan", {}) + block_verification_feedback = state.get("block_validation_feedback", "no feedback") + + if not improvement_plan: + logger.warning("No improvement plan found in state. Skipping ImprovementBlockBuilderNode.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + for sprite_name, sprite_improvements_data in improvement_plan.get("improvement_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_improvements_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operators_opcodes = plan_entry.get("operators", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + needed_opcodes = ( + motion_opcodes + control_opcodes + operators_opcodes + + sensing_opcodes + looks_opcodes + sound_opcodes + + events_opcodes + data_opcodes + ) + needed_opcodes = list(set(needed_opcodes)) + + # 2) build filtered runtime catalog (if you still need it) + filtered_catalog = { + op: ALL_SCRATCH_BLOCKS_CATALOG[op] + for op in needed_opcodes + if op in ALL_SCRATCH_BLOCKS_CATALOG + } + + # 3) merge human-written catalog + runtime entry for each opcode + combined_blocks = {} + for op in needed_opcodes: + catalog_def = find_block_in_all(op, all_catalogs) or {} + runtime_def = ALL_SCRATCH_BLOCKS_CATALOG.get(op, {}) + # merge: catalog fields first, then runtime overrides/adds + merged = {**catalog_def, **runtime_def} + combined_blocks[op] = merged + print("Combined blocks for this script:", json.dumps(combined_blocks, indent=2)) + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script based on an improvement plan.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is for the event with opcode '{event_opcode}'.\n" + f"The sequential logic to implement is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Based on the planning, the following Scratch block opcodes are expected to be used to implement this logic. Focus on using these specific opcodes where applicable, and refer to the ALL_SCRATCH_BLOCKS_CATALOG for their full structure and required inputs/fields:**\n" + f"Here is the comprehensive catalog of required Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(combined_blocks, indent=2)}\n```\n\n" + f"Here is feedback ans suggetion you should take care of:\n" + f" suggestion:{block_verification_feedback}\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON (EXTREMELY IMPORTANT - FOLLOW THESE EXAMPLES PRECISELY):**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'myBlockID123').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Handle `inputs` for parameters and substacks (CRITICAL DETAIL - PAY CLOSE ATTENTION TO EXAMPLES):**\n" + f" * The value for any key within the `inputs` dictionary MUST always be an **array** of two elements: `[type_code, value_or_block_id]`.\n" + f" * **STRICTLY FORBIDDEN MISTAKE:** DO NOT put an array like `[\"num\", \"value\"]` or `[\"_edge_\", null]` directly as the `value_or_block_id`. This is the source of past errors.\n" + f" * The `type_code` (first element of the array) indicates the nature of the input:\n" + f" * `1`: For a primitive value or a shadow block ID (e.g., a number, string, boolean, or reference to a shadow block). This is the most common type for direct values.\n" + f" * `2`: For a block ID where another block is directly plugged into this input (e.g., an operator block connected to an `if` condition, or the first block of a C-block's substack).\n" + f" * **Correct Example for Numerical Input (e.g., for `motion_movesteps` STEPS, or `motion_gotoxy` X/Y):**\n" + f" If you need to input the number `10` into a block, you MUST create a separate `math_number` shadow block for it, and then reference its ID.\n" + f" ```json\n" + f" // Main block using the number\n" + f" \"mainBlockID\": {{\n" + f" \"opcode\": \"motion_movesteps\",\n" + f" \"inputs\": {{\n" + f" \"STEPS\": [1, \"shadowNumID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the number '10'\n" + f" \"shadowNumID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"mainBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for Dropdown/Menu Input (e.g., `sensing_touchingobject` with 'edge'):**\n" + f" If you need to select 'edge' for the `touching ()?` block, you MUST create a separate `sensing_touchingobjectmenu` shadow block and reference its ID.\n" + f" ```json\n" + f" // Main block using the dropdown selection\n" + f" \"touchingBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\",\n" + f" \"inputs\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [1, \"shadowEdgeMenuID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the 'edge' menu option\n" + f" \"shadowEdgeMenuID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\",\n" + f" \"fields\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [\"_edge_\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"touchingBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block (e.g., `control_forever`):**\n" + f" The `control_forever` block MUST have a `SUBSTACK` input pointing to the first block inside its loop. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_FOREVER_LOOP_ID\"]`.\n" + f" ```json\n" + f" \"foreverBlockID\": {{\n" + f" \"opcode\": \"control_forever\",\n" + f" \"inputs\": {{\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideForeverID\"]\n" + f" }},\n" + f" \"next\": null,\n" + f" \"parent\": \"blockAboveForeverID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"firstBlockInsideForeverID\": {{\n" + f" // ... definition of the first block inside the forever loop\n" + f" \"parent\": \"foreverBlockID\",\n" + f" \"next\": \"secondBlockInsideForeverID\" // if there's another block\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block with condition (e.g., `control_if`):**\n" + f" The `control_if` block MUST have a `CONDITION` input (typically `type_code: 1` referencing a boolean reporter block) and a `SUBSTACK` input (`type_code: 2` referencing the first block inside the if-body).\n" + f" ```json\n" + f" \"ifBlockID\": {{\n" + f" \"opcode\": \"control_if\",\n" + f" \"inputs\": {{\n" + f" \"CONDITION\": [1, \"conditionBlockID\"],\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideIfID\"]\n" + f" }},\n" + f" \"next\": \"blockAfterIfID\",\n" + f" \"parent\": \"blockAboveIfID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"conditionBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", // Example condition block\n" + f" // ... definition for condition block, parent should be \"ifBlockID\"\n" + f" }},\n" + f" \"firstBlockInsideIfID\": {{\n" + f" // ... definition of the first block inside the if body\n" + f" \"parent\": \"ifBlockID\",\n" + f" \"next\": null // or next block if more\n" + f" }}\n" + f" ```\n" + f"5. **Define ALL Shadow Blocks Separately (THIS IS ESSENTIAL):** Every time a block's input requires a number, string literal, or a selection from a dropdown/menu, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow. Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f"6. **`fields` for direct dropdown values/text:** Use the `fields` dictionary ONLY IF the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., the `KEY_OPTION` field within the `event_whenkeypressed_keymenu` shadow block itself, or the `variable` field on a `data_setvariableto` block). Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Ensure Unique Block IDs:** Every block you generate (main blocks and shadow blocks) must have a unique ID within the entire script's block dictionary.\n" + f"9. **Strictly Use Catalog Opcodes:** You MUST only use `opcode` values that are present in the provided `ALL_SCRATCH_BLOCKS_CATALOG`. Do NOT use unlisted opcodes like `motion_jump`.\n" + f"10. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [ImprovementBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + + generated_blocks = extract_json_from_llm_response(raw_response) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Improvement blocks added for sprite '{sprite_name}', script '{event_opcode}' by ImprovementBlockBuilderNode.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with improvement nodes.") + print("Updated project JSON with improvement nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + +# Build the LangGraph workflow +workflow = StateGraph(GameState) + +# Add all nodes to the workflow +workflow.add_node("parse_query", parse_query_and_set_initial_positions) +workflow.add_node("game_description", game_description_node) +workflow.add_node("initial_plan_build", overall_planner_node) +workflow.add_node("plan_verifier", plan_verification_node) # Verifies the high-level plan +workflow.add_node("refined_planner", refined_planner_node) # Refines the action plan +workflow.add_node("block_builder", overall_block_builder_node) # Builds blocks from a plan +workflow.add_node("block_verifier", block_verification_node) # Verifies the generated blocks +workflow.add_node("improved_block_builder", improvement_block_builder_node) # For specific block-level improvements + +# Set the entry point +workflow.set_entry_point("game_description") + +# Define the standard initial flow +workflow.add_edge("game_description", "parse_query") +workflow.add_edge("parse_query", "initial_plan_build") +workflow.add_edge("initial_plan_build", "plan_verifier") + +# Define the conditional logic after plan_verifier (for high-level plan issues) +def decide_next_step_after_plan_verification(state: GameState): + if state.get("needs_improvement", False): + # If the plan needs refinement, go to the refined_planner + return "refined_planner" + else: + # If the plan is good, proceed to building blocks from this plan + return "block_builder" + +workflow.add_conditional_edges( + "plan_verifier", + decide_next_step_after_plan_verification, + { + "refined_planner": "refined_planner", # Path if plan needs refinement + "block_builder": "block_builder" # Path if plan is approved, proceeds to block building + } +) + +# --- CRITICAL CHANGE FOR THE PLAN REFINEMENT LOOP --- +# After refining the plan, it should go back to plan_verifier for re-verification. +workflow.add_edge("refined_planner", "plan_verifier") # This closes the loop for plan refinement and re-verification. + +# After blocks are built, they need to be verified +workflow.add_edge("block_builder", "block_verifier") + +# Define the conditional logic after block_verifier (for generated blocks issues) +def decide_after_block_verification(state: GameState): + if state.get("needs_improvement", False): + # If blocks need improvement, go to improved_block_builder. + # This assumes improved_block_builder handles specific block-level fixes. + return "improved_block_builder" + else: + # If blocks are good, end the workflow + return "END" + +workflow.add_conditional_edges( + "block_verifier", + decide_after_block_verification, + { + "improved_block_builder": "improved_block_builder", # Path if blocks need improvement + "END": END # Path if blocks are good + } +) + +# Create the loop: If blocks improved, re-verify them +workflow.add_edge("improved_block_builder", "block_verifier") + +# Compile the workflow graph +app_graph = workflow.compile() + +from IPython.display import Image, display +png_bytes = app_graph.get_graph().draw_mermaid_png() +with open("langgraph_workflow.png", "wb") as f: + f.write(png_bytes) + +### Modified `generate_game` Endpoint + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description", "") + backdrops = payload.get("backdrops", []) + sprites = payload.get("sprites", []) + + logger.info(f"Starting game generation for description: '{desc}'") + + # 1) Initial skeleton generation + project_skeleton = { + "targets": [ + { + "isStage": True, + "name":"Stage", + "objName": "Stage", # ADDED THIS FOR THE STAGE + "variables":{}, # This must be a dict: { "var_id": ["var_name", value] } + "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + for idx, s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project_skeleton["targets"].append({ + "isStage": False, + "name": s["name"], + "objName": s["name"], # objName is also important for sprites + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + logger.info("Initial project skeleton created.") + + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + + try: + os.makedirs(project_folder, exist_ok=True) + # Save initial skeleton and copy assets + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project_skeleton, f, indent=2) + logger.info(f"Initial project skeleton saved to {project_json_path}") + + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source backdrop asset not found: {src_path}") + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source sprite asset not found: {src_path}") + logger.info("Assets copied to project folder.") + + # Initialize the state for LangGraph as a dictionary matching the TypedDict structure + initial_state_dict: GameState = { # Type hint for clarity + "project_json": project_skeleton, + "description": desc, + "project_id": project_id, + "sprite_initial_positions": {}, + "action_plan": {}, + #"behavior_plan": {}, + "improvement_plan": {}, + "needs_improvement": False, + "plan_validation_feedback": {}, + "iteration_count": 0, + "review_block_feedback": {} + } + + # Run the LangGraph workflow with the dictionary input. + final_state_dict: GameState = app_graph.invoke(initial_state_dict) + + # Access elements from the final_state_dict + final_project_json = final_state_dict['project_json'] + + # Save the *final* filled project JSON, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(final_project_json, f, indent=2) + logger.info(f"Final project JSON saved to {project_json_path}") + + return jsonify({"message": "Game generated successfully", "project_id": project_id}) + + except Exception as e: + logger.error(f"Error during game generation workflow for project ID {project_id}: {e}", exc_info=True) + return jsonify(error=f"Error generating game: {e}"), 500 + + +if __name__=="__main__": + logger.info("Starting Flask application...") + app.run(debug=True,port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/logs.txt b/scratch_VLM/scratch_agent/logs.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff36f009d1a37cb340ef261e8349eeed44246fe2 --- /dev/null +++ b/scratch_VLM/scratch_agent/logs.txt @@ -0,0 +1,2065 @@ +[latest_logss] + +from flask import Flask, request, jsonify, render_template, send_from_directory +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +#from langgraph.graph import draw +from langchain.chat_models import ChatOpenAI +from PIL import Image +import os, json, re +import shutil +import uuid +from langgraph.graph import StateGraph, END +import logging +from typing import Dict, TypedDict, Optional, Any, List + +# --- Configure logging --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +# --- LLM / Vision model setup --- +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") +os.environ["OPENROUTER_API_KEY"] = os.getenv("OPENROUTER_API_KEY", "default_key_or_placeholder") +os.environ["OPENROUTER_BASE_URL"] = os.getenv("OPENROUTER_BASE_URL", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + logger.critical("GROQ_API_KEY environment variable is not set or invalid. Please set it to proceed.") + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +#Main LLM for the SCRATCH 3.0 Agent +llm = ChatGroq( + #model="deepseek-r1-distill-llama-70b", + #model="llama-3.3-70b-versatile", + #model="meta-llama/llama-4-maverick-17b-128e-instruct", + model="meta-llama/llama-4-scout-17b-16e-instruct", + temperature=0.0, +) + +# Json debugger [temporary] +llm2 = ChatGroq( + model="deepseek-r1-distill-llama-70b", + #model="llama-3.3-70b-versatile", + #model="meta-llama/llama-4-maverick-17b-128e-instruct", + #model="meta-llama/llama-4-scout-17b-16e-instruct", + temperature=0.0, +) + +# llm = ChatOpenAI( +# openai_api_key=os.environ["OPENROUTER_API_KEY"], +# openai_api_base=os.environ["OPENROUTER_BASE_URL"], +# model_name="deepseek/deepseek-r1-0528:free", +# model_kwargs={ +# "headers": { +# "HTTP-Referer": "https://localhost:5000/", +# "X-Title": "agent_scratch", +# } +# }, +#) + + +# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema, informed by comprehensive reference materials. When generating or modifying the `blocks` section, pay extremely close attention to the following: + +**Scratch Target and Block Schema Rules:** +1. **Target Structure:** + * Every target (Stage or Sprite) must have an `objName` property. For the Stage, it's typically `"objName": "Stage"`. For sprites, it's their name (e.g., `"objName": "Cat"`). + * `variables` dictionary: This dictionary maps unique variable IDs to arrays `[variable_name, initial_value]`. + Example: `"variables": { "myVarId123": ["score", 0] }` + +2. **Block Structure:** Every block must have `opcode`, `blockType`, `parent`, `next`, `inputs`, `fields`, `shadow`, `topLevel`. + * `opcode`: Unique internal identifier for the block's specific functionality (e.g., `"motion_movesteps"`)[cite: 439, 452]. + * `blockType`: Specifies the visual shape and general behavior. Common types include `"command"` (Stack block), `"reporter"` (Reporter block), `"Boolean"` (Boolean block), and `"hat"` (Hat block)[cite: 453, 454]. + * `parent`: ID of the block it's stacked on (or `null` for top-level). + * `next`: ID of the block directly below it (or `null` for end of stack). + * `topLevel`: `true` if it's a hat block or a standalone block, `false` otherwise. + * `shadow`: `true` if it's a shadow block (e.g., a default number input), `false` otherwise. + +3. **`inputs` vs. `fields`:** This is critical. + * **`inputs`**: Used for blocks or values *plugged into* a block (e.g., condition for an `if` block, target for `go to`). Values are **arrays**. + * `[1, ]`: Represents a Reporter or Boolean block *plugged in* (e.g., `operator_equals`). + * `[1, [, ""]]`: For a shadow block with a literal value. The `type` specifies the data type and can be "num" (number), "str" (string), "bool" (Boolean), "colour", "angle", "date", "dropdown", "iconmenu", or "variable"[cite: 457, 461]. + Example: `[1, ["num", "10"]]` for a number input. + * `[2, ]`: For a C-block's substack (e.g., `SUBSTACK` in `control_if`). + * **`fields`**: Used for dropdown menus or direct internal values *within* a block (e.g., key selected in `when key pressed`, variable name in `set variable`). Values are always **arrays** `["", null]`. + * Example for `event_whenkeypressed`: `"KEY": ["space", null]` + * Example for `data_setvariableto`: `"VARIABLE": ["score", null]` + +4. **Unique IDs:** All block IDs, variable IDs, and list IDs must be unique strings (e.g., "myBlock123", "myVarId456"). Do NOT use placeholder strings like "block_id_here". + +5. **No Nested `blocks` Dictionary:** The `blocks` dictionary should only appear once per `target` (sprite/stage). Do NOT nest a `blocks` dictionary inside an individual block definition. Blocks that are part of a substack (like inside an `if` or `forever` or `repeat` loop and other) are linked via the `SUBSTACK` input, where the input value is `[2, "ID_of_first_block_in_substack"]`. + +6. **Asset Properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY` should be correct for costumes/sounds. + +**General Principles and Important Considerations:** +* **Backward Compatibility:** Adhere strictly to existing Scratch 3.0 opcodes and schema to ensure backward compatibility with older projects[cite: 439, 440, 446]. +* **Forgiving Inputs:** Recognize that Scratch is designed to be "forgiving in its interpretation of inputs"[cite: 442, 459]. While generating valid JSON, understand that the Scratch VM handles potentially "invalid" inputs gracefully (e.g., type coercion, returning default values like zero or empty strings, rather than crashing)[cite: 443, 459]. This implies that precise type matching for inputs might be handled internally by Scratch, allowing for some flexibility in how values are provided, but the agent should aim for the most common and logical type. + +**When responding, you must:** +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, field, and variable declaration strictly follows the Scratch 3.0 schema and the rules above. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" + +SYSTEM_PROMPT_JSON_CORRECTOR =""" +You are an assistant that outputs JSON responses strictly following the given schema. +If the JSON you produce has any formatting errors, missing required fields, or invalid structure, you must identify the problems and correct them. +Always return only valid JSON that fully conforms to the schema below, enclosed in triple backticks (```), without any extra text or explanation. + +If you receive an invalid or incomplete JSON response, fix it by: +- Adding any missing required fields with appropriate values. +- Correcting syntax errors such as missing commas, brackets, or quotes. +- Ensuring the JSON structure matches the schema exactly. + +Remember: Your output must be valid JSON only, ready to be parsed without errors. +""" + + +# Main agent of the system agent for Scratch 3.0 +agent = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT +) + +# debugger and resolver agent for Scratch 3.0 +agent_json_resolver = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT_JSON_CORRECTOR +) + +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + backdrops = [] + sprites = [] + try: + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + logger.info("Successfully listed static assets.") + except Exception as e: + logger.error(f"Error listing static assets: {e}") + return jsonify({"error": "Failed to list assets"}), 500 + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + logger.warning(f"Could not determine image dimensions for {filename}: {e}. Setting center to 0,0.") + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- New endpoint to fetch project.json --- +@app.route("/get_project/", methods=["GET"]) +def get_project(project_id): + project_folder = os.path.join("generated_projects", project_id) + project_json_path = os.path.join(project_folder, "project.json") + + try: + if os.path.exists(project_json_path): + logger.info(f"Serving project.json for project ID: {project_id}") + return send_from_directory(project_folder, "project.json", as_attachment=True, download_name=f"{project_id}.json") + else: + logger.warning(f"Project JSON not found for ID: {project_id}") + return jsonify({"error": "Project not found"}), 404 + except Exception as e: + logger.error(f"Error serving project.json for ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve project"}), 500 + +# --- New endpoint to fetch assets --- +@app.route("/get_asset//", methods=["GET"]) +def get_asset(project_id, filename): + project_folder = os.path.join("generated_projects", project_id) + asset_path = os.path.join(project_folder, filename) + + try: + if os.path.exists(asset_path): + logger.info(f"Serving asset '{filename}' for project ID: {project_id}") + return send_from_directory(project_folder, filename) + else: + logger.warning(f"Asset '{filename}' not found for project ID: {project_id}") + return jsonify({"error": "Asset not found"}), 404 + except Exception as e: + logger.error(f"Error serving asset '{filename}' for project ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve asset"}), 500 + + +### LangGraph Workflow Definition + +# Define a state for your graph using TypedDict +class GameState(TypedDict): + project_json: dict + description: str + project_id: str + sprite_initial_positions: dict # Add this as well, as it's part of your state + action_plan: Optional[Dict] + #behavior_plan: Optional[Dict] + improvement_plan: Optional[Dict] + needs_improvement: bool + plan_validation_feedback: Optional[Dict] + iteration_count: int # Track the number of iterations for improvements + review_block_feedback: Optional[Dict] # Feedback from the agent on the blocks after verification + +# Helper function to update project JSON with sprite positions +import copy +def update_project_with_sprite_positions(project_json: dict, sprite_positions: dict) -> dict: + """ + Update the 'x' and 'y' coordinates of sprites in the Scratch project JSON. + + Args: + project_json (dict): Original Scratch project JSON. + sprite_positions (dict): Dict mapping sprite names to {'x': int, 'y': int}. + + Returns: + dict: Updated project JSON with new sprite positions. + """ + updated_project = copy.deepcopy(project_json) + + for target in updated_project.get("targets", []): + if not target.get("isStage", False): + sprite_name = target.get("name") + if sprite_name in sprite_positions: + pos = sprite_positions[sprite_name] + if "x" in pos and "y" in pos: + target["x"] = pos["x"] + target["y"] = pos["y"] + + return updated_project + +# Helper function to load the block catalog from a JSON file +def _load_block_catalog(file_path: str) -> Dict: + """Loads the Scratch block catalog from a specified JSON file.""" + try: + with open(file_path, 'r') as f: + catalog = json.load(f) + logger.info(f"Successfully loaded block catalog from {file_path}") + return catalog + except FileNotFoundError: + logger.error(f"Error: Block catalog file not found at {file_path}") + # Return an empty dict or raise an error, depending on desired behavior + return {} + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON from {file_path}: {e}") + return {} + except Exception as e: + logger.error(f"An unexpected error occurred while loading {file_path}: {e}") + return {} + +# --- Global variable for the block catalog --- +ALL_SCRATCH_BLOCKS_CATALOG = {} +BLOCK_CATALOG_PATH = r"blocks\blocks.json" # Define the path to your JSON file +HAT_BLOCKS_PATH = r"blocks\hat_blocks.json" # Path to the hat blocks JSON file +STACK_BLOCKS_PATH = r"blocks\stack_blocks.json" # Path to the stack blocks JSON file +REPORTER_BLOCKS_PATH = r"blocks\reporter_blocks.json" # Path to the reporter blocks JSON file +BOOLEAN_BLOCKS_PATH = r"blocks\boolean_blocks.json" # Path to the boolean blocks JSON file +C_BLOCKS_PATH = r"blocks\c_blocks.json" # Path to the C blocks JSON file +CAP_BLOCKS_PATH = r"blocks\cap_blocks.json" # Path to the cap blocks JSON file + +# Load the block catalogs from their respective JSON files +hat_block_data = _load_block_catalog(HAT_BLOCKS_PATH) +hat_description = hat_block_data["description"] +hat_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in hat_block_data["blocks"]]) +print("Hat blocks loaded successfully.", hat_description) +boolean_block_data = _load_block_catalog(BOOLEAN_BLOCKS_PATH) +boolean_description = boolean_block_data["description"] +boolean_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in boolean_block_data["blocks"]]) + +c_block_data = _load_block_catalog(C_BLOCKS_PATH) +c_description = c_block_data["description"] +c_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in c_block_data["blocks"]]) + +cap_block_data = _load_block_catalog(CAP_BLOCKS_PATH) +cap_description = cap_block_data["description"] +cap_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in cap_block_data["blocks"]]) + +reporter_block_data = _load_block_catalog(REPORTER_BLOCKS_PATH) +reporter_description = reporter_block_data["description"] +reporter_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in reporter_block_data["blocks"]]) + +stack_block_data = _load_block_catalog(STACK_BLOCKS_PATH) +stack_description = stack_block_data["description"] +stack_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in stack_block_data["blocks"]]) + +# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally +ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH) + +# Helper function to generate a unique block ID +def generate_block_id(): + """Generates a short, unique ID for a Scratch block.""" + return str(uuid.uuid4())[:10].replace('-', '') # Shorten for readability, ensure uniqueness + +# Placeholder for your extract_json_from_llm_response function +# # Helper function to extract JSON from LLM response +# def extract_json_from_llm_response(raw_response: str) -> dict: +# """ +# Extracts a JSON object from an LLM response string, robustly handling +# literal newlines inside string values by escaping them. +# """ +# # 1) Pull out the inner JSON block (```json … ```) +# json_block = raw_response +# m = re.search(r"```json\s*(\{.*\})\s*```", raw_response, re.DOTALL) +# if m: +# json_block = m.group(1) + +# # 2) Find the feedback field and escape its newlines +# # This pattern captures everything between "feedback": " … " +# feedback_pattern = r'("feedback"\s*:\s*")(.+?)("\s*,)' +# def _escape_feedback(m): +# prefix, val, suffix = m.groups() +# # replace each literal newline and carriage return with \n +# safe_val = val.replace("\r", "\\r").replace("\n", "\\n") +# return prefix + safe_val + suffix + +# json_sanitized = re.sub(feedback_pattern, _escape_feedback, json_block, flags=re.DOTALL) + +# # 3) Now try loading +# try: +# return json.loads(json_sanitized) +# except json.JSONDecodeError as e: +# logger.error(f"Failed to parse sanitized JSON: {e!r}\nContent was:\n{json_sanitized}") +# raise + +def extract_json_from_llm_response(raw_response: str) -> dict: + """ + Extracts a JSON object from an LLM response string, robustly handling + various common LLM output formats and parsing errors. + """ + json_string_to_parse = raw_response + + # --- Step 1: Try to extract JSON from markdown code block (most common and cleanest) --- + # This pattern is more robust: it matches "```json" or "```" (for general code blocks) + # and captures everything until the next "```" + markdown_match = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", raw_response) + if markdown_match: + json_string_to_parse = markdown_match.group(1).strip() + logger.debug("Extracted potential JSON from markdown block.") + else: + logger.debug("No markdown JSON block found. Attempting to parse raw response.") + + # --- Step 2: Pre-process for common LLM JSON generation errors --- + # 2.1) Escape newlines/carriage returns in string values + # This is a general fix for all string values, not just 'feedback' + # It looks for "key": "value" patterns and replaces newlines within 'value' + # This might be tricky to get right for all cases without a full parser. + # A safer bet is to hope the LLM formats strings correctly, or to rely on direct JSON parsing. + # For now, let's keep your feedback-specific one if you find LLM adds newlines there, + # but the primary error is structural. + + # Re-apply feedback escaping if you still suspect newlines in specific fields + feedback_pattern = r'("feedback"\s*:\s*")(.+?)("(?:\s*,|\s*\})?)' # Adjusted regex to handle end of object/array + def _escape_feedback(m): + prefix, val, suffix = m.groups() + safe_val = val.replace("\r", "\\r").replace("\n", "\\n") + return prefix + safe_val + suffix + json_string_to_parse = re.sub(feedback_pattern, _escape_feedback, json_string_to_parse, flags=re.DOTALL) + logger.debug("Applied feedback newline escaping.") + + + # --- Step 3: Attempt to parse the (sanitized) JSON string --- + try: + parsed_json = json.loads(json_string_to_parse) + logger.info("Successfully parsed JSON from LLM response.") + return parsed_json + except json.JSONDecodeError as original_error: + # If parsing fails, log the problematic string and the error for debugging + logger.error(f"Failed to parse JSON. Error: {original_error!r}") + logger.error(f"Problematic JSON string (start):\n{json_string_to_parse[:1000]}...") # Log first 1000 chars + logger.error(f"Problematic JSON string (full length: {len(json_string_to_parse)}):\n{json_string_to_parse}") # Log full for deep dive + + # Attempt a fallback for common structural issues (e.g., extra trailing commas) + # This is speculative and may not always work, but worth a try before failing. + try: + # Remove trailing commas from objects and arrays + # This regex is simplified and might not catch all cases, but handles common ones + # For example, {"a": 1,} -> {"a": 1} + # Or [1, 2,] -> [1, 2] + sanitized_for_trailing_commas = re.sub(r',\s*([}\]])', r'\1', json_string_to_parse) + logger.warning("Attempting to parse after removing potential trailing commas.") + return json.loads(sanitized_for_trailing_commas) + except json.JSONDecodeError as e_fallback: + logger.error(f"Fallback parsing also failed. Error: {e_fallback!r}") + # The original error is more informative for 'Expecting ,' delimiter, so raise that. + raise original_error # Re-raise the original, more specific JSONDecodeError + +# Node 1: Detailed description generator. +def game_description_node(state: GameState): + """ + Generates a detailed narrative description of the game based on the initial query. + """ + logger.info("--- Running GameDescriptionNode ---") + sprite_name = {} + initial_description = state.get("description", "A simple game.") + project_json = state["project_json"] + for target in project_json["targets"]: + sprite_name[target["name"]] = target["name"] + + description_prompt = ( + f"You are an AI assistant tasked with generating a detailed narrative description for a Scratch 3.0 game.\n" + f"The initial high-level description is: '{initial_description}'.\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Make sure you donot change Sprite and Stage name. Here are all the name: {sprite_name} \n" + f"Create a rich, engaging, and detailed description, including potential gameplay elements, objectives, and overall feel with the available resources\n" + f"The output should be a plain text description." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": description_prompt}]}) + detailed_game_description = response["messages"][-1].content + state["description"] = detailed_game_description + logger.info("Detailed game description generated by GameDescriptionNode.") + print(f"Detailed Game Description: {detailed_game_description}") + return state + except Exception as e: + logger.error(f"Error in GameDescriptionNode: {e}") + raise + +# Node 2: Analysis User Query and Initial Positions +def parse_query_and_set_initial_positions(state: GameState): + logger.info("--- Running ParseQueryNode ---") + + llm_query_prompt = ( + f"Based on the user's game description: '{state['description']}', " + f"and the current Scratch project JSON below, " + f"determine the most appropriate initial 'x' and 'y' coordinates for each sprite. " + f"Return ONLY a JSON object with a single key 'sprite_initial_positions' mapping sprite names to their {{'x': int, 'y': int}} coordinates.\n\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Example Json output:\n" + "```json\n" + "{\n" + " \"sprite_initial_positions\": {\n" + " \"Sprite1\": {\"x\": -160, \"y\": -110},\n" + " \"Sprite2\": {\"x\": 240, \"y\": -135}\n" + " }\n" + "}" + "```\n" + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_query_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM:", raw_response) + # json debugging and solving + try: + updated_data = extract_json_from_llm_response(raw_response) + sprite_positions = updated_data.get("sprite_initial_positions", {}) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT PARSER]: {correction_response}") + corrected_data = extract_json_from_llm_response(correction_response["messages"][-1].content) + sprite_positions = corrected_data.get("sprite_initial_positions", {}) + + new_project_json = update_project_with_sprite_positions(state["project_json"], sprite_positions) + state["project_json"]= new_project_json + print("Updated project JSON with sprite positions:", json.dumps(new_project_json, indent=2)) + return state + #return {"project_json": new_project_json} + except Exception as e: + logger.error(f"Error in ParseQueryNode: {e}") + raise + + +# Node 3: Sprite Action Plan Builder +def overall_planner_node(state: GameState): + """ + Generates a comprehensive action plan for sprites, including detailed Scratch block information. + This node acts as an overall planner, leveraging knowledge of all block shapes and categories. + """ + logger.info("--- Running OverallPlannerNode ---") + + description = state.get("description", "") + project_json = state["project_json"] + sprite_names = [t["name"] for t in project_json["targets"] if not t["isStage"]] + + # Get sprite positions from the project_json if available, or set defaults + sprite_positions = {} + for target in project_json["targets"]: + if not target["isStage"]: + sprite_positions[target["name"]] = {"x": target.get("x", 0), "y": target.get("y", 0)} + + planning_prompt = ( + "Generate a detailed action plan for the game's sprites based on the user query and sprite details.\n\n" + f"**Game Description:** '{description}'\n\n" + f"**Sprites in Game:** {', '.join(sprite_names)}\n" + f"**Current Sprite Positions:** {json.dumps(sprite_positions)}\n\n" + "--- Scratch 3.0 Block Reference ---\n" + "This section provides a comprehensive reference of Scratch 3.0 blocks, categorized by shape, including their opcodes and functional descriptions. Use this to accurately identify block types and behavior.\n\n" + f"### Hat Blocks\nDescription: {hat_description}\nBlocks:\n{hat_opcodes_functionalities}\n\n" + f"### Boolean Blocks\nDescription: {boolean_description}\nBlocks:\n{boolean_opcodes_functionalities}\n\n" + f"### C Blocks\nDescription: {c_description}\nBlocks:\n{c_opcodes_functionalities}\n\n" + f"### Cap Blocks\nDescription: {cap_description}\nBlocks:\n{cap_opcodes_functionalities}\n\n" + f"### Reporter Blocks\nDescription: {reporter_description}\nBlocks:\n{reporter_opcodes_functionalities}\n\n" + f"### Stack Blocks\nDescription: {stack_description}\nBlocks:\n{stack_opcodes_functionalities}\n\n" + "-----------------------------------\n\n" + "Your task is to define the primary actions and movements for each sprite.\n" + "The output should be a JSON object with a single key 'action_overall_flow'. Each key inside this object should be a sprite name (e.g., 'Player', 'Enemy'), and its value must include a 'description' and a list of 'plans'.\n" + "Each plan must begin with a **single Scratch Hat Block** (e.g., 'event_whenflagclicked') and should contain:\n" + "1. **'event'**: the exact `opcode` of the hat block that initiates the logic.\n" + "2. **'logic'**: a natural language breakdown of each step taken after the event. Separate each step with a semicolon ';'. Ensure clarity and granularity—each described action should map closely to a Scratch block or tight sequence.\n" + " - For example: for a jump: 'change y by N; wait M seconds; change y by -N'.\n" + " - Use 'forever: ...' to prefix repeating logic explicitly.\n" + " - Use Scratch-consistent verbs: 'move', 'change', 'wait', 'hide', 'say', etc.\n" + "3. **Opcode Lists**: include relevant Scratch opcodes grouped under `motion`, `control`, `operator`, `sensing`, `looks`, `sounds`, `events`, and `data`. List only the non-empty categories. Use exact opcodes including shadow/helper blocks (e.g., 'math_number').\n\n" + "Use sprite names exactly as listed in `sprite_names`. Do NOT rename or invent new sprites.\n" + "Ensure the plan reflects accurate opcode usage derived strictly from the block reference above.\n\n" + "Example structure for 'action_overall_flow':\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at starting point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"repeat(10): change y by 10; wait 0.1 seconds; change y by -10;\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_repeat\", \"control_wait\"],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement and interaction\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135; forever: glide 2 seconds to x:-240 y:-135; if x position < -235, then set x to 240; if touching Sprite1, then hide;\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\", \"motion_setx\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operator\": [\"operator_lt\"],\n" + " \"sensing\": [\"sensing_istouching\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [\"looks_hide\"],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Based on the provided context, generate the `action_overall_flow`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM [OverallPlannerNode]:", raw_response) # Uncomment for debugging + # json debugging and solving + try: + overall_plan = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT OVERALLPLANNERNODE ]: {correction_response}") + overall_plan= extract_json_from_llm_response(correction_response["messages"][-1].content) + + state["action_plan"] = overall_plan + logger.info("Overall plan generated by OverallPlannerNode.") + return state + + except Exception as e: + logger.error(f"Error in OverallPlannerNode: {e}") + raise + +# Helper function to get a block by its opcode from a single catalog +def get_block_by_opcode(catalog_data: dict, opcode: str) -> dict | None: + """ + Search a single catalog (with keys "description" and "blocks": List[dict]) + for a block whose 'op_code' matches the given opcode. + Returns the block dict or None if not found. + """ + for block in catalog_data["blocks"]: + if block.get("op_code") == opcode: + return block + return None + +# Helper function to find a block in all catalogs by opcode +def find_block_in_all(opcode: str, all_catalogs: list[dict]) -> dict | None: + """ + Search across multiple catalogs for a given opcode. + Returns the first matching block dict or None. + """ + for catalog in all_catalogs: + blk = get_block_by_opcode(catalog, opcode) + if blk is not None: + return blk + return None + +# Node 4: Sprite Plan Verification Node +def plan_verification_node(state: GameState): + """ + Validates the generated action plan/blocks, identifies missing logic, + provides feedback, and determines if further improvements are needed. + Also manages the iteration count for the improvement loop. + """ + logger.info(f"--- Running VerificationNode (Iteration: {state.get('iteration_count', 0)}) ---") + + MAX_IMPROVEMENT_ITERATIONS = 1 # Set a sensible limit to prevent infinite loops + current_iteration = state.get("iteration_count", 0) + #project_json = state["project_json"] + action_plan = state.get("action_plan", {}) + print(f"[action_plan before verification] on ({current_iteration}): {json.dumps(action_plan, indent=2)}") + #improvement_plan = state.get("improvement_plan", {}) # May contain prior improvement guidance + + # Corrected validation_prompt + validation_prompt = ( + f"You are an AI validator for Scratch project plans and generated blocks. " + f"Your task is to review the current state of the game's action plan and block structure. " + f"Critically analyze if there are any missing logic, structural inconsistencies, or unclear intentions. " + f"Provide **precise** and **constructive** feedback for improvement.\n\n" + f"**Game Description:**\n" + f"{state.get('description', '')}\n\n" + f"**Current Action Plan (High-Level Logic):**\n" + f"```json\n{json.dumps(action_plan, indent=2)}\n```\n\n" + # Uncomment below if needed + # f"**Current Project JSON (Generated Blocks):**\n" + # f"```json\n{json.dumps(project_json, indent=2)}\n```\n\n" + f"**Previous Feedback (if any):**\n" + f"{state.get('plan_validation_feedback', 'None')}\n\n" + f"Based on the above, return a response strictly in the following JSON format:\n" + "```json\n" + "{\n" + " \"feedback\": \"Detailed comments on any missing logic, inconsistencies, or unclear intent. Be concise but specific. If everything is perfect, state that explicitly.\",\n" + " \"needs_improvement\": true,\n" + " \"suggested_description_updates\": \"Concise revision of the game description if needed. Use an empty string if no change is required.\"\n" + "}\n" + "```\n" + "**Important:**\n" + "- The `needs_improvement` field must be strictly `true` or `false` (boolean). Do **not** include any other text or explanation inside the JSON.\n" + "- Be strict in evaluation. If **any** part of the plan or block logic appears incomplete, ambiguous, or incorrect, set `needs_improvement` to `true`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": validation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [VerificationNode]: {raw_response[:500]}...") + # json debugging and solving + try: + validation_result = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT PLANVERIFICATIONNODE ]: {correction_response}") + validation_result = extract_json_from_llm_response(correction_response["messages"][-1].content) + + # Update state with feedback and improvement flag + state["plan_validation_feedback"] = validation_result.get("feedback", "No specific feedback provided.") + state["needs_improvement"] = validation_result.get("needs_improvement", False) + + suggested_description_updates = validation_result.get("suggested_description_updates", "") + if suggested_description_updates: + # You might want to append or intelligently merge this with the existing detailed_game_description + # For simplicity, let's just append for now or update a specific field + #current_description = state.get("detailed_game_description", state.get("description", "")) + current_description = state.get(state.get("description", "")) + #state["detailed_game_description"] = f"{current_description}\n\nSuggested Update: {suggested_description_updates}"\ + state["description"] = f"{current_description}\n\nSuggested Update: {suggested_description_updates}" + logger.info("Updated detailed game description based on validation feedback.") + + + # Manage iteration count + if state["needs_improvement"]: + state["iteration_count"] = current_iteration + 1 + if state["iteration_count"] >= MAX_IMPROVEMENT_ITERATIONS: + logger.warning(f"Max improvement iterations ({MAX_IMPROVEMENT_ITERATIONS}) reached. Forcing 'needs_improvement' to False.") + state["needs_improvement"] = False + state["plan_validation_feedback"] += "\n(Note: Max iterations reached, stopping further improvements.)" + else: + state["iteration_count"] = 0 # Reset if no more improvement needed + + logger.info(f"Verification completed. Needs Improvement: {state['needs_improvement']}. Feedback: {state['plan_validation_feedback'][:100]}...") + print(f"[updated action_plan after verification] on ({current_iteration}): {json.dumps(state.get("action_plan", {}), indent=2)}") + return state + except Exception as e: + logger.error(f"Error in VerificationNode: {e}") + state["needs_improvement"] = False # Force end loop on error + state["plan_validation_feedback"] = f"Validation error: {e}" + raise + +# Node 5: Refined Planner Node +def refined_planner_node(state: GameState): + """ + Refines the action plan based on validation feedback and game description. + """ + logger.info("--- Running RefinedPlannerNode ---") + + #detailed_game_description = state.get("detailed_game_description", state.get("description", "A game.")) + detailed_game_description = state.get("description","") + current_action_plan = state.get("action_plan", {}) + print(f"[current_action_plan before refinement] on ({state.get('iteration_count', 0)}): {json.dumps(current_action_plan, indent=2)}") + plan_validation_feedback = state.get("plan_validation_feedback", "No specific feedback provided. Assume general refinement is needed.") + project_json = state["project_json"] + sprite_names = [t["name"] for t in project_json["targets"] if not t["isStage"]] + + # Get sprite positions from the project_json if available, or set defaults + sprite_positions = {} + for target in project_json["targets"]: + if not target["isStage"]: + sprite_positions[target["name"]] = {"x": target.get("x", 0), "y": target.get("y", 0)} + + refinement_prompt = ( + "Refine the existing action plan for the game's sprites based on the detailed game description and the validation feedback provided.\n\n" + f"**Detailed Game Description:** '{detailed_game_description}'\n\n" + f"**Sprites in Game:** {', '.join(sprite_names)}\n" + f"**Current Sprite Positions:** {json.dumps(sprite_positions)}\n\n" + f"**Current Action Plan (to be refined):**\n" + f"```json\n{json.dumps(current_action_plan, indent=2)}\n```\n\n" + f"**Validation Feedback:**\n" + f"'{plan_validation_feedback}'\n\n" + "--- Scratch 3.0 Block Reference ---\n" + f"### Hat Blocks\nDescription: {hat_description}\nBlocks:\n{hat_opcodes_functionalities}\n\n" + f"### Boolean Blocks\nDescription: {boolean_description}\nBlocks:\n{boolean_opcodes_functionalities}\n\n" + f"### C Blocks\nDescription: {c_description}\nBlocks:\n{c_opcodes_functionalities}\n\n" + f"### Cap Blocks\nDescription: {cap_description}\nBlocks:\n{cap_opcodes_functionalities}\n\n" + f"### Reporter Blocks\nDescription: {reporter_description}\nBlocks:\n{reporter_opcodes_functionalities}\n\n" + f"### Stack Blocks\nDescription: {stack_description}\nBlocks:\n{stack_opcodes_functionalities}\n\n" + "-----------------------------------\n\n" + "Your task is to refine the JSON object 'action_overall_flow'.\n" + "Use sprite names exactly as provided in `sprite_names` (e.g., 'Sprite1', 'soccer ball'); do **NOT** rename them.\n\n" + "Follow this exact format for the output (example):\n" + "Example structure for 'action_overall_flow':\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at starting point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"repeat(10): change y by 10 → wait 0.1 sec → change y by -10\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_repeat\", \"control_wait\"],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement and interaction\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135 → forever glide to x:-240 y:-135 → if x position < -235 then set x to 240 → if touching Sprite1 then hide\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\", \"motion_setx\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operator\": [\"operator_lt\"],\n" + " \"sensing\": [\"sensing_istouching\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [\"looks_hide\"],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Use the validation feedback to address errors, fill in missing logic, or enhance clarity.\n" + "example of few possible improvements: 1.event_whenflagclicked is used to control sprite but its used for actual start scratch project and reset scratch. 2. looping like forever used where we should use iterative. 3. missing of for variable we used in the block\n" + "- Maintain the **exact JSON structure** shown above.\n" + "- All `logic` fields must be **clear and granular**.\n" + "- Only include opcode categories that contain relevant opcodes.\n" + "- Ensure that each opcode matches its intended Scratch functionality.\n" + "- If feedback suggests major change, **rethink the entire plan** for the affected sprite(s).\n" + "- If feedback is minor, make precise, minimal improvements only." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": refinement_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [RefinedPlannerNode]: {raw_response[:500]}...") + # json debugging and solving + try: + refined_plan = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT REFINEPLANNER ]: {correction_response}") + refined_plan = extract_json_from_llm_response(correction_response["messages"][-1].content) + logger.info("Refined plan corrected by JSON resolver agent.") + + if refined_plan: + #state["action_plan"] = refined_plan.get("action_overall_flow", {}) # Update to the key 'action_overall_flow' [error] + state["action_plan"] = refined_plan.get("action_overall_flow", {}) # Update the main the prompt includes updated only + logger.info("Action plan refined by RefinedPlannerNode.") + else: + logger.warning("RefinedPlannerNode did not return a valid 'action_overall_flow' structure. Keeping previous plan.") + print("[Refined Action Plan]:", json.dumps(state["action_plan"], indent=2)) + print("[current state after refinement]:", json.dumps(state, indent=2)) + return state + except Exception as e: + logger.error(f"Error in RefinedPlannerNode: {e}") + raise + +# Node 4: Overall Block Builder Node +def overall_block_builder_node(state: GameState): + logger.info("--- Running OverallBlockBuilderNode ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + # Pre-load all block-catalog JSONs once + all_catalogs = [ + hat_block_data, + boolean_block_data, + c_block_data, + cap_block_data, + reporter_block_data, + stack_block_data + ] + action_plan = state.get("action_plan", {}) + print("[Overall Action Plan received at the block generator]:", json.dumps(action_plan, indent=2)) + if not action_plan: + logger.warning("No action plan found in state. Skipping OverallBlockBuilderNode.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + # This is the handler which ensure if somehow json response changed it handle it.[DONOT REMOVE BELOW LOGIC] + if action_plan.get("action_overall_flow", {})=={}: + plan_data = action_plan.items() + else: + plan_data= action_plan.get("action_overall_flow", {}).items() + + for sprite_name, sprite_actions_data in plan_data: + #for sprite_name, sprite_actions_data in action_plan.items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_actions_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operator_opcodes = plan_entry.get("operator", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + # Create a string representation of the identified opcodes for the prompt + identified_opcodes_str = "" + if motion_opcodes: + identified_opcodes_str += f" Motion Blocks (opcodes): {', '.join(motion_opcodes)}\n" + if control_opcodes: + identified_opcodes_str += f" Control Blocks (opcodes): {', '.join(control_opcodes)}\n" + if operator_opcodes: + identified_opcodes_str += f" Operator Blocks (opcodes): {', '.join(operator_opcodes)}\n" + if sensing_opcodes: + identified_opcodes_str += f" Sensing Blocks (opcodes): {', '.join(sensing_opcodes)}\n" + if looks_opcodes: + identified_opcodes_str += f" Looks Blocks (opcodes): {', '.join(looks_opcodes)}\n" + if sound_opcodes: + identified_opcodes_str += f" Sound Blocks (opcodes): {', '.join(sound_opcodes)}\n" + if events_opcodes: + identified_opcodes_str += f" Event Blocks (opcodes): {', '.join(events_opcodes)}\n" + if data_opcodes: + identified_opcodes_str += f" Data Blocks (opcodes): {', '.join(data_opcodes)}\n" + + needed_opcodes = motion_opcodes + control_opcodes + operator_opcodes + sensing_opcodes + looks_opcodes + sound_opcodes + events_opcodes + data_opcodes + needed_opcodes = list(set(needed_opcodes)) + + + # 2) build filtered runtime catalog (if you still need it) + filtered_catalog = { + op: ALL_SCRATCH_BLOCKS_CATALOG[op] + for op in needed_opcodes + if op in ALL_SCRATCH_BLOCKS_CATALOG + } + + # 3) merge human‑written catalog + runtime entry for each opcode + combined_blocks = {} + for op in needed_opcodes: + catalog_def = find_block_in_all(op, all_catalogs) or {} + runtime_def = ALL_SCRATCH_BLOCKS_CATALOG.get(op, {}) + # merge: catalog fields first, then runtime overrides/adds + merged = {**catalog_def, **runtime_def} + combined_blocks[op] = merged + print("[Combined blocks for this script]:", json.dumps(combined_blocks, indent=2)) + + llm_block_generation_prompt = ( + f"You are an AI assistant generating a **single complete Scratch 3.0 script** in JSON format.\n" + f"The current sprite is '{sprite_name}'.\n" + f"This script must start with the event block (Hat Block) for opcode '{event_opcode}'.\n" + f"The sequential logic to implement for this script is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Required Block Opcodes & Catalog:**\n" + f"Based on the planning, the following specific Scratch block opcodes are expected to be used. You MUST use these opcodes where applicable.\n" + f"Here is the comprehensive catalog for these blocks, including their structure and required inputs/fields:\n" + f"```json\n{json.dumps(combined_blocks, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (provided for context; you are generating a NEW, complete script):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**CRITICAL INSTRUCTIONS FOR GENERATING THE BLOCK JSON (READ CAREFULLY AND FOLLOW PRECISELY):**\n" + f"1. **Unique Block IDs:** Generate a **globally unique ID** for EVERY block (main and shadow blocks) within the entire JSON output for this script. Example: 'myBlockID123'.\n" + f"2. **Script Initiation (Hat Block - VERY IMPORTANT):**\n" + f" * The **first block** of the script (the Hat block, opcode '{event_opcode}') MUST have `\"topLevel\": true` and `\"parent\": null`.\n" + f" * ONLY this Hat block should have `\"topLevel\": true`. All other blocks in the script MUST have `\"topLevel\": false`.\n" + f" * Set its `x` and `y` coordinates (e.g., `x: 0, y: 0` or similar for clear placement).\n" + f"3. **Strict Block Chaining (`next` and `parent`):**\n" + f" * Use the `next` field to point to the ID of the block DIRECTLY BELOW it in the stack. If a block has a `next`, its `next` block's `parent` MUST point back to the current block's ID.\n" + f" * Use the `parent` field to point to the ID of the block DIRECTLY ABOVE it in the stack. If a block has a `parent`, then that `parent` block's `next` MUST point to the current block's ID.\n" + f" * The **last block** in a linear stack (e.g., a Stack block not containing other blocks, or a Cap block) MUST have `\"next\": null`.\n" + f" * Blocks plugged into inputs (like Boolean reporters or Reporter blocks) or substacks (like inside C-blocks) do NOT use `next` to connect to the block holding them. Their connection is solely via the `inputs` field of their parent.\n" + f"4. **`inputs` Field Structure (ABSOLUTELY CRITICAL - ADHERE TO THIS RIGIDLY):**\n" + f" * The value for ANY key within the `inputs` dictionary MUST be an **array of EXACTLY two elements**: `[type_code, value_or_block_id]`.\n" + f" * **NEVER embed direct primitive values or arrays within `inputs` without a `type_code`**: E.g., `\"INPUT_NAME\": [\"value\", null]` or `\"INPUT_NAME\": [\"nestedArray\"]` are **STRICTLY FORBIDDEN and will cause errors**.\n" + f" * **`type_code` (first element):**\n" + f" * `1`: Use when `value_or_block_id` refers to a **primitive value** (number, string, boolean) or the ID of a **shadow block** (e.g., `math_number`, `text`, menu blocks).\n" + f" * `2`: Use when `value_or_block_id` refers to the ID of a **non-shadow block** that is PLUGGED IN (e.g., a Boolean block in a condition, or the first block of a C-block's `SUBSTACK`).\n" + f" * **Correct Examples (Re-emphasized):**\n" + f" * **Numerical Input (`motion_movesteps`):** To input `10` steps, define a separate `math_number` shadow block and reference its ID:\n" + f" ```json\n" + f" // Main block\n" + f" \"moveStepsID\": {{\n" + f" \"opcode\": \"motion_movesteps\", \"inputs\": {{ \"STEPS\": [1, \"shadowNumID\"] }},\n" + f" \"parent\": \"parentBlockID\", \"next\": \"nextBlockID\", \"topLevel\": false, \"shadow\": false\n" + f" }},\n" + f" // Separate shadow block for '10'\n" + f" \"shadowNumID\": {{\n" + f" \"opcode\": \"math_number\", \"fields\": {{ \"NUM\": [\"10\", null] }},\n" + f" \"parent\": \"moveStepsID\", \"shadow\": true, \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Dropdown/Menu Input (`sensing_touchingobject` with 'edge'):** Define a separate menu shadow block and reference its ID:\n" + f" ```json\n" + f" // Main block\n" + f" \"touchingBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", \"inputs\": {{ \"TOUCHINGOBJECTMENU\": [1, \"shadowEdgeMenuID\"] }},\n" + f" \"parent\": \"parentBlockID\", \"next\": null, \"topLevel\": false, \"shadow\": false\n" + f" }},\n" + f" // Separate shadow block for '_edge_'\n" + f" \"shadowEdgeMenuID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\", \"fields\": {{ \"TOUCHINGOBJECTMENU\": [\"_edge_\", null] }},\n" + f" \"parent\": \"touchingBlockID\", \"shadow\": true, \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **C-block (`control_forever`):** Its `SUBSTACK` input MUST point to the first block inside its loop using `type_code: 2`.\n" + f" ```json\n" + f" \"foreverBlockID\": {{\n" + f" \"opcode\": \"control_forever\", \"inputs\": {{ \"SUBSTACK\": [2, \"firstBlockInsideForeverID\"] }},\n" + f" \"next\": null, \"parent\": \"blockAboveForeverID\", \"shadow\": false, \"topLevel\": false\n" + f" }},\n" + f" \"firstBlockInsideForeverID\": {{\n" + f" \"opcode\": \"motion_movesteps\", \"parent\": \"foreverBlockID\", \"next\": null, \"topLevel\": false, \"shadow\": false\n" + f" }}\n" + f" ```\n" + f" * **C-block with Condition (`control_if`):** `CONDITION` input uses `type_code: 1` (referencing a Boolean reporter), and `SUBSTACK` uses `type_code: 2` (referencing the first block inside the if-body).\n" + f" ```json\n" + f" \"ifBlockID\": {{\n" + f" \"opcode\": \"control_if\",\n" + f" \"inputs\": {{\n" + f" \"CONDITION\": [1, \"conditionBlockID\"],\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideIfID\"]\n" + f" }},\n" + f" \"next\": \"blockAfterIfID\", \"parent\": \"blockAboveIfID\", \"shadow\": false, \"topLevel\": false\n" + f" }},\n" + f" \"conditionBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", \"parent\": \"ifBlockID\", \"shadow\": false, \"topLevel\": false // ... and its own inputs/fields\n" + f" }},\n" + f" \"firstBlockInsideIfID\": {{\n" + f" \"opcode\": \"looks_sayforsecs\", \"parent\": \"ifBlockID\", \"next\": null, \"topLevel\": false, \"shadow\": false\n" + f" }}\n" + f" ```\n" + f"5. **Separate Shadow Blocks (MANDATORY):** Every time a block's input requires a numeric value, string literal, or a selection from a dropdown/menu, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow. Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f"6. **`fields` for Direct Values:** Use the `fields` dictionary ONLY if the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., `NUM` field in `math_number`, `KEY_OPTION` in `event_whenkeypressed`, `VARIABLE` field in `data_setvariableto`). Example: `\"fields\": {{\"NUM\": [\"10\", null]}}`.\n" + f"7. **Strictly Use Catalog Opcodes:** You MUST only use `opcode` values that are present in the provided `combined_blocks` catalog. Do NOT use unlisted or hypothetical opcodes (e.g., `motion_jump`).\n" + f"8. **Output Format:** Return **ONLY the JSON object** representing all the blocks for THIS SINGLE SCRIPT. Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [OverallBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + print(f"Raw response from LLM [OverallBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response}") # Uncomment for debugging + try: + generated_blocks = extract_json_from_llm_response(raw_response) + + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT OVERALLBLOCKBUILDER ]: {correction_response}") + generated_blocks = extract_json_from_llm_response(correction_response["messages"][-1].content) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + + # setting the iteration count for the script + state["iteration_count"] = 0 + + logger.info(f"Action blocks added for sprite '{sprite_name}', script '{event_opcode}' by OverallBlockBuilderNode.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with action nodes.") + print("Updated project JSON with action nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + + +#helper function to identify the shape of block utilized by the block verifier +def get_block_type(opcode: str) -> str: + """Determines the general type of a Scratch block based on its opcode.""" + if not opcode: + return "unknown" + if opcode.startswith("event_when") or opcode == "control_start_as_clone": + return "hat" + elif opcode.startswith("control_") and ("if" in opcode or "repeat" in opcode or "forever" in opcode): + return "c_block" + elif opcode in ["operator_equals", "operator_gt", "operator_lt", "operator_and", "operator_or", "operator_not"] or \ + (opcode.startswith("sensing_") and ("mousedown" in opcode or "keypressed" in opcode or "touching" in opcode)): + return "boolean" + elif opcode.endswith("menu"): # For dropdown shadow blocks, treat as reporter for type checking + return "reporter" + elif any(s in opcode for s in ["position", "direction", "size", "volume", "costume", "backdrop", "random", "add", "subtract", "multiply", "divide", "length", "item", "of"]): + # A more comprehensive check for reporters + return "reporter" + elif "stop_all" in opcode or "delete_this_clone" in opcode or "procedures_definition" in opcode: + return "cap" + # Default to stack for most command blocks if not explicitly defined + return ALL_SCRATCH_BLOCKS_CATALOG.get(opcode, {}).get("blockType", "stack") + +#helper function to identify the shape of block utilized by the block verifier +def filter_script_blocks(all_blocks: dict, hat_block_id: str) -> dict: + """ + Filters and returns only the blocks that are part of a specific script + starting from the given hat_block_id, including connected reporters/shadows. + """ + script_blocks = {} + q = [hat_block_id] + visited = set() + + while q: + current_block_id = q.pop(0) + if current_block_id in visited: + continue + visited.add(current_block_id) + + block_data = all_blocks.get(current_block_id) + if not block_data: + continue + + script_blocks[current_block_id] = block_data + + # Add next block in sequence + next_id = block_data.get("next") + if next_id and next_id in all_blocks: + q.append(next_id) + + # Add blocks connected via inputs (e.g., reporters, shadow blocks, substacks) + if "inputs" in block_data: + for input_name, input_value in block_data["inputs"].items(): + if isinstance(input_value, list) and len(input_value) >= 2: + value_or_block_id = input_value[1] + if isinstance(value_or_block_id, str) and value_or_block_id in all_blocks: + q.append(value_or_block_id) + # For type code 3 (reporter with default value), the third element might be a connected block + if len(input_value) >= 3 and isinstance(input_value[2], str) and input_value[2] in all_blocks: + q.append(input_value[2]) + + # For C-blocks, add blocks in substacks (if present) + # SUBSTACKs are inputs that have type code 2 and contain the block ID of the first block in the substack + if get_block_type(block_data.get("opcode")) == "c_block": + for input_key, input_val in block_data.get("inputs", {}).items(): + if input_key.startswith("SUBSTACK") and isinstance(input_val, list) and len(input_val) >= 2 and input_val[0] == 2 and isinstance(input_val[1], str) and input_val[1] in all_blocks: + q.append(input_val[1]) + + return script_blocks + + +def analyze_script_structure(script_blocks: dict, hat_block_id: str, sprite_name: str) -> list: + """ + Analyzes the structure of a single Scratch script for common errors. + Returns a list of issue strings. + """ + issues = [] + + # 1. Validate the hat block + hat_block = script_blocks.get(hat_block_id) + if not hat_block: + issues.append(f"Script for sprite '{sprite_name}' (hat ID: {hat_block_id}) has no hat block data.") + return issues # Cannot proceed without a hat block + + if not hat_block.get("topLevel") or hat_block.get("parent") is not None: + issues.append(f"Hat block '{hat_block_id}' for sprite '{sprite_name}' is not marked as topLevel or has a parent.") + + # 2. Check all blocks within the script + for block_id, block_data in script_blocks.items(): + opcode = block_data.get("opcode") + if not opcode: + issues.append(f"Block '{block_id}' for sprite '{sprite_name}' is missing an opcode.") + continue + + # Check if opcode exists in the catalog (simplified check for this example) + if opcode not in ALL_SCRATCH_BLOCKS_CATALOG: + issues.append(f"Block '{block_id}' for sprite '{sprite_name}' has unknown opcode '{opcode}'.") + + # Parent-Child and Next-Previous Linkage + parent_id = block_data.get("parent") + next_id = block_data.get("next") + + if parent_id: + parent_block = script_blocks.get(parent_id) + if not parent_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' references non-existent parent '{parent_id}'.") + else: + parent_block_type = get_block_type(parent_block.get("opcode")) + current_block_type = get_block_type(opcode) + + if parent_block_type in ["stack", "hat"]: + # For stack and hat blocks, the parent's 'next' should point to this block + if parent_block.get("next") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode})'s parent '{parent_id}' does not link back to it via 'next'.") + elif parent_block_type == "c_block": + # For C-blocks, this block should be in a substack input + found_in_substack = False + for input_key, input_val in parent_block.get("inputs", {}).items(): + if isinstance(input_val, list) and len(input_val) >= 2 and input_val[1] == block_id: + if input_val[0] == 2 and input_key.startswith("SUBSTACK"): # Type code 2 for substacks + found_in_substack = True + break + # Shadow blocks and reporter blocks can also be inputs to C-blocks but not as substacks + if not found_in_substack and not block_data.get("shadow") and current_block_type not in ["reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) has parent '{parent_id}' but is not a substack, shadow, or reporter/boolean connection to a C-block.") + elif parent_block_type in ["reporter", "boolean"]: + # Reporter/Boolean blocks can be parents if they are input to another block + found_in_input = False + for input_key, input_val in parent_block.get("inputs", {}).items(): + if isinstance(input_val, list) and len(input_val) >= 2 and input_val[1] == block_id: + found_in_input = True + break + if not found_in_input: + issues.append(f"Block '{block_id}' (opcode: {opcode}) has reporter/boolean parent '{parent_id}' but is not linked via an input.") + else: # e.g., cap blocks cannot be parents to other blocks in a sequence + issues.append(f"Block '{block_id}' (opcode: {opcode}) has an unexpected parent block type '{parent_block_type}' for parent '{parent_id}'.") + + + if next_id: + next_block = script_blocks.get(next_id) + if not next_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' references non-existent next block '{next_id}'.") + elif next_block.get("parent") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode})'s next block '{next_id}' does not link back to it via 'parent'.") + elif block_data.get("shadow"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection.") + elif get_block_type(opcode) == "hat": # Hat blocks should not generally have a 'next' in a linear script sequence + issues.append(f"Hat block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection as a sequential block.") + elif get_block_type(opcode) == "cap": + issues.append(f"Cap block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection as it signifies script end.") + + + # Input validation + if "inputs" in block_data: + for input_name, input_value in block_data["inputs"].items(): + if not isinstance(input_value, list) or len(input_value) < 2: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' has malformed input '{input_name}': {input_value}.") + continue + + type_code = input_value[0] + value_or_block_id = input_value[1] + + # Type code 1: number/string/boolean literal or block ID (reporter/boolean) + # Type code 2: block ID (substack or reporter/boolean plugged in) + # Type code 3: number/string/boolean literal with shadow, or reporter with default value if nothing plugged in + if type_code not in [1, 2, 3]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has invalid type code: {type_code}. Expected 1, 2, or 3.") + + if isinstance(value_or_block_id, str): + # It's a block ID, check if it exists and its parent link is correct + connected_block = script_blocks.get(value_or_block_id) + if not connected_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' references non-existent block ID '{value_or_block_id}'.") + elif connected_block.get("parent") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' connects to '{value_or_block_id}', but '{value_or_block_id}'s parent is not '{block_id}'.") + # Check for type code consistency with connected block type + elif type_code == 2 and get_block_type(connected_block.get("opcode")) not in ["stack", "hat", "c_block", "cap", "reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has type code 2 but connected block '{value_or_block_id}' is not a valid block type for substack/input.") + elif type_code == 1 and get_block_type(connected_block.get("opcode")) not in ["reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has type code 1 but connected block '{value_or_block_id}' is not a reporter or boolean (expected for direct value input).") + + # Specific checks for C-blocks' SUBSTACK + if block_data.get("blockType") == "c_block" and input_name.startswith("SUBSTACK"): # Changed to startswith as SUBSTACK1, SUBSTACK2 might exist + if not (isinstance(value_or_block_id, str) and script_blocks.get(value_or_block_id) and type_code == 2): + issues.append(f"C-block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' has an invalid or missing SUBSTACK input configuration.") + + + # Shadow block specific checks + if block_data.get("shadow"): + if block_data.get("topLevel"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' is incorrectly marked as topLevel.") + if not block_data.get("parent"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' is missing a parent.") + if block_data.get("next"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection.") + # Check fields for math_number and menu blocks + if opcode == "math_number" and not (isinstance(block_data.get("fields", {}).get("NUM"), list) and len(block_data["fields"]["NUM"]) >= 1 and isinstance(block_data["fields"]["NUM"][0], (str, int, float))): + issues.append(f"Math_number shadow block '{block_id}' (opcode: {opcode}) has malformed 'NUM' field.") + # This logic for menu shadow blocks assumes the field name is the opcode in uppercase without _MENU + # Example: for "looks_costumemenu", it expects a field "COSTUME" + if opcode.endswith("menu"): + expected_field = opcode.upper().replace("_MENU", "") + if not (isinstance(block_data.get("fields", {}).get(expected_field), list) and len(block_data["fields"][expected_field]) >= 1 and isinstance(block_data["fields"][expected_field][0], str)): + issues.append(f"Menu shadow block '{block_id}' (opcode: {opcode}) has malformed field '{expected_field}' for its specific menu option.") + + return issues + +# Node 5: Verification Node +def block_verification_node(state: dict) -> dict: + """ + The block verifier check for the if any improvement need if any through logical if else and then add it to improvement_plan. + aftet improvement plan the llm reviewer node also check for other error or issues if any and at last give the review as feedback. + + Args: + state (dict): _description_ + + Returns: + dict: _description_ + """ + logger.info(f"--- Running BlockVerificationNode (Iteration: {state.get('iteration_count', 0)}) ---") + + MAX_IMPROVEMENT_ITERATIONS = 1 # Set a sensible limit to prevent infinite loops + current_iteration = state.get("iteration_count", 0) + project_json = state["project_json"] + targets = project_json["targets"] + + # Initialize needs_improvement for the current run + state["needs_improvement"] = False + block_validation_feedback_overall = [] + + improvement_plan = {"sprite_issues": {}} + + + for target in targets: + sprite_name = target["name"] + all_blocks_for_sprite = target.get("blocks", {}) + + if not all_blocks_for_sprite: + logger.info(f"Sprite '{sprite_name}' has no blocks. Skipping verification.") + continue + + sprite_issues = [] + hat_block_ids = [ + block_id for block_id, block_data in all_blocks_for_sprite.items() + if block_data.get("topLevel") and get_block_type(block_data.get("opcode")) == "hat" + ] + + processed_script_blocks = set() + + if not hat_block_ids: + sprite_issues.append("No top-level hat blocks found for this sprite. Scripts may not run.") + for block_id, block_data in all_blocks_for_sprite.items(): + if block_data.get("topLevel") and not get_block_type(block_data.get("opcode")) == "hat": + sprite_issues.append(f"Top-level block '{block_id}' (opcode: {block_data.get('opcode')}) is not a hat block, so it will not run automatically.") + if not block_data.get("topLevel") and not block_data.get("parent") and not block_data.get("shadow"): + sprite_issues.append(f"Orphaned block '{block_id}' (opcode: {block_data.get('opcode')}) is not top-level, has no parent, and is not a shadow block.") + + for hat_id in hat_block_ids: + logger.info(f"Verifying script starting with hat block '{hat_id}' for sprite '{sprite_name}'.") + + current_script_blocks = filter_script_blocks(all_blocks_for_sprite, hat_id) + processed_script_blocks.update(current_script_blocks.keys()) + + script_issues = analyze_script_structure(current_script_blocks, hat_id, sprite_name) + if script_issues: + sprite_issues.append(f"Issues in script starting with '{hat_id}':") + sprite_issues.extend([f" - {issue}" for issue in script_issues]) + else: + logger.info(f"Script starting with '{hat_id}' for sprite '{sprite_name}' passed basic verification.") + + orphaned_blocks_overall = { + block_id for block_id in all_blocks_for_sprite.keys() + if block_id not in processed_script_blocks + and not all_blocks_for_sprite[block_id].get("topLevel") + and not all_blocks_for_sprite[block_id].get("parent") + } + + if orphaned_blocks_overall: + sprite_issues.append(f"Found {len(orphaned_blocks_overall)} truly orphaned blocks not connected to any valid script: {', '.join(list(orphaned_blocks_overall)[:5])}{'...' if len(orphaned_blocks_overall) > 5 else ''}.") + + if sprite_issues: + improvement_plan["sprite_issues"][sprite_name] = sprite_issues + logger.warning(f"Verification found issues for sprite '{sprite_name}'.") + block_validation_feedback_overall.append(f"Issues for {sprite_name}:\n" + "\n".join([f"- {issue}" for issue in sprite_issues])) + state["needs_improvement"] = True + print(f"\n--- Verification Report (Issues Found for {sprite_name}) ---") + print(json.dumps({sprite_name: sprite_issues}, indent=2)) + else: + logger.info(f"Sprite '{sprite_name}' passed all verification checks.") + + # Consolidate feedback for the LLM + state["block_validation_feedback"] = "\n\n".join(block_validation_feedback_overall) + + print(F"[OVERALL IMPROVEMENT PLAN ON ITERATION {current_iteration}]: {improvement_plan}") + + if state["needs_improvement"]: + + llm_reviewer_prompt = ( + "You are an expert Scratch project reviewer. Your task is to analyze the provided " + "structural issues found in a Scratch project's sprites and suggest improvements or " + "further insights. Focus on clarity, accuracy, and actionable advice.\n\n" + "Here are the detected structural issues:\n" + f"{json.dumps(improvement_plan['sprite_issues'], indent=2)}\n\n" + "Here are the block validation feedback:\n" + f"{json.dumps(state["block_validation_feedback"], indent=2)}\n\n" + + "Please review these issues and provide a consolidated report with potential causes " + "and recommendations for fixing them. If an issue is minor or expected in certain " + "scenarios (e.g., hidden blocks for backward compatibility), please note that." + "Structure your response as a JSON object with 'review_summary' as the key, " + "containing a dictionary where keys are sprite names and values are lists of suggested improvements." + "Example:\n" + "```json\n" + "{\n" + " \"review_summary\": {\n" + " \"Sprite1\": [\n" + " \"Issue: Hat block 'abc' is not topLevel. Recommendation: Ensure all scripts start with a top-level hat block that has no parent.\",\n" + " \"Issue: Block 'xyz' has unknown opcode 'motion_nonexistent'. Recommendation: Verify the opcode against the Scratch 3.0 block reference. This might be a typo or a deprecated block.\"\n" + " ],\n" + " \"Sprite2\": [\n" + " \"Issue: Found 3 orphaned blocks. Recommendation: Reconnect these blocks to existing scripts or remove them if no longer needed.\"\n" + " ]\n" + " }\n" + "}\n" + "```" + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_reviewer_prompt}]}) + raw_review_response = response["messages"][-1].content + try: + state["review_block_feedback"] = extract_json_from_llm_response(raw_review_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_review_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT BLOCKVERFIER ]: {correction_response}") + state["review_block_feedback"] = extract_json_from_llm_response(correction_response["messages"][-1].content) + + logger.info("Agent review feedback added to the state.") + print("\n--- Agent Review Feedback ---") + print(json.dumps(state["review_block_feedback"], indent=2)) + + except Exception as e: + logger.error(f"Error invoking agent for review in BlockVerificationNode: {e}") + state["review_block_feedback"] = {"review_summary": {"Overall": [f"Error during LLM review: {e}"]}} + else: + logger.info("BlockVerificationNode completed: No issues found in any sprite blocks.") + print("\n--- Verification Report (No Issues Found) ---") + state["block_validation_feedback"] = "No issues found in sprite blocks." + state["review_block_feedback"] = {"review_summary": {"Overall": ["No issues found in sprite blocks. All good!"]}} + + # Manage iteration count based on overall needs_improvement flag + if state["needs_improvement"]: + state["iteration_count"] = current_iteration + 1 + if state["iteration_count"] >= MAX_IMPROVEMENT_ITERATIONS: + logger.warning(f"Max improvement iterations ({MAX_IMPROVEMENT_ITERATIONS}) reached for block verification. Forcing 'needs_improvement' to False.") + state["needs_improvement"] = False + state["block_validation_feedback"] += "\n(Note: Max iterations reached for block verification, stopping further improvements.)" + state["improvement_plan"] = improvement_plan + state["review_block_feedback"] = {"review_summary": {"Overall": ["No issues found in sprite blocks. All good!"]}} + logger.info("BlockVerificationNode found issues and added an improvement plan to the state.") + print("\n--- Overall Verification Report (Issues Found) ---") + print(json.dumps(improvement_plan, indent=2)) + else: + state["iteration_count"] = 0 # Reset if no more improvement needed for blocks + + logger.info(f"Block verification completed. Needs Improvement: {state['needs_improvement']}. Feedback: {state['block_validation_feedback'][:100]}...") + print("===========================================================================") + print(f"[BLOCK VERIFICATION NODE: (improvement_plan)]:{state["improvement_plan"]}") + print(f"[BLOCK VERIFICATION NODE: (review_block_feedback)]:{state["review_block_feedback"]}") + return state + +def improvement_block_builder_node(state: GameState): + logger.info("--- Running ImprovementBlockBuilderNode ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + # Pre-load all block-catalog JSONs once + all_catalogs = [ + hat_block_data, + boolean_block_data, + c_block_data, + cap_block_data, + reporter_block_data, + stack_block_data + ] + + improvement_plan = state.get("improvement_plan", {}) + block_verification_feedback = state.get("block_validation_feedback", "no feedback") + + if not improvement_plan: + logger.warning("No improvement plan found in state. Skipping ImprovementBlockBuilderNode.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + # This is the handler which ensure if somehow json response changed it handle it.[DONOT REMOVE BELOW LOGIC] + if improvement_plan.get("improvement_overall_flow", {})=={}: + plan_data = improvement_plan.items() + else: + plan_data= improvement_plan.get("action_overall_flow", {}).items() + + for sprite_name, sprite_improvements_data in plan_data: + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_improvements_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operator_opcodes = plan_entry.get("operator", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + needed_opcodes = ( + motion_opcodes + control_opcodes + operator_opcodes + + sensing_opcodes + looks_opcodes + sound_opcodes + + events_opcodes + data_opcodes + ) + needed_opcodes = list(set(needed_opcodes)) + + # 2) build filtered runtime catalog (if you still need it) + filtered_catalog = { + op: ALL_SCRATCH_BLOCKS_CATALOG[op] + for op in needed_opcodes + if op in ALL_SCRATCH_BLOCKS_CATALOG + } + + # 3) merge human-written catalog + runtime entry for each opcode + combined_blocks = {} + for op in needed_opcodes: + catalog_def = find_block_in_all(op, all_catalogs) or {} + runtime_def = ALL_SCRATCH_BLOCKS_CATALOG.get(op, {}) + # merge: catalog fields first, then runtime overrides/adds + merged = {**catalog_def, **runtime_def} + combined_blocks[op] = merged + print("Combined blocks for this script:", json.dumps(combined_blocks, indent=2)) + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script based on an improvement plan.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is for the event with opcode '{event_opcode}'.\n" + f"The sequential logic to implement is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Based on the planning, the following Scratch block opcodes are expected to be used to implement this logic. Focus on using these specific opcodes where applicable, and refer to the ALL_SCRATCH_BLOCKS_CATALOG for their full structure and required inputs/fields:**\n" + f"Here is the comprehensive catalog of required Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(combined_blocks, indent=2)}\n```\n\n" + f"Here is feedback ans suggetion you should take care of:\n" + f" suggestion:{block_verification_feedback}\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON (EXTREMELY IMPORTANT - FOLLOW THESE EXAMPLES PRECISELY):**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'myBlockID123').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Handle `inputs` for parameters and substacks (CRITICAL DETAIL - PAY CLOSE ATTENTION TO EXAMPLES):**\n" + f" * The value for any key within the `inputs` dictionary MUST always be an **array** of two elements: `[type_code, value_or_block_id]`.\n" + f" * **STRICTLY FORBIDDEN MISTAKE:** DO NOT put an array like `[\"num\", \"value\"]` or `[\"_edge_\", null]` directly as the `value_or_block_id`. This is the source of past errors.\n" + f" * The `type_code` (first element of the array) indicates the nature of the input:\n" + f" * `1`: For a primitive value or a shadow block ID (e.g., a number, string, boolean, or reference to a shadow block). This is the most common type for direct values.\n" + f" * `2`: For a block ID where another block is directly plugged into this input (e.g., an operator block connected to an `if` condition, or the first block of a C-block's substack).\n" + f" * **Correct Example for Numerical Input (e.g., for `motion_movesteps` STEPS, or `motion_gotoxy` X/Y):**\n" + f" If you need to input the number `10` into a block, you MUST create a separate `math_number` shadow block for it, and then reference its ID.\n" + f" ```json\n" + f" // Main block using the number\n" + f" \"mainBlockID\": {{\n" + f" \"opcode\": \"motion_movesteps\",\n" + f" \"inputs\": {{\n" + f" \"STEPS\": [1, \"shadowNumID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the number '10'\n" + f" \"shadowNumID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"mainBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for Dropdown/Menu Input (e.g., `sensing_touchingobject` with 'edge'):**\n" + f" If you need to select 'edge' for the `touching ()?` block, you MUST create a separate `sensing_touchingobjectmenu` shadow block and reference its ID.\n" + f" ```json\n" + f" // Main block using the dropdown selection\n" + f" \"touchingBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\",\n" + f" \"inputs\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [1, \"shadowEdgeMenuID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the 'edge' menu option\n" + f" \"shadowEdgeMenuID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\",\n" + f" \"fields\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [\"_edge_\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"touchingBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block (e.g., `control_forever`):**\n" + f" The `control_forever` block MUST have a `SUBSTACK` input pointing to the first block inside its loop. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_FOREVER_LOOP_ID\"]`.\n" + f" ```json\n" + f" \"foreverBlockID\": {{\n" + f" \"opcode\": \"control_forever\",\n" + f" \"inputs\": {{\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideForeverID\"]\n" + f" }},\n" + f" \"next\": null,\n" + f" \"parent\": \"blockAboveForeverID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"firstBlockInsideForeverID\": {{\n" + f" // ... definition of the first block inside the forever loop\n" + f" \"parent\": \"foreverBlockID\",\n" + f" \"next\": \"secondBlockInsideForeverID\" // if there's another block\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block with condition (e.g., `control_if`):**\n" + f" The `control_if` block MUST have a `CONDITION` input (typically `type_code: 1` referencing a boolean reporter block) and a `SUBSTACK` input (`type_code: 2` referencing the first block inside the if-body).\n" + f" ```json\n" + f" \"ifBlockID\": {{\n" + f" \"opcode\": \"control_if\",\n" + f" \"inputs\": {{\n" + f" \"CONDITION\": [1, \"conditionBlockID\"],\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideIfID\"]\n" + f" }},\n" + f" \"next\": \"blockAfterIfID\",\n" + f" \"parent\": \"blockAboveIfID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"conditionBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", // Example condition block\n" + f" // ... definition for condition block, parent should be \"ifBlockID\"\n" + f" }},\n" + f" \"firstBlockInsideIfID\": {{\n" + f" // ... definition of the first block inside the if body\n" + f" \"parent\": \"ifBlockID\",\n" + f" \"next\": null // or next block if more\n" + f" }}\n" + f" ```\n" + f"5. **Define ALL Shadow Blocks Separately (THIS IS ESSENTIAL):** Every time a block's input requires a number, string literal, or a selection from a dropdown/menu, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow. Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f"6. **`fields` for direct dropdown values/text:** Use the `fields` dictionary ONLY IF the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., the `KEY_OPTION` field within the `event_whenkeypressed` shadow block itself, or the `variable` field on a `data_setvariableto` block). Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Ensure Unique Block IDs:** Every block you generate (main blocks and shadow blocks) must have a unique ID within the entire script's block dictionary.\n" + f"9. **Strictly Use Catalog Opcodes:** You MUST only use `opcode` values that are present in the provided `ALL_SCRATCH_BLOCKS_CATALOG`. Do NOT use unlisted opcodes like `motion_jump`.\n" + f"10. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [ImprovementBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + try: + generated_blocks = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT IMPROVEMENTBLOCKBUILDER ]: {correction_response}") + generated_blocks = extract_json_from_llm_response(correction_response["messages"][-1].content) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Improvement blocks added for sprite '{sprite_name}', script '{event_opcode}' by ImprovementBlockBuilderNode.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with improvement nodes.") + print("Updated project JSON with improvement nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + +#temporarry time delay for handling TPM issue +import time +def delay_for_tpm_node(state: GameState): + logger.info("--- Running DelayForTPMNode ---") + time.sleep(80) # Adjust the delay as needed + logger.info("Delay completed.") + return state + + +# Build the LangGraph workflow +workflow = StateGraph(GameState) + +# Add all nodes to the workflow +workflow.add_node("parse_query", parse_query_and_set_initial_positions) +workflow.add_node("game_description", game_description_node) +workflow.add_node("time_delay_1", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_2", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_3", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_4", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_5", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("initial_plan_build", overall_planner_node) # High-level planning node +workflow.add_node("plan_verifier", plan_verification_node) # Verifies the high-level plan +workflow.add_node("refined_planner", refined_planner_node) # Refines the action plan +workflow.add_node("block_builder", overall_block_builder_node) # Builds blocks from a plan +workflow.add_node("block_verifier", block_verification_node) # Verifies the generated blocks +workflow.add_node("improved_block_builder", improvement_block_builder_node) # For specific block-level improvements + +# Set the entry point +workflow.set_entry_point("game_description") + +# Define the standard initial flow +workflow.add_edge("game_description", "parse_query") +workflow.add_edge("parse_query", "time_delay_1") +workflow.add_edge("time_delay_1", "initial_plan_build") +workflow.add_edge("initial_plan_build", "time_delay_5") +workflow.add_edge("time_delay_5", "plan_verifier") +# Define the conditional logic after plan_verifier (for high-level plan issues) +def decide_next_step_after_plan_verification(state: GameState): + if state.get("needs_improvement", False): + # If the plan needs refinement, go to the refined_planner + return "refined_planner" + else: + # If the plan is good, proceed to building blocks from this plan + return "block_builder" + +workflow.add_conditional_edges( + "plan_verifier", + decide_next_step_after_plan_verification, + { + "refined_planner": "refined_planner", # Path if plan needs refinement + "block_builder": "block_builder" # Path if plan is approved, proceeds to block building + } +) + +# --- CRITICAL CHANGE FOR THE PLAN REFINEMENT LOOP --- +# After refining the plan, it should go back to plan_verifier for re-verification. +workflow.add_edge("refined_planner", "time_delay_2") +workflow.add_edge("time_delay_2", "plan_verifier") # This closes the loop for plan refinement and re-verification. +# Note: The original code had workflow.add_edge("time_delay", "block_builder") here, +# but after refined_planner -> time_delay -> plan_verifier, the decision is made by plan_verifier. +# So, this edge might be redundant or incorrect depending on the desired flow. +# Assuming the intent is for plan_verifier to always decide the next step. + +# After blocks are built, they need to be verified +#workflow.add_edge("time_delay_3", "block_builder") +workflow.add_edge("block_builder", "time_delay_3") +workflow.add_edge("time_delay_3", "block_verifier") + +# Define the conditional logic after block_verifier (for generated blocks issues) +def decide_after_block_verification(state: GameState): + if state.get("needs_improvement", False): + # If blocks need improvement, go to improved_block_builder. + # This assumes improved_block_builder handles specific block-level fixes. + return "improved_block_builder" + else: + # If blocks are good, end the workflow + return "END" + +workflow.add_conditional_edges( + "block_verifier", + decide_after_block_verification, + { + "improved_block_builder": "improved_block_builder", # Path if blocks need improvement + "END": END # Path if blocks are good + } +) + +# Create the loop: If blocks improved, re-verify them +#workflow.add_edge("time_delay_4", "improved_block_builder") +workflow.add_edge("improved_block_builder", "time_delay_4") +workflow.add_edge("time_delay_4", "block_verifier") + +# Compile the workflow graph +app_graph = workflow.compile() + +# # Build the LangGraph workflow +# workflow = StateGraph(GameState) + +# # Add all nodes to the workflow +# workflow.add_node("parse_query", parse_query_and_set_initial_positions) +# workflow.add_node("game_description", game_description_node) +# workflow.add_node("initial_plan_build", overall_planner_node) # High-level planning node +# workflow.add_node("plan_verifier", plan_verification_node) # Verifies the high-level plan +# workflow.add_node("refined_planner", refined_planner_node) # Refines the action plan +# workflow.add_node("block_builder", overall_block_builder_node) # Builds blocks from a plan +# workflow.add_node("block_verifier", block_verification_node) # Verifies the generated blocks +# workflow.add_node("improved_block_builder", improvement_block_builder_node) # For specific block-level improvements + +# # Set the entry point +# workflow.set_entry_point("game_description") + +# # Define the standard initial flow +# workflow.add_edge("game_description", "parse_query") +# workflow.add_edge("parse_query", "initial_plan_build") +# workflow.add_edge("initial_plan_build", "plan_verifier") + +# # Define the conditional logic after plan_verifier (for high-level plan issues) +# def decide_next_step_after_plan_verification(state: GameState): +# if state.get("needs_improvement", False): +# # If the plan needs refinement, go to the refined_planner +# return "refined_planner" +# else: +# # If the plan is good, proceed to building blocks from this plan +# return "block_builder" + +# workflow.add_conditional_edges( +# "plan_verifier", +# decide_next_step_after_plan_verification, +# { +# "refined_planner": "refined_planner", # Path if plan needs refinement +# "block_builder": "block_builder" # Path if plan is approved, proceeds to block building +# } +# ) + +# # --- CRITICAL CHANGE FOR THE PLAN REFINEMENT LOOP --- +# # After refining the plan, it should go back to plan_verifier for re-verification. +# workflow.add_edge("refined_planner", "plan_verifier") # This closes the loop for plan refinement and re-verification. + +# # After blocks are built, they need to be verified +# workflow.add_edge("block_builder", "block_verifier") + +# # Define the conditional logic after block_verifier (for generated blocks issues) +# def decide_after_block_verification(state: GameState): +# if state.get("needs_improvement", False): +# # If blocks need improvement, go to improved_block_builder. +# # This assumes improved_block_builder handles specific block-level fixes. +# return "improved_block_builder" +# else: +# # If blocks are good, end the workflow +# return "END" + +# workflow.add_conditional_edges( +# "block_verifier", +# decide_after_block_verification, +# { +# "improved_block_builder": "improved_block_builder", # Path if blocks need improvement +# "END": END # Path if blocks are good +# } +# ) + +# # Create the loop: If blocks improved, re-verify them +# workflow.add_edge("improved_block_builder", "block_verifier") + +# # Compile the workflow graph +# app_graph = workflow.compile() + + +from IPython.display import Image, display +png_bytes = app_graph.get_graph().draw_mermaid_png() +with open("langgraph_workflow.png", "wb") as f: + f.write(png_bytes) + +### Modified `generate_game` Endpoint + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description", "") + backdrops = payload.get("backdrops", []) + sprites = payload.get("sprites", []) + + logger.info(f"Starting game generation for description: '{desc}'") + + # 1) Initial skeleton generation + project_skeleton = { + "targets": [ + { + "isStage": True, + "name":"Stage", + "objName": "Stage", # ADDED THIS FOR THE STAGE + "variables":{}, # This must be a dict: { "var_id": ["var_name", value] } + "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + for idx, s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project_skeleton["targets"].append({ + "isStage": False, + "name": s["name"], + "objName": s["name"], # objName is also important for sprites + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + logger.info("Initial project skeleton created.") + + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + + try: + os.makedirs(project_folder, exist_ok=True) + # Save initial skeleton and copy assets + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project_skeleton, f, indent=2) + logger.info(f"Initial project skeleton saved to {project_json_path}") + + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source backdrop asset not found: {src_path}") + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source sprite asset not found: {src_path}") + logger.info("Assets copied to project folder.") + + # Initialize the state for LangGraph as a dictionary matching the TypedDict structure + initial_state_dict: GameState = { # Type hint for clarity + "project_json": project_skeleton, + "description": desc, + "project_id": project_id, + "sprite_initial_positions": {}, + "action_plan": {}, + #"behavior_plan": {}, + "improvement_plan": {}, + "needs_improvement": False, + "plan_validation_feedback": {}, + "iteration_count": 0, + "review_block_feedback": {} + } + + # Run the LangGraph workflow with the dictionary input. + final_state_dict: GameState = app_graph.invoke(initial_state_dict) + + # Access elements from the final_state_dict + final_project_json = final_state_dict['project_json'] + + # Save the *final* filled project JSON, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(final_project_json, f, indent=2) + logger.info(f"Final project JSON saved to {project_json_path}") + + return jsonify({"message": "Game generated successfully", "project_id": project_id}) + + except Exception as e: + logger.error(f"Error during game generation workflow for project ID {project_id}: {e}", exc_info=True) + return jsonify(error=f"Error generating game: {e}"), 500 + + +if __name__=="__main__": + logger.info("Starting Flask application...") + app.run(debug=True,port=5000) + + diff --git a/scratch_VLM/scratch_agent/multi_Agent_scratch.py b/scratch_VLM/scratch_agent/multi_Agent_scratch.py new file mode 100644 index 0000000000000000000000000000000000000000..59164a9ff163b33bec73433d67101eff87e7fc70 --- /dev/null +++ b/scratch_VLM/scratch_agent/multi_Agent_scratch.py @@ -0,0 +1,298 @@ +# app.py + +import json + +# This would typically be an LLM API call +def call_llm(prompt_text, temperature=0.7, max_tokens=1024): + """ + Simulates an LLM API call. In a real application, this would interact + with a service like Google Gemini, OpenAI GPT, or Anthropic Claude. + """ + print(f"\n--- LLM Call ---") + print(f"Prompt: {prompt_text[:500]}...") # Print a truncated prompt + # Placeholder for actual LLM response logic + # For demonstration, we'll return a predefined structure based on the agent's role. + if "Role: User Intent & Query Interpreter" in prompt_text: + return { + "identified_objects": { + "Cat": {"type": "Sprite", "role": "player_character"}, + "Obstacle": {"type": "Sprite", "role": "object_to_avoid"}, + "Score": {"type": "Variable", "role": "game_score"} + }, + "high_level_behaviors": [ + {"entity": "Cat", "action": "jump", "trigger": "spacebar_press"}, + {"entity": "Obstacle", "action": "appear_randomly", "frequency": "periodic", "movement": "left_across_screen"}, + {"entity": "Score", "action": "increment", "trigger": "cat_jumps_over_obstacle"}, + {"entity": "Game", "action": "end", "condition": "Cat_touches_Obstacle"} + ], + "constraints": ["game_over_on_collision"] + } + elif "Role: Visual Context Analyzer" in prompt_text: + return { + "visual_elements": { + "cat_image.png": {"inferred_role": "Sprite", "properties": {"costume": "cat_default", "initial_size": "medium"}}, + "obstacle_image.png": {"inferred_role": "Sprite", "properties": {"costume": "rock", "initial_size": "small"}}, + "background_image.png": {"inferred_role": "Backdrop", "properties": {"background_name": "grassland"}} + }, + "inferred_details": ["cat_is_player", "obstacle_is_moving_hazard"] + } + elif "Role: Game Logic & Object Planner" in prompt_text: + return { + "plan_id": "cat_jumping_game_v1", + "sprites": { + "Cat": { + "properties": {"x": -200, "y": -100, "costume": "cat_default"}, + "behaviors": [ + {"event": "when_key_pressed", "key": "space", "action_sequence": ["change_y_up", "change_y_down"]}, + {"event": "when_touching_obstacle", "action_sequence": ["game_over"]} + ] + }, + "Obstacle": { + "properties": {"x": 250, "y": -100, "costume": "rock", "speed": 5}, + "behaviors": [ + {"event": "when_green_flag_clicked", "action_sequence": ["hide", "wait_random", "go_to_random_x", "show", "glide_left_until_edge", "hide", "reset_position_and_repeat"]}, + ] + } + }, + "variables": { + "Score": {"initial_value": 0} + }, + "backdrops": { + "Stage": {"name": "grassland"} + }, + "game_flow": { + "start": "Green Flag Clicked", + "score_mechanic": "Increment Score when Cat jumps over Obstacle", # This is a high-level logic, needs decomposition + "game_over_condition": "Cat touches Obstacle", + "game_over_action": "Stop all scripts, display 'Game Over' message" + } + } + elif "Role: Scratch Block Generator" in prompt_text: + # Example of raw Scratch blocks (simplified JSON for demonstration) + return { + "Cat_Scripts": [ + { + "type": "event_whenkeypressed", + "id": "event_1", + "fields": {"KEY_OPTION": ["space"]}, + "next": { + "type": "motion_changeyby", + "id": "motion_1_up", + "inputs": {"DY": ["10"]}, + "next": { + "type": "control_wait", + "id": "control_1_wait", + "inputs": {"DURATION": ["0.1"]}, + "next": { + "type": "motion_changeyby", + "id": "motion_1_down", + "inputs": {"DY": ["-10"]}, + "next": None + } + } + } + }, + { + "type": "event_whenflagclicked", + "id": "event_2", + "next": { + "type": "control_forever", + "id": "control_2_forever", + "inputs": { + "SUBSTACK": { + "type": "sensing_touchingobject", + "id": "sensing_1", + "fields": {"TOUCHINGOBJECTMENU": ["_mouse_"]}, # Placeholder, should be Obstacle + "next": { + "type": "control_if", + "id": "control_3_if", + "inputs": { + "CONDITION": ["sensing_1"], + "SUBSTACK": { + "type": "control_stop", + "id": "control_4_stop", + "fields": {"STOP_OPTION": ["all"]}, + "next": { + "type": "looks_sayforsecs", + "id": "looks_1_say", + "inputs": {"MESSAGE": ["Game Over!"], "SECS": ["2"]}, + "next": None + } + } + } + } + } + } + } + } + ], + "Obstacle_Scripts": [ + # ... (simplified) + ] + } + elif "Role: Code Validator & Refinement" in prompt_text: + # Simulate validation result + if "motion_1_up" in prompt_text and "motion_1_down" in prompt_text and "control_forever" in prompt_text: + return { + "validation_status": "PASS", + "errors": [], + "refined_blocks": "Validated Scratch JSON or visual representation" + } + else: + return { + "validation_status": "FAIL", + "errors": [{"type": "LogicalError", "description": "Jump sequence incomplete or missing 'touching obstacle' condition.", "agent_to_reprompt": "Game Logic & Object Planner"}], + "refined_blocks": None + } + return "LLM processing failed or returned unexpected output." + +class AIAgent: + def __init__(self, name, role, description): + self.name = name + self.role = role + self.description = description + + def generate_prompt(self, context, task_description): + # Basic prompt structure for each agent + prompt = f"Agent Name: {self.name}\n" + prompt += f"Role: {self.role}\n" + prompt += f"Description: {self.description}\n\n" + prompt += f"Context: {json.dumps(context, indent=2)}\n\n" + prompt += f"Task: {task_description}\n\n" + prompt += "Please provide your output in a structured JSON format as specified for your role." + return prompt + + def execute(self, context, task_description): + prompt = self.generate_prompt(context, task_description) + response = call_llm(prompt) + return response + +class MultiAgentSystem: + def __init__(self): + self.user_intent_agent = AIAgent( + name="UserIntent&QueryInterpreter", + role="User Intent & Query Interpreter", + description="Processes natural language input, extracts explicit and implicit requirements, identifies entities, actions, and constraints. Translates ambiguous human language into structured, machine-interpretable intent." + ) + self.visual_context_agent = AIAgent( + name="VisualContextAnalyzer", + role="Visual Context Analyzer", + description="Processes image inputs, identifies objects, infers potential roles (sprites/backdrops), extracts visual properties, and infers missing details based on visual cues." + ) + self.game_logic_planner_agent = AIAgent( + name="GameLogic&ObjectPlanner", + role="Game Logic & Object Planner", + description="Synthesizes a comprehensive game plan from structured intent and visual context. Defines objects, properties, and actions, mapping high-level game mechanics to Scratch-compatible concepts. Crucial for inferring logical connections and default behaviors." + ) + self.scratch_block_generator_agent = AIAgent( + name="ScratchBlockGenerator", + role="Scratch Block Generator", + description="Translates the detailed game logic plan into specific Scratch code blocks. Accurately selects block categories and types, populates parameters, and arranges them in a valid, executable sequence. STRICTLY adheres to Scratch's block connection rules, parent-child relations, and hierarchical levels." + ) + self.code_validator_agent = AIAgent( + name="CodeValidator&Refinement", + role="Code Validator & Refinement", + description="Reviews generated block sequences for logical consistency, completeness, and strict adherence to Scratch's execution model and syntax. Checks for correct block hierarchy, proper connections (parent-child), and identifies errors. Provides structured feedback for refinement." + ) + + def run_workflow(self, user_query, images=None): + print(f"Starting workflow for query: '{user_query}'") + + # Step 1 & 2: User Input, Intent & Visual Analysis (Parallel & Consolidated) + intent_data = self.user_intent_agent.execute( + context={"user_input": user_query}, + task_description="Analyze the user query to extract key entities, desired actions, and overarching intent. Classify intent (e.g., 'create a game', 'add feature')." + ) + print(f"\nUser Intent: {json.dumps(intent_data, indent=2)}") + + visual_data = {} + if images: + visual_data = self.visual_context_agent.execute( + context={"images_provided": images}, + task_description="Process provided images to detect objects, infer roles (sprite/backdrop), and extract relevant visual properties. Correlate visual information with textual intent." + ) + print(f"\nVisual Context: {json.dumps(visual_data, indent=2)}") + + # Consolidate initial context for the planner + initial_context = { + "user_intent": intent_data, + "visual_context": visual_data + } + + # Step 3: Game Logic Planning + game_plan = self.game_logic_planner_agent.execute( + context=initial_context, + task_description="Formulate a detailed game plan including sprites, backdrops, variables, properties, and high-level logical behaviors. Infer and complete any missing details based on common game patterns." + ) + print(f"\nGame Plan: {json.dumps(game_plan, indent=2)}") + + # Step 4: Scratch Block Generation + # Emphasize parent-child and level handling here for the LLM + block_generation_task = ( + "Translate the detailed game plan into specific Scratch code blocks. " + "Ensure accurate selection, parameterization, and arrangement of blocks. " + "STRICTLY ADHERE to Scratch's block connection rules, especially for **parent-child relationships** " + "and **top-level vs. lower-level block placement**. " + "Hat blocks must be top-level. Stack blocks form sequential chains (parent-child). " + "C-blocks (e.g., 'if-then', 'repeat') must contain child blocks within their 'C' shape. " + "Boolean and Reporter blocks must fit into their designated slots as child inputs. " + "Output in a JSON format consistent with Scratch's internal representation. " + "Example of parent-child/level: " + " Event block (Hat, top-level) -> " + " Forever block (C-block, child of Event, parent of inner blocks) -> " + " Motion block (Stack, child of Forever) -> " + " Looks block (Stack, child of Motion)" + ) + raw_scratch_blocks = self.scratch_block_generator_agent.execute( + context={"game_plan": game_plan}, + task_description=block_generation_task + ) + print(f"\nRaw Scratch Blocks (JSON): {json.dumps(raw_scratch_blocks, indent=2)}") + + # Step 5: Validation & Refinement + validation_report = self.code_validator_agent.execute( + context={"raw_scratch_blocks": raw_scratch_blocks, "original_game_plan": game_plan}, + task_description=( + "Perform rigorous static analysis on the generated Scratch blocks. " + "Verify correct block hierarchy (e.g., Hat blocks at top, C-blocks containing children). " + "Ensure proper block connections, validating parent-child relationships and sequential flow. " + "Check for logical consistency, completeness against the game plan, and absence of orphaned/disconnected blocks. " + "Identify potential errors such as uninitialized variables or infinite loops without exit conditions. " + "If errors are found, suggest specific refinements or indicate which upstream agent needs to be re-prompted (e.g., 'Game Logic & Object Planner' for fundamental logic flaws)." + ) + ) + print(f"\nValidation Report: {json.dumps(validation_report, indent=2)}") + + # Iterative refinement loop (simplified for this example) + if validation_report.get("validation_status") == "FAIL": + print("\nValidation failed. Initiating refinement process (simplified).") + # In a real system, this would trigger a more complex re-prompting loop + # based on validation_report.get("agent_to_reprompt") + # For now, just indicate that refinement is needed. + return {"status": "Refinement Needed", "details": validation_report["errors"]} + else: + print("\nScratch blocks generated and validated successfully!") + return {"status": "Success", "final_scratch_json": raw_scratch_blocks} + +if __name__ == "__main__": + system = MultiAgentSystem() + + # Example 1: Simple Cat Jumping Game + user_query_1 = "Make a cat jumping game with score count. The cat jumps when I press spacebar. An obstacle appears and moves across the screen. Score increases when the cat jumps over the obstacle. Game over when cat touches obstacle." + images_1 = ["cat_image.png", "obstacle_image.png", "background_image.png"] + result_1 = system.run_workflow(user_query_1, images_1) + print(f"\n--- Final Result 1 ---") + print(json.dumps(result_1, indent=2)) + + print("\n" + "="*80 + "\n") + + # Example 2: Modified Behavior (demonstrates potential refinement need) + # This example specifically aims to highlight how the validation could catch + # issues if the block generation doesn't handle parent-child correctly. + # The simulated LLM responses will (for now) still "pass" in a simplified manner, + # but the prompt emphasizes the structural requirements. + user_query_2 = "Create a game where a character moves left and right, and collects coins. Display the score." + images_2 = ["character_image.png", "coin_image.png"] + result_2 = system.run_workflow(user_query_2, images_2) + print(f"\n--- Final Result 2 ---") + print(json.dumps(result_2, indent=2)) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/project.json b/scratch_VLM/scratch_agent/project.json new file mode 100644 index 0000000000000000000000000000000000000000..dec62d61a8bc4cc9d65fe13c5c94aa062f96ec0e --- /dev/null +++ b/scratch_VLM/scratch_agent/project.json @@ -0,0 +1,573 @@ +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "variables": { + "`jEk@4|i[#Fk?(8x)AV.-my variable": [ + "my variable", + 0 + ] + }, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 3, + "costumes": [ + { + "name": "backdrop1", + "dataFormat": "svg", + "assetId": "cd21514d0531fdffb22204e0ec5ed84a", + "md5ext": "cd21514d0531fdffb22204e0ec5ed84a.svg", + "rotationCenterX": 240, + "rotationCenterY": 180 + }, + { + "name": "Blue Sky", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "e7c147730f19d284bcd7b3f00af19bb6", + "rotationCenterX": 240, + "rotationCenterY": 180 + }, + { + "name": "backdrop2", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "cd21514d0531fdffb22204e0ec5ed84a", + "md5ext": "cd21514d0531fdffb22204e0ec5ed84a.svg", + "rotationCenterX": 0, + "rotationCenterY": 0 + }, + { + "name": "Blue Sky2", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "e7c147730f19d284bcd7b3f00af19bb6", + "rotationCenterX": 240, + "rotationCenterY": 180 + } + ], + "sounds": [ + { + "name": "pop", + "assetId": "83a9787d4cb6f3b7632b4ddfebf74367", + "dataFormat": "wav", + "format": "", + "rate": 48000, + "sampleCount": 1123, + "md5ext": "83a9787d4cb6f3b7632b4ddfebf74367.wav" + } + ], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Sprite1", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": { + "HeG?L#U~_ildB(ic9ZwO": { + "opcode": "event_whenkeypressed", + "next": "!@|Vg5}f][PzBxn7,h%j", + "parent": null, + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "space", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 382, + "y": -1089 + }, + "?*7?sGY(4|RK(`J{`YvK": { + "opcode": "motion_changeyby", + "next": "ev{jmIX(#gCrUv$SKe}3", + "parent": "NN-[c:W-u{LT~3-#DVjQ", + "inputs": { + "DY": [ + 1, + [ + 4, + "10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "uQb.+am;/3j{KtfvvY=?": { + "opcode": "motion_changeyby", + "next": "b|XvAr*(skbOSkVU:u2E", + "parent": "TRZwkak|p_::+$CC;C^X", + "inputs": { + "DY": [ + 1, + [ + 4, + "-10" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "NN-[c:W-u{LT~3-#DVjQ": { + "opcode": "control_repeat", + "next": "TRZwkak|p_::+$CC;C^X", + "parent": "{Aa*M~f6BX*#bBX2$)JG", + "inputs": { + "TIMES": [ + 1, + [ + 6, + "10" + ] + ], + "SUBSTACK": [ + 2, + "?*7?sGY(4|RK(`J{`YvK" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "TRZwkak|p_::+$CC;C^X": { + "opcode": "control_repeat", + "next": "1Ogqg13fokw/A*=yiyh2", + "parent": "NN-[c:W-u{LT~3-#DVjQ", + "inputs": { + "TIMES": [ + 1, + [ + 6, + "10" + ] + ], + "SUBSTACK": [ + 2, + "uQb.+am;/3j{KtfvvY=?" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "{Aa*M~f6BX*#bBX2$)JG": { + "opcode": "looks_nextcostume", + "next": "NN-[c:W-u{LT~3-#DVjQ", + "parent": "!@|Vg5}f][PzBxn7,h%j", + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "1Ogqg13fokw/A*=yiyh2": { + "opcode": "looks_nextcostume", + "next": null, + "parent": "TRZwkak|p_::+$CC;C^X", + "inputs": {}, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "!@|Vg5}f][PzBxn7,h%j": { + "opcode": "sound_play", + "next": "{Aa*M~f6BX*#bBX2$)JG", + "parent": "HeG?L#U~_ildB(ic9ZwO", + "inputs": { + "SOUND_MENU": [ + 1, + "WkNn[~_gzFkfuW-PMF=C" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "WkNn[~_gzFkfuW-PMF=C": { + "opcode": "sound_sounds_menu", + "next": null, + "parent": "!@|Vg5}f][PzBxn7,h%j", + "inputs": {}, + "fields": { + "SOUND_MENU": [ + "Meow", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "AJf.#me!i1j||Sq@67[|": { + "opcode": "motion_gotoxy", + "next": "I7)jIy8M@sq7Sd8BCpYX", + "parent": "`y8dy4h(h@cr*LA.UXVV", + "inputs": { + "X": [ + 1, + [ + 4, + "-160" + ] + ], + "Y": [ + 1, + [ + 4, + "-110" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "I7)jIy8M@sq7Sd8BCpYX": { + "opcode": "looks_switchcostumeto", + "next": null, + "parent": "AJf.#me!i1j||Sq@67[|", + "inputs": { + "COSTUME": [ + 1, + "x!JZEl$}gc#kFEk:%w8-" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "x!JZEl$}gc#kFEk:%w8-": { + "opcode": "looks_costume", + "next": null, + "parent": "I7)jIy8M@sq7Sd8BCpYX", + "inputs": {}, + "fields": { + "COSTUME": [ + "costume1", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "`y8dy4h(h@cr*LA.UXVV": { + "opcode": "event_whenkeypressed", + "next": "AJf.#me!i1j||Sq@67[|", + "parent": null, + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "r", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 375, + "y": -1326 + }, + "ev{jmIX(#gCrUv$SKe}3": { + "opcode": "control_wait", + "next": null, + "parent": "?*7?sGY(4|RK(`J{`YvK", + "inputs": { + "DURATION": [ + 1, + [ + 5, + "0.0001" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "b|XvAr*(skbOSkVU:u2E": { + "opcode": "control_wait", + "next": null, + "parent": "uQb.+am;/3j{KtfvvY=?", + "inputs": { + "DURATION": [ + 1, + [ + 5, + "0.0001" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + } + }, + "comments": {}, + "currentCostume": 0, + "costumes": [ + { + "name": "costume1", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "bcf454acf82e4504149f7ffe07081dbc", + "md5ext": "bcf454acf82e4504149f7ffe07081dbc.svg", + "rotationCenterX": 48, + "rotationCenterY": 50 + }, + { + "name": "costume2", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "0fb9be3e8397c983338cb71dc84d0b25", + "md5ext": "0fb9be3e8397c983338cb71dc84d0b25.svg", + "rotationCenterX": 46, + "rotationCenterY": 53 + } + ], + "sounds": [ + { + "name": "Meow", + "assetId": "83c36d806dc92327b9e7049a565c6bff", + "dataFormat": "wav", + "format": "", + "rate": 48000, + "sampleCount": 40681, + "md5ext": "83c36d806dc92327b9e7049a565c6bff.wav" + } + ], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": -160, + "y": -110, + "size": 100, + "direction": 90, + "draggable": false, + "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "Soccer Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": { + "|SKxcVN5TNKDX,gr~az]": { + "opcode": "motion_gotoxy", + "next": "N,aJ1Xfm7L]wAh3~vW(G", + "parent": "9AB%|aA#uIm%c[*E@-G!", + "inputs": { + "X": [ + 1, + [ + 4, + "240" + ] + ], + "Y": [ + 1, + [ + 4, + "-135" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "N,aJ1Xfm7L]wAh3~vW(G": { + "opcode": "motion_glidesecstoxy", + "next": null, + "parent": "|SKxcVN5TNKDX,gr~az]", + "inputs": { + "SECS": [ + 1, + [ + 4, + "3" + ] + ], + "X": [ + 1, + [ + 4, + "-240" + ] + ], + "Y": [ + 1, + [ + 4, + "-135" + ] + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "9AB%|aA#uIm%c[*E@-G!": { + "opcode": "control_forever", + "next": null, + "parent": "j$vhNZV0BX6K,kQ`0z*6", + "inputs": { + "SUBSTACK": [ + 2, + "|SKxcVN5TNKDX,gr~az]" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "_i*kwQ)aOwUC]~GA7fi9": { + "opcode": "control_wait_until", + "next": "o6.@O)$`Uj!4wn_J51@,", + "parent": "X++2DZfwF[Y8/8_81qvx", + "inputs": { + "CONDITION": [ + 2, + "*2`Y810BXugG=NK-0.VH" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "*2`Y810BXugG=NK-0.VH": { + "opcode": "sensing_touchingobject", + "next": null, + "parent": "_i*kwQ)aOwUC]~GA7fi9", + "inputs": { + "TOUCHINGOBJECTMENU": [ + 1, + "xI/cOqm1ddbNeJ8d0#Rb" + ] + }, + "fields": {}, + "shadow": false, + "topLevel": false + }, + "xI/cOqm1ddbNeJ8d0#Rb": { + "opcode": "sensing_touchingobjectmenu", + "next": null, + "parent": "*2`Y810BXugG=NK-0.VH", + "inputs": {}, + "fields": { + "TOUCHINGOBJECTMENU": [ + "Sprite1", + null + ] + }, + "shadow": true, + "topLevel": false + }, + "j$vhNZV0BX6K,kQ`0z*6": { + "opcode": "event_whenkeypressed", + "next": "9AB%|aA#uIm%c[*E@-G!", + "parent": null, + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "r", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 386, + "y": -1221 + }, + "X++2DZfwF[Y8/8_81qvx": { + "opcode": "event_whenkeypressed", + "next": "_i*kwQ)aOwUC]~GA7fi9", + "parent": null, + "inputs": {}, + "fields": { + "KEY_OPTION": [ + "r", + null + ] + }, + "shadow": false, + "topLevel": true, + "x": 399, + "y": -1510 + }, + "o6.@O)$`Uj!4wn_J51@,": { + "opcode": "control_stop", + "next": null, + "parent": "_i*kwQ)aOwUC]~GA7fi9", + "inputs": {}, + "fields": { + "STOP_OPTION": [ + "all", + null + ] + }, + "shadow": false, + "topLevel": false, + "mutation": { + "tagName": "mutation", + "children": [], + "hasnext": "false" + } + } + }, + "comments": {}, + "currentCostume": 0, + "costumes": [ + { + "name": "soccer ball", + "bitmapResolution": 1, + "dataFormat": "svg", + "assetId": "5d973d7a3a8be3f3bd6e1cd0f73c32b5", + "md5ext": "5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg", + "rotationCenterX": 23, + "rotationCenterY": 22 + } + ], + "sounds": [ + { + "name": "basketball bounce", + "assetId": "1727f65b5f22d151685b8e5917456a60", + "dataFormat": "wav", + "format": "adpcm", + "rate": 22050, + "sampleCount": 8129, + "md5ext": "1727f65b5f22d151685b8e5917456a60.wav" + } + ], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": -113.75999999999999, + "y": -135, + "size": 100, + "direction": 90, + "draggable": false, + "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36" + } +} \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/promptS.txt b/scratch_VLM/scratch_agent/promptS.txt new file mode 100644 index 0000000000000000000000000000000000000000..714b825c418429198df7387bca3bc681cb2d31e4 --- /dev/null +++ b/scratch_VLM/scratch_agent/promptS.txt @@ -0,0 +1,560 @@ +You are GameScratchAgent, an AI that generates Scratch‐VM 3.x game projects as JSON. For every user request (e.g. “create a jumping cat game over obstacle”), do the following: + +0. — Skeleton Generation +Output exactly this minimal JSON structure first (replace assetIds/md5ext with placeholders if unknown): +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least one backdrop */ ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Cat", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least two costumes for jump */ ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, "y": 0, "size":100, "direction":90, + "draggable": false, "rotationStyle":"all around" + }, + { + "isStage": false, + "name": "Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* one obstacle image */ ], + "sounds": [], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": 0, "y": 0, "size":100, "direction":90, + "draggable": false, "rotationStyle":"all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "" + } +} + +1. — Phase 1: Resource Collector +• Identify and add all required costumes and sounds for each target. +• Fill in their assetId, md5ext, bitmapResolution, and rotation centers. + +2. — Phase 2: Action Node Builder +• Populate each target’s blocks with valid Scratch opcodes: + – Cat: jumping logic (key‐press → change y by +n, wait, change y by –n), costume cycling, “Meow” sound. + – Ball: continuous glide or change x toward Cat, reset position on restart. +• Use proper event_whenflagclicked, event_whenkeypressed, control_forever, motion_changeyby, looks_nextcostume, sound_play, etc. + +3. — Phase 3: Behaviour Collector +• Define collision detection: control_wait_until + sensing_touchingobject. +• On touch, broadcast “game over” or call control_stop all. +• Add a restart script: event_whenkeypressed “r” → go to initial positions, reset variables. + +4. — Variables, Lists, Broadcasts, Comments +• If the user mentions scores or timers, create entries under variables. +• Use lists for inventories or high-score tables. +• Populate broadcasts with any custom messages (e.g. “game over”, “restart”). +• Leave comments empty unless the user explicitly requests inline notes. + +5. — Final Output +• After skeleton + phases, output one consolidated JSON object—no additional text. +• Ensure monitors, extensions and meta are present and correct. +• All block IDs must be unique strings. +• Do not wrap the JSON in any markdown or explanatory text. + +--- + +Whenever the user says “create …,” run steps 0→1→2→3→4 and emit the final JSON. + + +####################################################################################################################################################################################### +####################################################################################################################################################################################### +SYSTEM_PROMPT = """ +You are GameScratchAgent, an AI specialized in generating Scratch-VM 3.x game projects as JSON. For every user request (e.g., "create a jumping cat game over obstacle with score count"), follow these steps carefully: + +--- + +**Scratch Concepts to Use:** + +- Resources are categorized as: + 1. motion: position and movement blocks (e.g., change x/y, glide, go to) + 2. looks: costumes, costume switching, speech bubbles + 3. sound: play sound, change volume, pitch, etc. + 4. event: triggers like when flag clicked, when key pressed + 5. control: loops, conditionals, forever, wait, stop + 6. sensing: detect touching objects, key pressed, mouse position + 7. operator: arithmetic, comparisons, random numbers + 8. variable: set, change, show, hide variables + +--- + +**Step 0: Skeleton Generation** + +Output exactly this minimal JSON structure first (replace assetIds/md5ext with placeholders if unknown): + +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least one backdrop */ ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Cat", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least two costumes for jump */ ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* one obstacle image */ ], + "sounds": [], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "" + } +} + +--- + +**Step 1: Resource Collector** + +- Identify all costumes, sounds, and backdrops needed for each target. +- Assign valid Scratch assetIds and md5ext for each resource or use placeholders if unknown. +- Include rotation centers and bitmapResolution where applicable. +- Add any variables or lists mentioned by the user (e.g., score, timer). + +--- + +**Step 2: Action Node Builder** + +- For each sprite, build blocks using Scratch opcodes: + - Cat: jumping logic triggered by "space" key (change y by +n, wait, change y by -n), costume cycling, play "Meow" sound. + - Ball: continuous movement (e.g., glide or change x) towards the cat, reset position on restart. +- Use event blocks like `event_whenflagclicked`, `event_whenkeypressed`. +- Use control blocks like `control_forever`, `control_repeat`, `control_wait`. +- Use motion blocks like `motion_changeyby`, `motion_gotoxy`, `motion_glidesecstoxy`. +- Use looks blocks like `looks_nextcostume`, `looks_switchcostumeto`. +- Use sound blocks like `sound_play`, `sound_sounds_menu`. + +--- + +**Step 3: Behaviour Collector** + +- Add sensing blocks to detect collisions: `control_wait_until` + `sensing_touchingobject`. +- On collision, broadcast "game over" or use `control_stop all`. +- Add restart logic triggered by key "r": reset positions and variables. + +--- + +**Step 4: Variables, Lists, Broadcasts, Comments** + +- Create variables for score or timers if mentioned. +- Use lists for inventories or high-score tables if needed. +- Define broadcasts for messages like "game over", "restart". +- Leave comments empty unless user requests inline notes. + +--- + +**Step 5: Final Output** + +- Output one consolidated JSON object with all targets, monitors, extensions, and meta. +- Ensure all block IDs are unique strings. +- Do not include any markdown, explanation, or commentary—only the JSON. + +--- + +**Example user input:** "create a jumping cat game over obstacle with score count" + +**Your output must be a valid Scratch-VM 3.x JSON project matching the above structure and logic.** + +--- + +Whenever the user says "create ...", run steps 0→1→2→3→4 and emit the final JSON. +""" + +####################################################################################################################################################################################### +####################################################################################################################################################################################### +SYSTEM_PROMPT = """ +You are GameScratchAgent, an AI that generates Scratch-VM 3.x game projects as JSON. +When the user says “create …”, run steps 0→1→2→3→4 and emit **only** the final JSON object—no markdown or commentary. + +0. — Skeleton Generation +Output exactly this minimal JSON structure first (use `'placeholder.ext'` for any unknown assetIds or md5ext). **Note:** include both `name` and `objName` on every target to satisfy SB3 schema. + +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "objName": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least one backdrop */ ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Cat", + "objName": "Cat", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least two costumes for jump */ ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "Ball", + "objName": "Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* one obstacle image */ ], + "sounds": [], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "" + } +} + +1. — Phase 1: Resource Collector +• Identify and add all required costumes and sounds for each target. +• Fill in their `assetId`, `md5ext`, `bitmapResolution`, and `rotationCenterX`/`rotationCenterY`. +• If any are unknown, use `"placeholder"` for `assetId` and `"placeholder.ext"` for `md5ext`. + +2. — Phase 2: Action Node Builder +Populate each target’s `"blocks"` with valid Scratch opcodes based on the game description. +**General Rules:** + - **Hat Blocks** (`event_whenflagclicked`, `event_whenkeypressed`): + - `topLevel: true`, `parent: null`, and `next: null` unless spawning a substack via inputs. + - **Stack Blocks** (`motion_movesteps`, `sound_play`): + - Vertical chain with `"next": ""`. + - **C-Blocks** (`control_repeat`, `control_if_else`): + - Nested via inputs (`"SUBSTACK"`, `"SUBSTACK2"`), include `mutation` when needed (e.g. `hasnext="true"`). + - **Boolean Blocks** (`sensing_touchingobject`, `operator_and`): + - In `inputs` of a C-Block or reporter. + - **Reporter Blocks** (`variable_get`, `sensing_mousex`): + - In rounded input slots. + - **Cap Blocks** (`control_stop`): + - `next: null`. + - Example: + ```json + "capBlock1": { + "opcode":"control_stop", + "parent":"ifBlock123", + "next":null, + "inputs":{}, + "fields":{"STOP_OPTION":["all",null]}, + "shadow":false, + "topLevel":false + } + ``` + +• **IDs:** Scratch-style alphanumeric (e.g. `abc123`, `q2$e34`), unique across the project. +• **Inputs:** `[, ""]` where type=1 (shadow), 2 (block+shadow), 3 (block only). +• **Fields:** `["value", null]`. +• **shadow:** `true` for inline shadows; otherwise `false`. +• **topLevel:** `true` only for Hat blocks. + +3. — Phase 3: Behaviour Collector +• **Interactions:** e.g. use `sensing_touchingobject` for collisions → broadcast or `control_stop`. +• **Game State:** + - **Start:** `event_whenflagclicked` initializes positions/variables. + - **Game Over:** define a broadcast map like: + ```json + "broadcasts": { + "gameOver": "gameOver" + } + ``` + then `control_stop all`. + - **Scoring:** `data_changevariableby` in relevant scripts. + - **Restart:** `event_whenkeypressed "r"` resets positions and variables. + +4. — Variables, Lists, Broadcasts, Comments +• **Variables:** + ```json + "variables": { + "`id123-myScore": ["myScore", 0] + } + +####################################################################################################################################################################################### +####################################################################################################################################################################################### + + """ +SYSTEM_PROMPT = """ +You are GameScratchAgent, an AI that generates Scratch-VM 3.x game projects as JSON. +When the user says “create …”, run steps 0→1→2→3→4 and emit **only** the final JSON object—no markdown or commentary. + +0. — Skeleton Generation +Output exactly this minimal JSON structure first (use 'placeholder' for unknown assetId and 'placeholder.ext' for unknown md5ext). **Include both `name` and `objName` on every target**: + +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "objName": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least one backdrop */ ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Cat", + "objName": "Cat", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least two costumes for jump */ ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "Ball", + "objName": "Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* one obstacle image */ ], + "sounds": [], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "" + } +} + +1. — Phase 1: Resource Collector +• Identify and add all required costumes and sounds. +• Fill in `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY`. +• If unknown, use `"placeholder"` for `assetId` and `"placeholder.ext"` for `md5ext`. + +2. — Phase 2: Action Node Builder +Populate each target’s `"blocks"` with valid Scratch opcodes based on the game description. +**General Block-Generation Rules:** + • **Hat Blocks** (`event_whenflagclicked`, `event_whenkeypressed`): + - Always start scripts. + - `topLevel: true`, `parent: null`, `next: null` unless they spawn a substack via an input. + • **Stack Blocks** (`motion_movesteps`, `looks_say`, `sound_playuntildone`): + - Form a vertical chain. + - Each (except the last) has `"next": ""`. + • **C-Blocks** (`control_repeat`, `control_if`, `control_if_else`, `control_forever`): + - Contain nested stacks via inputs like `"SUBSTACK": [2, "childId"]` (type 2 = block+shadow). + - For `if_else`, include a `mutation`: + ```json + "mutation": {"tagName":"mutation","children":[],"haselse":"true"} + ``` + • **Boolean Blocks** (`sensing_touchingobject`, `operator_and`): + - Go in C-block inputs as `[2, "booleanId"]`. + • **Reporter Blocks** (`variable_get`, `sensing_mousex`): + - Go in rounded input slots as `[2, "reporterId"]`. + • **Cap Blocks** (`control_stop`): + - End scripts: `next: null`, `shadow: false`, `topLevel: false`. + - Example: + ```json + "cap1": { + "opcode":"control_stop", + "parent":"if1", + "next":null, + "inputs":{}, + "fields":{"STOP_OPTION":["all",null]}, + "shadow":false, + "topLevel":false + } + ``` +**ID Conventions:** +- Use Scratch-style alphanumeric (e.g., `abc123`, `q2$e34`), unique across the project. +- **Inputs** always arrays: `[, ""]` (type 1=shadow, 2=block+shadow, 3=block only). +- **Fields** always: `["value", null]`. +- **Broadcasts** a simple string map: `"gameOver": "gameOver"`. + +3. — Phase 3: Behaviour Collector +• **Interactions**: e.g. detect collisions with `sensing_touchingobject` → broadcast or `control_stop`. +• **Game State**: + - **Start**: `event_whenflagclicked` for setup. + - **Game Over**: + ```json + "broadcasts": {"gameOver":"gameOver"} + ``` + then `control_stop all`. + - **Scoring**: `data_changevariableby` with inputs `[2,"varId"]`. + - **Restart**: `event_whenkeypressed "r"` resets everything. + +4. — Variables, Lists, Broadcasts, Comments +• **Variables**: + ```json + "variables": {"`id123-score":["score",0]} +```` + +• **Lists**: + +```json +"lists": {"`id456-items":["items",[]]} +``` + +• **Broadcasts**: string‐to‐string map. +• **Comments**: empty unless explicitly requested. + +5. — Final Output + Emit one consolidated JSON object only. + Include `"monitors"`, `"extensions"`, `"meta"`, and **all** `objName` fields. + Ensure every block ID, input, and field strictly follows the Scratch 3.0 schema. + +**If the user provides partial JSON**, detect any missing `objName`, incorrect input types, or broadcast formats, and fill in all missing pieces using the rules above. +""" +####################################################################################################################################################################################### +####################################################################################################################################################################################### + + +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema, including: +- **Target structures:** `isStage`, `name`, `objName`, `variables`, `lists`, `broadcasts`, `blocks`, `costumes`, `sounds`, etc. +- **Block types:** Hat, Stack, C-Blocks, Boolean, Reporter, Cap blocks. +- **Block structure:** `opcode`, `parent`, `next`, `inputs`, `fields`, `mutation`, `topLevel`, `shadow`. +- **ID Conventions:** Unique alphanumeric IDs for blocks. +- **Input/Field Formats:** `[, ""]` for inputs, `["value", null]` for fields. +- **Asset properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY`. +- **Standard game mechanics:** Initial setup, movement, collisions, scoring, game over, reset logic. + +When responding, you must: +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, and field strictly follows the Scratch 3.0 schema. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/requirements.txt b/scratch_VLM/scratch_agent/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..efa3b050e5c364238dbef68f6fa0d77c2e4cc25b --- /dev/null +++ b/scratch_VLM/scratch_agent/requirements.txt @@ -0,0 +1,151 @@ +gradio +requests +accelerate +agent +aiohappyeyeballs +aiohttp +aiosignal +annotated-types +anyio +arxiv +asttokens +attrs +beautifulsoup4 +bidict +blinker +bs4 +certifi +charset-normalizer +click +colorama +comm +dataclasses-json +debugpy +decorator +distro +dnspython +duckduckgo_search +duckduckgo-search +eventlet +executing +faiss-cpu +ffmpeg-python +filelock +Flask +Flask-SocketIO +frozenlist +fsspec +greenlet +groq +gunicorn +h11 +httpcore +httpx +httpx-sse +huggingface-hub +idna +ipykernel +ipython +ipython_pygments_lexers +itsdangerous +jedi +Jinja2 +joblib +jsonpatch +jsonpointer +jupyter_client +jupyter_core +langchain +langchain-community +langchain-core +langchain_experimental +langchain-groq +langchain-mistralai +langchain-pinecone +langchain-text-splitters +langgraph +langgraph-checkpoint +langgraph-prebuilt +langgraph-sdk +langsmith +lxml +markdown2 +MarkupSafe +marshmallow +matplotlib-inline +mpmath +multidict +mypy_extensions +nest-asyncio +networkx +numexpr +numpy +orjson +ormsgpack +pandas +packaging +parso +pillow +platformdirs +playwright +primp +prompt_toolkit +propcache +psutil +pure_eval +pydantic +pydantic-settings +pydantic_core +pinecone +pinecone-client +pyee +Pygments +PyMuPDF +PyPDF2 +pytesseract +python-dateutil +python-dotenv +python-engineio +python-socketio +PyYAML +pyzmq +regex +requests +requests-toolbelt +safetensors +scikit-learn +scipy +sentence-transformers +setuptools +simple-websocket +six +sniffio +soupsieve +SQLAlchemy +stack-data +sympy +tenacity +threadpoolctl +tokenizers +torch +tornado +tqdm +traitlets +transformers +typing-inspect +typing-inspection +typing_extensions +urllib3 +wcwidth +Werkzeug +wsproto +wikipedia +xxhash +yarl +zstandard +youtube-transcript-api +pytube +newspaper3k +SpeechRecognition +python-docx +lxml_html_clean \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/scratch_agent.py b/scratch_VLM/scratch_agent/scratch_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..91c7d16a1fd6ec5304f929e0de4aa692a9682f39 --- /dev/null +++ b/scratch_VLM/scratch_agent/scratch_agent.py @@ -0,0 +1,233 @@ +from flask import Flask, request, jsonify, render_template +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +import os +import json +from dotenv import load_dotenv +from langchain_core.messages import AIMessage +import re +# Load environment variables from .env file +load_dotenv() # This line ensures .env variables are loaded + +app = Flask(__name__) + +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +llm = ChatGroq( + model="deepseek-r1-distill-llama-70b", # or your preferred model + temperature=0.0, # Set to 0 for deterministic output +) +# Your detailed system prompt as a single string +SYSTEM_PROMPT = """ +You are GameScratchAgent, an AI specialized in generating Scratch-VM 3.x game projects as JSON. For every user request (e.g., "create a jumping cat game over obstacle with score count"), follow these steps carefully: + +--- + +**Scratch Concepts to Use:** + +- Resources are categorized as: + 1. motion: position and movement blocks (e.g., change x/y, glide, go to) + 2. looks: costumes, costume switching, speech bubbles + 3. sound: play sound, change volume, pitch, etc. + 4. event: triggers like when flag clicked, when key pressed + 5. control: loops, conditionals, forever, wait, stop + 6. sensing: detect touching objects, key pressed, mouse position + 7. operator: arithmetic, comparisons, random numbers + 8. variable: set, change, show, hide variables + +--- + +**Step 0: Skeleton Generation** + +Output exactly this minimal JSON structure first (replace assetIds/md5ext with placeholders if unknown): + +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least one backdrop */ ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Cat", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least two costumes for jump */ ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* one obstacle image */ ], + "sounds": [], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "" + } +} + +--- + +**Step 1: Resource Collector** + +- Identify all costumes, sounds, and backdrops needed for each target. +- Assign valid Scratch assetIds and md5ext for each resource or use placeholders if unknown. +- Include rotation centers and bitmapResolution where applicable. +- Add any variables or lists mentioned by the user (e.g., score, timer). + +--- + +**Step 2: Action Node Builder** + +- For each sprite, build blocks using Scratch opcodes: + - Cat: jumping logic triggered by "space" key (change y by +n, wait, change y by -n), costume cycling, play "Meow" sound. + - Ball: continuous movement (e.g., glide or change x) towards the cat, reset position on restart. +- Use event blocks like `event_whenflagclicked`, `event_whenkeypressed`. +- Use control blocks like `control_forever`, `control_repeat`, `control_wait`. +- Use motion blocks like `motion_changeyby`, `motion_gotoxy`, `motion_glidesecstoxy`. +- Use looks blocks like `looks_nextcostume`, `looks_switchcostumeto`. +- Use sound blocks like `sound_play`, `sound_sounds_menu`. + +--- + +**Step 3: Behaviour Collector** + +- Add sensing blocks to detect collisions: `control_wait_until` + `sensing_touchingobject`. +- On collision, broadcast "game over" or use `control_stop all`. +- Add restart logic triggered by key "r": reset positions and variables. + +--- + +**Step 4: Variables, Lists, Broadcasts, Comments** + +- Create variables for score or timers if mentioned. +- Use lists for inventories or high-score tables if needed. +- Define broadcasts for messages like "game over", "restart". +- Leave comments empty unless user requests inline notes. + +--- + +**Step 5: Final Output** + +- Output one consolidated JSON object with all targets, monitors, extensions, and meta. +- Ensure all block IDs are unique strings. +- Do not include any markdown, explanation, or commentary—only the JSON. + +--- + +**Example user input:** "create a jumping cat game over obstacle with score count" + +**Your output must be a valid Scratch-VM 3.x JSON project matching the above structure and logic.** + +--- + +Whenever the user says "create ...", run steps 0→1→2→3→4 and emit the final JSON. +""" + +# Create the LangGraph agent with the system prompt +agent = create_react_agent( + model=llm, # or your preferred model + tools=[], + prompt=SYSTEM_PROMPT +) + +@app.route("/", methods=["GET"]) +def index(): + return render_template("index3.html") + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + data = request.json + user_input = data.get("request", "") + if not user_input: + return jsonify({"error": "Missing 'request' field in JSON"}), 400 + + prompt = f"create {user_input}" + + response = agent.invoke({"messages": [{"role": "user", "content": prompt}]}) + print("Agent response:", response) + + try: + # The 'response' object is a dictionary with a 'messages' key + # The last message in the 'messages' list is typically the AI's final response + last_message = response.get("messages")[-1] + + if isinstance(last_message, AIMessage): + raw_response_content = last_message.content + else: + # Fallback if it's not an AIMessage, although for final output it should be. + raw_response_content = str(last_message) + + # Use regex to extract the JSON string enclosed in ```json ... ``` + json_match = re.search(r"```json\n(.*?)\n```", raw_response_content, re.DOTALL) + + if json_match: + result_text = json_match.group(1).strip() # Extract the content of the first group + else: + # If no JSON block found, try to parse the whole thing (might work for very simple cases) + result_text = raw_response_content.strip() + + + result_json = json.loads(result_text) + + except json.JSONDecodeError as e: + print(f"JSONDecodeError: {e}") + # Return a more descriptive error if JSON parsing fails + return jsonify({"error": "Agent did not return valid JSON or JSON block not found/parseable", "raw_response": raw_response_content, "extracted_text_attempt": result_text if 'result_text' in locals() else 'N/A'}), 500 + except KeyError as e: + print(f"KeyError: {e} - 'messages' key not found or list is empty in response.") + return jsonify({"error": f"Unexpected response structure from agent: {str(e)}", "raw_response": str(response)}), 500 + except IndexError: + print("IndexError: 'messages' list is empty in response.") + return jsonify({"error": "Agent response 'messages' list is empty.", "raw_response": str(response)}), 500 + except Exception as e: + print(f"Unexpected error: {e}") + return jsonify({"error": f"An unexpected error occurred: {str(e)}", "raw_response": str(response)}), 500 + + return jsonify(result_json) + +if __name__ == "__main__": + app.run(debug=True, port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/scratch_app.py b/scratch_VLM/scratch_agent/scratch_app.py new file mode 100644 index 0000000000000000000000000000000000000000..56f517e624e01fc3c40b69cb729c4ce3876c039f --- /dev/null +++ b/scratch_VLM/scratch_agent/scratch_app.py @@ -0,0 +1,374 @@ +from flask import Flask, request, jsonify, render_template +from langgraph.prebuilt import create_react_agent +from langchain_core.messages import AIMessage +from flask import Flask, request, jsonify, render_template +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +from PIL import Image +import os, json, re +from PIL import Image +import shutil # Import shutil for copying files +import uuid # Import uuid for generating unique folder names +from langgraph.graph import StateGraph, END +from langgraph.prebuilt import create_react_agent + +app = Flask(__name__) + +# ——— LLM / Vision model setup ——— +# Replace ChatGroq with the vision-capable llama-4-maverick +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +# llm = ChatGroq( +# model="meta-llama/llama-4-maverick-17b-128e-instruct", # or your preferred model +# temperature=0.0, # Set to 0 for deterministic output +# ) + +llm = ChatGroq( + model="deepseek-r1-distill-llama-70b", # or your preferred model + temperature=0.0, # Set to 0 for deterministic output +) + +SYSTEM_PROMPT = """ +You are GameScratchAgent, an AI that generates Scratch-VM 3.x game projects as JSON. +When the user says “create …”, run steps 0→1→2→3→4 and emit **only** the final JSON object—no markdown or commentary. + +0. — Skeleton Generation +Output exactly this minimal JSON structure first (use 'placeholder' for unknown assetId and 'placeholder.ext' for unknown md5ext). **Include both `name` and `objName` on every target**: + +{ + "targets": [ + { + "isStage": true, + "name": "Stage", + "objName": "Stage", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least one backdrop */ ], + "sounds": [], + "volume": 100, + "layerOrder": 0, + "tempo": 60, + "videoTransparency": 50, + "videoState": "on", + "textToSpeechLanguage": null + }, + { + "isStage": false, + "name": "Cat", + "objName": "Cat", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* at least two costumes for jump */ ], + "sounds": [], + "volume": 100, + "layerOrder": 2, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + }, + { + "isStage": false, + "name": "Ball", + "objName": "Ball", + "variables": {}, + "lists": {}, + "broadcasts": {}, + "blocks": {}, + "comments": {}, + "currentCostume": 0, + "costumes": [ /* one obstacle image */ ], + "sounds": [], + "volume": 100, + "layerOrder": 1, + "visible": true, + "x": 0, "y": 0, "size": 100, "direction": 90, + "draggable": false, "rotationStyle": "all around" + } + ], + "monitors": [], + "extensions": [], + "meta": { + "semver": "3.0.0", + "vm": "11.1.0", + "agent": "" + } +} + +1. — Phase 1: Resource Collector +• Identify and add all required costumes and sounds. +• Fill in `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY`. +• If unknown, use `"placeholder"` for `assetId` and `"placeholder.ext"` for `md5ext`. + +2. — Phase 2: Action Node Builder +Populate each target’s `"blocks"` with valid Scratch opcodes based on the game description. +**General Block-Generation Rules:** + • **Hat Blocks** (`event_whenflagclicked`, `event_whenkeypressed`): + - Always start scripts. + - `topLevel: true`, `parent: null`, `next: null` unless they spawn a substack via an input. + • **Stack Blocks** (`motion_movesteps`, `looks_say`, `sound_playuntildone`): + - Form a vertical chain. + - Each (except the last) has `"next": ""`. + • **C-Blocks** (`control_repeat`, `control_if`, `control_if_else`, `control_forever`): + - Contain nested stacks via inputs like `"SUBSTACK": [2, "childId"]` (type 2 = block+shadow). + - For `if_else`, include a `mutation`: + ```json + "mutation": {"tagName":"mutation","children":[],"haselse":"true"} + ``` + • **Boolean Blocks** (`sensing_touchingobject`, `operator_and`): + - Go in C-block inputs as `[2, "booleanId"]`. + • **Reporter Blocks** (`variable_get`, `sensing_mousex`): + - Go in rounded input slots as `[2, "reporterId"]`. + • **Cap Blocks** (`control_stop`): + - End scripts: `next: null`, `shadow: false`, `topLevel: false`. + - Example: + ```json + "cap1": { + "opcode":"control_stop", + "parent":"if1", + "next":null, + "inputs":{}, + "fields":{"STOP_OPTION":["all",null]}, + "shadow":false, + "topLevel":false + } + ``` +**ID Conventions:** +- Use Scratch-style alphanumeric (e.g., `abc123`, `q2$e34`), unique across the project. +- **Inputs** always arrays: `[, ""]` (type 1=shadow, 2=block+shadow, 3=block only). +- **Fields** always: `["value", null]`. +- **Broadcasts** a simple string map: `"gameOver": "gameOver"`. + +3. — Phase 3: Behaviour Collector +• **Interactions**: e.g. detect collisions with `sensing_touchingobject` → broadcast or `control_stop`. +• **Game State**: + - **Start**: `event_whenflagclicked` for setup. + - **Game Over**: + ```json + "broadcasts": {"gameOver":"gameOver"} + ``` + then `control_stop all`. + - **Scoring**: `data_changevariableby` with inputs `[2,"varId"]`. + - **Restart**: `event_whenkeypressed "r"` resets everything. + +4. — Variables, Lists, Broadcasts, Comments +• **Variables**: + ```json + "variables": {"`id123-score":["score",0]} +```` + +• **Lists**: + +```json +"lists": {"`id456-items":["items",[]]} +``` + +• **Broadcasts**: string‐to‐string map. +• **Comments**: empty unless explicitly requested. + +5. — Final Output + Emit one consolidated JSON object only. + Include `"monitors"`, `"extensions"`, `"meta"`, and **all** `objName` fields. + Ensure every block ID, input, and field strictly follows the Scratch 3.0 schema. + +**If the user provides partial JSON**, detect any missing `objName`, incorrect input types, or broadcast formats, and fill in all missing pieces using the rules above. +""" + +# Create your agent exactly as before, but passing vision_llm + tokenizer +agent = create_react_agent( + model=llm, # or your preferred model + tools=[], + prompt=SYSTEM_PROMPT +) + +# --- Serve the form --- +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + # point at the two directories + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + # if they don’t exist, just return an empty list + backdrops = [] + sprites = [] + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + """ + Build a Scratch costume dict for a given asset file. + - SVGs get hard-coded centers. + - Rasters (png/jpg) use PIL to find width/height. + """ + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + # path on disk + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + # default rotation center for SVGs + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + # for sprites, you can adjust these based on your art + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + # raster images: open and compute center + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + # fallback if PIL still fails for some reason + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- Generate endpoint --- +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description","") + backdrops = payload.get("backdrops",[]) + sprites = payload.get("sprites",[]) + + # 1) build skeleton JSON + project = { + "targets": [ + { + "isStage": True, "name":"Stage", + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + # append each sprite target + for idx,s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project["targets"].append({ + "isStage": False, + "name": s["name"], + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + print("Initial project skeleton:", json.dumps(project, indent=2)) + + # Generate a unique folder name + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + os.makedirs(project_folder, exist_ok=True) + + # Save project.json + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project, f, indent=2) + print(f"Initial project skeleton saved to {project_json_path}") + + # Copy assets directly to the project folder + # No need to create a separate 'assets' subfolder + + # Copy backdrops + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + # Destination path is now directly inside project_folder + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + print(f"Copied backdrop {b['filename']} to {project_folder}") + + # Copy sprites + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + # Destination path is now directly inside project_folder + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + print(f"Copied sprite {s['filename']} to {project_folder}") + + # 2) build a little wrapper prompt to LLM so it knows to fill in the blocks + prompt = ( + f"Below is the partially-built Scratch project JSON. " + f"Fill in all the 'blocks' sections for each target ('Stage' and sprites) " + f"with the correct opcodes and structure to create a “{desc}” game, " + f"strictly following the block generation rules (Phase 2) and behavior logic (Phase 3) " + f"outlined in your SYSTEM_PROMPT. " + f"Ensure proper parent-child relationships, sequential flow, and correct block type usage. " + f"Do not touch any of the existing asset entries.\n\n" + f"```json\n{json.dumps(project,indent=2)}\n```" + ) + + # 3) invoke our agent + response = agent.invoke({"messages":[{"role":"user","content":prompt}]}) + print("LLM response:", response) + last = response["messages"][-1] + raw = last.content if isinstance(last, AIMessage) else str(last) + + # 4) strip out JSON + m = re.search(r"```json\n(.+?)```", raw, re.S) + body = m.group(1) if m else raw + try: + filled = json.loads(body) + except json.JSONDecodeError: + return jsonify(error="LLM did not return valid JSON", raw=raw),500 + + # Save the *final* filled project JSON as project.json, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(filled, f, indent=2) + print(f"Final project JSON saved to {project_json_path}") + + + return jsonify(filled) + +if __name__=="__main__": + app.run(debug=True,port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/static/assets/backdrops/cd21514d0531fdffb22204e0ec5ed84a.svg b/scratch_VLM/scratch_agent/static/assets/backdrops/cd21514d0531fdffb22204e0ec5ed84a.svg new file mode 100644 index 0000000000000000000000000000000000000000..15f73119b9c3271c6c411fc4233090e37f42ec37 --- /dev/null +++ b/scratch_VLM/scratch_agent/static/assets/backdrops/cd21514d0531fdffb22204e0ec5ed84a.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/static/assets/backdrops/e7c147730f19d284bcd7b3f00af19bb6.svg b/scratch_VLM/scratch_agent/static/assets/backdrops/e7c147730f19d284bcd7b3f00af19bb6.svg new file mode 100644 index 0000000000000000000000000000000000000000..0326edc8bdd7da45b33bac907c15ae1c0d1026c1 --- /dev/null +++ b/scratch_VLM/scratch_agent/static/assets/backdrops/e7c147730f19d284bcd7b3f00af19bb6.svg @@ -0,0 +1,19 @@ + + + + blue sky + Created with Sketch. + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/static/assets/sounds/1727f65b5f22d151685b8e5917456a60.wav b/scratch_VLM/scratch_agent/static/assets/sounds/1727f65b5f22d151685b8e5917456a60.wav new file mode 100644 index 0000000000000000000000000000000000000000..f5be3a1606476bfbee8a4321f41de6a1a0ead520 Binary files /dev/null and b/scratch_VLM/scratch_agent/static/assets/sounds/1727f65b5f22d151685b8e5917456a60.wav differ diff --git a/scratch_VLM/scratch_agent/static/assets/sounds/83a9787d4cb6f3b7632b4ddfebf74367.wav b/scratch_VLM/scratch_agent/static/assets/sounds/83a9787d4cb6f3b7632b4ddfebf74367.wav new file mode 100644 index 0000000000000000000000000000000000000000..fc3b2724a9c7cfef378eeb65499d44236ad2add8 Binary files /dev/null and b/scratch_VLM/scratch_agent/static/assets/sounds/83a9787d4cb6f3b7632b4ddfebf74367.wav differ diff --git a/scratch_VLM/scratch_agent/static/assets/sounds/83c36d806dc92327b9e7049a565c6bff.wav b/scratch_VLM/scratch_agent/static/assets/sounds/83c36d806dc92327b9e7049a565c6bff.wav new file mode 100644 index 0000000000000000000000000000000000000000..45742d5ef6f09d05b0f0788cb055ffe54abfd9ad Binary files /dev/null and b/scratch_VLM/scratch_agent/static/assets/sounds/83c36d806dc92327b9e7049a565c6bff.wav differ diff --git a/scratch_VLM/scratch_agent/static/assets/sprites/0fb9be3e8397c983338cb71dc84d0b25.svg b/scratch_VLM/scratch_agent/static/assets/sprites/0fb9be3e8397c983338cb71dc84d0b25.svg new file mode 100644 index 0000000000000000000000000000000000000000..5ff997fd11132a3505e71ce46f5e14e34dbb4430 --- /dev/null +++ b/scratch_VLM/scratch_agent/static/assets/sprites/0fb9be3e8397c983338cb71dc84d0b25.svg @@ -0,0 +1,42 @@ + + + + costume2.1 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/static/assets/sprites/5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg b/scratch_VLM/scratch_agent/static/assets/sprites/5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg new file mode 100644 index 0000000000000000000000000000000000000000..3a3908a99eb18cdeaacf1222caec50828656871e --- /dev/null +++ b/scratch_VLM/scratch_agent/static/assets/sprites/5d973d7a3a8be3f3bd6e1cd0f73c32b5.svg @@ -0,0 +1,29 @@ + + + + Slice 1 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/static/assets/sprites/bcf454acf82e4504149f7ffe07081dbc.svg b/scratch_VLM/scratch_agent/static/assets/sprites/bcf454acf82e4504149f7ffe07081dbc.svg new file mode 100644 index 0000000000000000000000000000000000000000..03df23e29ad059d88e559d48bf5e2717870455f3 --- /dev/null +++ b/scratch_VLM/scratch_agent/static/assets/sprites/bcf454acf82e4504149f7ffe07081dbc.svg @@ -0,0 +1,42 @@ + + + + costume1.1 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/sys_scratch_2.py b/scratch_VLM/scratch_agent/sys_scratch_2.py new file mode 100644 index 0000000000000000000000000000000000000000..ab51159b54b1b6288f89824b323c4ec5c1b36d41 --- /dev/null +++ b/scratch_VLM/scratch_agent/sys_scratch_2.py @@ -0,0 +1,1199 @@ +from flask import Flask, request, jsonify, render_template, send_from_directory +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +#from langgraph.graph import draw +from langchain.chat_models import ChatOpenAI +from PIL import Image +import os, json, re +import shutil +import uuid +from langgraph.graph import StateGraph, END +import logging +from typing import Dict, TypedDict, Optional + +# --- Configure logging --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +# --- LLM / Vision model setup --- +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + logger.critical("GROQ_API_KEY environment variable is not set or invalid. Please set it to proceed.") + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +llm = ChatGroq( + #model="deepseek-r1-distill-llama-70b", + #model="llama-3.3-70b-versatile", + model="meta-llama/llama-4-maverick-17b-128e-instruct", + #model="meta-llama/llama-4-scout-17b-16e-instruct", + temperature=0.0, +) + +# llm = ChatOpenAI( +# openai_api_key=getenv("OPENROUTER_API_KEY"), +# openai_api_base=getenv("OPENROUTER_BASE_URL"), +# model_name="", +# model_kwargs={ +# "headers": { +# "HTTP-Referer": getenv("YOUR_SITE_URL"), +# "X-Title": getenv("YOUR_SITE_NAME"), +# } +# }, +# ) + +# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema. When generating or modifying the `blocks` section, pay extremely close attention to the following: + +**Scratch Target and Block Schema Rules:** +1. **Target Structure:** + * Every target (Stage or Sprite) must have an `objName` property. For the Stage, it's typically `"objName": "Stage"`. For sprites, it's their name (e.g., `"objName": "Cat"`). + * `variables` dictionary: This dictionary maps unique variable IDs to arrays `[variable_name, initial_value]`. + Example: `"variables": { "myVarId123": ["score", 0] }` + +2. **Block Structure:** Every block must have `opcode`, `parent`, `next`, `inputs`, `fields`, `shadow`, `topLevel`. + * `parent`: ID of the block it's stacked on (or `null` for top-level). + * `next`: ID of the block directly below it (or `null` for end of stack). + * `topLevel`: `true` if it's a hat block or a standalone block, `false` otherwise. + * `shadow`: `true` if it's a shadow block (e.g., a default number input), `false` otherwise. + +3. **`inputs` vs. `fields`:** This is critical. + * **`inputs`**: Used for blocks or values *plugged into* a block (e.g., condition for an `if` block, target for `go to`). Values are **arrays**. + * `[1, ]`: Represents a Reporter or Boolean block *plugged in* (e.g., `operator_equals`). + * `[1, [, ""]]`: For a shadow block with a literal value (e.g., `[1, ["num", "10"]]` for a number input). The `type` can be "num", "str", "bool", "colour", etc. + * `[2, ]`: For a C-block's substack (e.g., `SUBSTACK` in `control_if`). + * **`fields`**: Used for dropdown menus or direct internal values *within* a block (e.g., key selected in `when key pressed`, variable name in `set variable`). Values are always **arrays** `["", null]`. + * Example for `event_whenkeypressed`: `"KEY": ["space", null]` + * Example for `data_setvariableto`: `"VARIABLE": ["score", null]` + +4. **Unique IDs:** All block IDs, variable IDs, and list IDs must be unique strings (e.g., "myBlock123", "myVarId456"). Do NOT use placeholder strings like "block_id_here". + +5. **No Nested `blocks` Dictionary:** The `blocks` dictionary should only appear once per `target` (sprite/stage). Do NOT nest a `blocks` dictionary inside an individual block definition. Blocks that are part of a substack (like inside an `if` or `forever` loop) are linked via the `SUBSTACK` input, where the input value is `[2, "ID_of_first_block_in_substack"]`. + +6. **Asset Properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY` should be correct for costumes/sounds. + +**When responding, you must:** +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, field, and variable declaration strictly follows the Scratch 3.0 schema and the rules above. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" + +agent = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT +) + +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + backdrops = [] + sprites = [] + try: + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + logger.info("Successfully listed static assets.") + except Exception as e: + logger.error(f"Error listing static assets: {e}") + return jsonify({"error": "Failed to list assets"}), 500 + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + logger.warning(f"Could not determine image dimensions for {filename}: {e}. Setting center to 0,0.") + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- New endpoint to fetch project.json --- +@app.route("/get_project/", methods=["GET"]) +def get_project(project_id): + project_folder = os.path.join("generated_projects", project_id) + project_json_path = os.path.join(project_folder, "project.json") + + try: + if os.path.exists(project_json_path): + logger.info(f"Serving project.json for project ID: {project_id}") + return send_from_directory(project_folder, "project.json", as_attachment=True, download_name=f"{project_id}.json") + else: + logger.warning(f"Project JSON not found for ID: {project_id}") + return jsonify({"error": "Project not found"}), 404 + except Exception as e: + logger.error(f"Error serving project.json for ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve project"}), 500 + +# --- New endpoint to fetch assets --- +@app.route("/get_asset//", methods=["GET"]) +def get_asset(project_id, filename): + project_folder = os.path.join("generated_projects", project_id) + asset_path = os.path.join(project_folder, filename) + + try: + if os.path.exists(asset_path): + logger.info(f"Serving asset '{filename}' for project ID: {project_id}") + return send_from_directory(project_folder, filename) + else: + logger.warning(f"Asset '{filename}' not found for project ID: {project_id}") + return jsonify({"error": "Asset not found"}), 404 + except Exception as e: + logger.error(f"Error serving asset '{filename}' for project ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve asset"}), 500 + + +### LangGraph Workflow Definition + +# Define a state for your graph using TypedDict +class GameState(TypedDict): + project_json: dict + description: str + project_id: str + sprite_initial_positions: dict # Add this as well, as it's part of your state + action_plan: Optional[Dict] + behavior_plan: Optional[Dict] + +# Helper function to extract JSON from LLM response +def extract_json_from_llm_response(raw_response: str) -> dict: + m = re.search(r"```json\n(.+?)```", raw_response, re.S) + body = m.group(1) if m else raw_response + try: + json_data = json.loads(body) + logger.debug("Successfully extracted and parsed JSON from LLM response.") + return json_data + except json.JSONDecodeError as e: + logger.error(f"LLM did not return valid JSON. Error: {e}\nRaw response: {raw_response}") + raise ValueError(f"LLM did not return valid JSON: {e}\nRaw response: {raw_response}") + except Exception as e: + logger.error(f"An unexpected error occurred during JSON extraction: {e}\nRaw response: {raw_response}") + raise +# Helper function to update project JSON with sprite positions +import copy +def update_project_with_sprite_positions(project_json: dict, sprite_positions: dict) -> dict: + """ + Update the 'x' and 'y' coordinates of sprites in the Scratch project JSON. + + Args: + project_json (dict): Original Scratch project JSON. + sprite_positions (dict): Dict mapping sprite names to {'x': int, 'y': int}. + + Returns: + dict: Updated project JSON with new sprite positions. + """ + updated_project = copy.deepcopy(project_json) + + for target in updated_project.get("targets", []): + if not target.get("isStage", False): + sprite_name = target.get("name") + if sprite_name in sprite_positions: + pos = sprite_positions[sprite_name] + if "x" in pos and "y" in pos: + target["x"] = pos["x"] + target["y"] = pos["y"] + + return updated_project + +# +# --- Global variable for the block catalog --- +ALL_SCRATCH_BLOCKS_CATALOG = {} +BLOCK_CATALOG_PATH = "blocks_content.json" # Define the path to your JSON file + +# Helper function to load the block catalog from a JSON file +def _load_block_catalog(file_path: str) -> Dict: + """Loads the Scratch block catalog from a specified JSON file.""" + try: + with open(file_path, 'r') as f: + catalog = json.load(f) + logger.info(f"Successfully loaded block catalog from {file_path}") + return catalog + except FileNotFoundError: + logger.error(f"Error: Block catalog file not found at {file_path}") + # Return an empty dict or raise an error, depending on desired behavior + return {} + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON from {file_path}: {e}") + return {} + except Exception as e: + logger.error(f"An unexpected error occurred while loading {file_path}: {e}") + return {} + + +# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally +ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH) + +# Helper function to generate a unique block ID +def generate_block_id(): + """Generates a short, unique ID for a Scratch block.""" + return str(uuid.uuid4())[:10].replace('-', '') # Shorten for readability, ensure uniqueness + +# Placeholder for your extract_json_from_llm_response function +def extract_json_from_llm_response(response_string: str) -> Dict: + """ + Extracts a JSON object from an LLM response string. + Assumes the JSON is enclosed within triple backticks (```json ... ```). + """ + try: + # Find the start and end of the JSON block + start_marker = "```json" + end_marker = "```" + start_index = response_string.find(start_marker) + end_index = response_string.rfind(end_marker) + + if start_index != -1 and end_index != -1 and start_index < end_index: + # Extract the substring between the markers + json_str = response_string[start_index + len(start_marker):end_index].strip() + return json.loads(json_str) + else: + # If markers are not found, try to parse the whole string as JSON + # This handles cases where the LLM might omit markers + logger.warning("JSON markers not found in LLM response. Attempting to parse raw string.") + return json.loads(response_string.strip()) + except json.JSONDecodeError as e: + logger.error(f"Failed to decode JSON from LLM response: {e}") + logger.error(f"Raw response: {response_string}") + raise ValueError("Invalid JSON response from LLM") + except Exception as e: + logger.error(f"An unexpected error occurred in extract_json_from_llm_response: {e}") + raise + + +# Node 1: Analysis User Query and Initial Positions +def parse_query_and_set_initial_positions(state: GameState): + logger.info("--- Running ParseQueryNode ---") + + llm_query_prompt = ( + f"Based on the user's game description: '{state['description']}', " + f"and the current Scratch project JSON below, " + f"determine the most appropriate initial 'x' and 'y' coordinates for each sprite. " + f"Return ONLY a JSON object with a single key 'sprite_initial_positions' mapping sprite names to their {{'x': int, 'y': int}} coordinates.\n\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Example Json output:\n" + "```json\n" + "{\n" + " \"sprite_initial_positions\": {\n" + " \"Sprite1\": {\"x\": -160, \"y\": -110},\n" + " \"Sprite2\": {\"x\": 240, \"y\": -135}\n" + " }\n" + "}" + "```\n" + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_query_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM:", raw_response) + + updated_data = extract_json_from_llm_response(raw_response) + sprite_positions = updated_data.get("sprite_initial_positions", {}) + + new_project_json = update_project_with_sprite_positions(state["project_json"], sprite_positions) + print("Updated project JSON with sprite positions:", json.dumps(new_project_json, indent=2)) + return {"project_json": new_project_json, "sprite_initial_positions": sprite_positions} + #return {"project_json": new_project_json} + + except Exception as e: + logger.error(f"Error in ParseQueryNode: {e}") + raise + +# Node 3: Sprite Action Plan Builder (MODIFIED) +def plan_sprite_actions(state: GameState): + logger.info("--- Running PlanSpriteActionsNode ---") + + description = state.get("description","") + project_json = state["project_json"] + sprite_names = [t["name"] for t in project_json["targets"] if not t["isStage"]] + + # Get sprite positions from the project_json if available, or set defaults + sprite_positions = {} + for target in project_json["targets"]: + if not target["isStage"]: + sprite_positions[target["name"]] = {"x": target.get("x", 0), "y": target.get("y", 0)} + + planning_prompt = ( + "Generate a detailed action plan for the game's sprites based on the user query and sprite details.\n\n" + f"The game description is: '{description}'.\n\n" + f"Sprites in game: {', '.join(sprite_names)}\n" + f"Sprites current positions: {json.dumps(sprite_positions)}\n\n" # Use actual positions + "Your task is to define the primary actions and movements for each sprite. " + "The output should be a JSON object with a single key 'action_overall_flow'. " + "The value should be a dictionary where keys are sprite names (e.g., 'Player', 'Enemy') " + "and values are dictionaries containing a 'description' and a 'plans' list. " + "Each 'plan' in the list must correspond to a single Scratch Hat Block (event block). " + "For each plan, you must identify and include the following:\n" + "1. The 'event' field: the exact `opcode` of the hat block (e.g., 'event_whenflagclicked', 'event_whenkeypressed', 'control_start_as_clone', 'event_whenbroadcastreceived'). " + "2. The 'logic' field: a natural language description of all sequential actions under that single event. " + " Separate distinct actions within the 'logic' string with a semicolon ';'. " + " Each action should be as granular as possible, describing one Scratch block or a very small, tightly coupled sequence of directly connected blocks. " + " Avoid combining complex control flows or multiple distinct operations into a single action if they can be broken down naturally." + " For example, for a jump, combine: 'change y by N; wait M seconds; change y by -N'." + " For continuous movement, differentiate 'move N steps' from 'glide N seconds to x:Y y:Z'." + " When an action needs to repeat indefinitely (e.g., with a 'forever' block), explicitly state 'forever: ' at the beginning of the logic for that section, followed by the actions inside it." + "3. Explicit lists of Scratch block opcodes by category: `motion`, `control`, `operators`, `sensing`, `looks`, `sound`, `events`, `data`. " + " These lists should contain ALL relevant Scratch block `opcode`s (not just names) that would be used to implement the 'logic' for that specific plan, including shadow blocks for values or dropdowns if they are an integral part of the block's operation (e.g., `math_number` for numerical inputs, `event_whenkeypressed_keymenu` for key options). " + " **Crucially, derive these opcodes from the detailed actions described in the 'logic' string. To do this, use your comprehensive knowledge of Scratch 3.0 block functionalities and opcodes.** " # Added sentence here + " Include only the categories that have relevant blocks for that plan; omit empty lists." + "Ensure the logic clearly distinguishes between 'move', 'glide', 'go to', and 'change x/y by' as they are distinct block types." + "Refer to sprite names precisely as they appear in `sprite_names` (e.g., 'Sprite1', 'soccer ball')." + "\n\nExample structure for 'action_overall_flow' (follow this precise format, including all relevant opcode lists and ensuring correct opcode identification):\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at staring point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [\"control_if\"],\n" + " \"operators\": [],\n" + " \"sensing\": []\n" + " \"looks\": []\n" + " \"sounds\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"change y by value 10; wait 0.1 seconds; change y by value -10\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_wait\"],\n" + " \"operators\": [],\n" + " \"sensing\": []\n" + " \"looks\": []\n" + " \"sounds\": []\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135; forever: glide 2 seconds to x:-240 y:-135; if x position < -235, then set x to 240\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operators\": [\"operator_lt\"],\n" + " \"sensing\": []\n" + " \"looks\": []\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Based on the provided context, generate the `action_overall_flow`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM [PlanSpriteActionsNode]:", raw_response) # Changed to print for direct visibility + action_plan = extract_json_from_llm_response(raw_response) + state["action_plan"] = action_plan + logger.info("Sprite action plan generated by PlanSpriteActionsNode.") + return state # Return state object, not a dict + except Exception as e: + logger.error(f"Error in PlanSpriteActionsNode: {e}") + raise + +# Node 4: Action Node Builder +def build_action_nodes(state: GameState): + logger.info("--- Running ActionNodeBuilder ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + + action_plan = state.get("action_plan", {}) + if not action_plan: + logger.warning("No action plan found in state. Skipping ActionNodeBuilder.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + for sprite_name, sprite_actions_data in action_plan.get("action_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_actions_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operators_opcodes = plan_entry.get("operators", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + # Create a string representation of the identified opcodes for the prompt + identified_opcodes_str = "" + if motion_opcodes: + identified_opcodes_str += f" Motion Blocks (opcodes): {', '.join(motion_opcodes)}\n" + if control_opcodes: + identified_opcodes_str += f" Control Blocks (opcodes): {', '.join(control_opcodes)}\n" + if operators_opcodes: + identified_opcodes_str += f" Operator Blocks (opcodes): {', '.join(operators_opcodes)}\n" + if sensing_opcodes: + identified_opcodes_str += f" Sensing Blocks (opcodes): {', '.join(sensing_opcodes)}\n" + if looks_opcodes: + identified_opcodes_str += f" Looks Blocks (opcodes): {', '.join(looks_opcodes)}\n" + if sound_opcodes: + identified_opcodes_str += f" Sound Blocks (opcodes): {', '.join(sound_opcodes)}\n" + if events_opcodes: + identified_opcodes_str += f" Event Blocks (opcodes): {', '.join(events_opcodes)}\n" + if data_opcodes: + identified_opcodes_str += f" Data Blocks (opcodes): {', '.join(data_opcodes)}\n" + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is for the event with opcode '{event_opcode}'.\n" + f"The sequential logic to implement is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Based on the planning, the following Scratch block opcodes are expected to be used to implement this logic. Focus on using these specific opcodes where applicable, and refer to the ALL_SCRATCH_BLOCKS_CATALOG for their full structure and required inputs/fields:**\n" + f"{identified_opcodes_str if identified_opcodes_str else ' No specific block types pre-identified (rely on logic description and ALL_SCRATCH_BLOCKS_CATALOG).'} \n\n" + f"Here is the comprehensive catalog of all available Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(ALL_SCRATCH_BLOCKS_CATALOG, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON (VERY IMPORTANT - FOLLOW PRECISELY):**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'abc123def4').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Handle `inputs` for parameters and substacks (CRITICAL DETAIL):**\n" + f" * The value for any key within the `inputs` dictionary MUST always be an **array** of two elements: `[type_code, value_or_block_id]`.\n" + f" * **Do NOT embed an object with `shadow: true` or `fields` directly as an input value.** These properties belong to a separate, distinct shadow block definition, not within the `inputs` array.\n" + f" * The `type_code` (first element of the array) indicates the nature of the input:\n" + f" * `1`: For a primitive value or a shadow block ID (e.g., a number, string, boolean, or reference to a shadow block).\n" + f" * `2`: For a block ID where another block is directly plugged into this input (e.g., an operator block connected to an `if` condition).\n" + f" * `3`: For a variable/list reference (less common in simple scripts).\n" + f" * **For inputs that take a literal value or dropdown selection (e.g., 'STEPS' in `motion_movesteps`, 'KEY_OPTION' in `event_whenkeypressed`):** The input should always point to a `shadow: true` block via `[1, \"SHADOW_BLOCK_ID\"]`.\n" + f" * **For C-blocks (e.g., `control_forever`, `control_if`, `control_repeat`):** Use the `SUBSTACK` input. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_SUBSTACK_ID\"]`.\n" + f"5. **Define ALL Shadow Blocks Separately:** Whenever a block's input requires a number, string literal, a selection from a dropdown menu, or a broadcast message, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow.\n" + f" * Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f" * The `parent` field of the shadow block MUST point to the ID of the block that uses it as an input.\n" + f" * **Example for a number literal (e.g., '10' for `motion_movesteps`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_NUM_ID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BLOCK_ID_THAT_USES_THIS_SHADOW\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, it would be: `\"STEPS\": [1, \"UNIQUE_SHADOW_NUM_ID\"]`\n" + f" * **Example for a dropdown menu selection (e.g., 'space' key for `when key pressed`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_MENU_ID\": {{\n" + f" \"opcode\": \"event_whenkeypressed_keymenu\",\n" + f" \"fields\": {{\n" + f" \"KEY_OPTION\": [\"space\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BLOCK_ID_THAT_USES_THIS_SHADOW\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, it would be: `\"KEY_OPTION\": [1, \"UNIQUE_SHADOW_MENU_ID\"]`\n" + f" * **Example for a Broadcast Message (e.g., 'gameover' for `broadcast message`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_BROADCAST_ID\": {{\n" + f" \"opcode\": \"event_broadcast_menu\",\n" + f" \"fields\": {{\n" + f" \"BROADCAST_OPTION\": [\"gameover\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"PARENT_BROADCAST_BLOCK_ID_THAT_USES_THIS_SHADOW\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent `event_broadcast` or `event_broadcastandwait` block's `inputs`, it would be: `\"BROADCAST_INPUT\": [1, \"UNIQUE_SHADOW_BROADCAST_ID\"]`\n" + f"6. **`fields` for direct dropdown values/text (ONLY when NOT an input):** Use the `fields` dictionary ONLY IF the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., the `KEY_OPTION` field within the `event_whenkeypressed_keymenu` shadow block itself, or the `variable` field on a `data_setvariableto` block). Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Ensure Unique Block IDs:** Every block you generate (main blocks and shadow blocks) must have a unique ID within the entire script's block dictionary.\n" + f"9. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [ActionNodeBuilder - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + + generated_blocks = extract_json_from_llm_response(raw_response) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Action blocks added for sprite '{sprite_name}', script '{event_opcode}' by ActionNodeBuilder.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with action nodes.") + print("Updated project JSON with action nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + +# Node 5: Behaviour Collector (Sensing/Collision) +def plan_behaviors_node(state: GameState): + logger.info("--- Running PlanBehaviorsNode ---") + + # Get relevant context from the state + description = state.get("description","") + action_plan = state.get("action_plan", {}) + initial_positions = state.get("initial_positions", {}) # Assuming this will be populated by a prior node if needed + + # Construct the prompt with comprehensive context + planning_prompt = ( + "Generate a detailed plan for the game's behaviors based on the following context:\n\n" + f"The game description is: '{description}'.\n\n" + f"Action Plan: {json.dumps(action_plan, indent=2)}\n" + f"Initial Positions: {json.dumps(initial_positions, indent=2)}\n\n" + "Your task is to define the game's behaviors, including scoring, collision detection, " + "game state management (e.g., win/loss conditions, resets), and any necessary sprite " + "initialization (like starting positions or visibility)." + "The output should be a JSON object with a single key 'behavior_overall_flow'. " + "The value should be a dictionary where keys are sprite names (e.g., 'Stage', 'Player', 'Enemy', 'Coin') " + "and values are dictionaries containing a 'plans' list. " + "Each 'plan' in the list must correspond to a single Scratch Hat Block (event block). " + "For each plan, you must identify and include the following:\n" + "1. The 'event' field: a high-level trigger for the behavior (e.g., 'Game Initialization', 'Collision with Enemy', 'When green flag clicked'). If it's a Scratch hat block, use its exact `opcode` (e.g., 'event_whenflagclicked', 'event_whenkeypressed', 'event_whenbroadcastreceived').\n" + "2. The 'logic' field: a natural language description of all sequential actions under that single event. Separate distinct actions within the 'logic' string with a semicolon ';'.\n" + "3. Explicit lists of Scratch block opcodes by category: `motion`, `control`, `operators`, `sensing`, `looks`, `sound`, `events`, `data`. These lists should contain ALL relevant Scratch block `opcode`s that would be used to implement the 'logic' for that specific plan, including any necessary shadow block opcodes (e.g., `sensing_touchingobjectmenu` for `sensing_touchingobject`). Include empty lists for categories that have no relevant blocks.\n" + "Refer to sprite names precisely as they appear in the action plan (e.g., 'Sprite1', 'soccer ball')." + "\n\nExample structure for 'behavior_overall_flow' (follow this precise format, including all relevant opcode lists and ensuring correct opcode identification, with empty lists for unused categories):\n" + "```json\n" + "{\n" + " \"behavior_overall_flow\": {\n" + " \"Stage\": {\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"Reset score to 0; show score variable; reset timer; hide all other sprites.\",\n" + " \"motion\": [],\n" + " \"control\": [],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [\"looks_showvariable\"],\n" + " \"sound\": [],\n" + " \"events\": [],\n" + " \"data\": [\"data_setvariableto\", \"data_showvariable\"]\n" + " }\n" + " ]\n" + " },\n" + " \"Sprite1\": {\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"When green flag clicked, forever: if touching 'soccer ball', then stop all.\",\n" + " \"motion\": [],\n" + " \"control\": [\"control_forever\", \"control_if\", \"control_stop\"],\n" + " \"operators\": [],\n" + " \"sensing\": [\"sensing_touchingobject\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [],\n" + " \"sound\": [],\n" + " \"events\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenbroadcastreceived\",\n" + " \"logic\": \"When 'reset_game' is received, go to x:-160 y:-110; show.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [\"looks_show\"],\n" + " \"sound\": [],\n" + " \"events\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenbroadcastreceived\",\n" + " \"logic\": \"When 'reset_game' is received, go to x:240 y:-135; show.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operators\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [\"looks_show\"],\n" + " \"sound\": [],\n" + " \"events\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Based on the provided context, generate the `behavior_overall_flow`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [PlanBehaviorsNode]: {raw_response[:200]}...") + + behavior_plan = extract_json_from_llm_response(raw_response) + if "behavior_overall_flow" not in behavior_plan: + raise ValueError("LLM response for behavior plan missing 'behavior_overall_flow' key.") + + state["behavior_plan"] = behavior_plan + logger.info("Behavior plan generated and added to state.") + return state + except Exception as e: + logger.error(f"Error generating behavior plan: {e}") + raise + +def build_behavior_nodes(state: GameState): + logger.info("--- Running BehaviorNodeBuilder ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target # Ensure Stage is in sprite_map + + behavior_plan = state.get("behavior_plan", {}) + if not behavior_plan: + logger.warning("No behavior plan found in state. Skipping BehaviorNodeBuilder.") + return state + + # Initialize script Y offsets for each sprite + # We can assume a default starting X for all scripts in a sprite (e.g., 50) + script_x_offset = 50 + script_y_offset = {name: 50 for name in sprite_map.keys()} # Start all scripts at y=50 initially + + for sprite_name, sprite_behaviors_data in behavior_plan.get("behavior_overall_flow", {}).items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + for plan_entry in sprite_behaviors_data.get("plans", []): + event_type = plan_entry["event"] # This holds the opcode or a descriptive event + logic_description = plan_entry["logic"] + + # Now, event_type holds the relevant info, e.g., "event_whenflagclicked" + # Removed 'event_description' as it was a redundant or unused variable. + + # Extract identified opcodes from the plan_entry, if available + identified_opcodes = { + "motion": plan_entry.get("motion", []), + "control": plan_entry.get("control", []), + "operators": plan_entry.get("operators", []), + "sensing": plan_entry.get("sensing", []), + "looks": plan_entry.get("looks", []), + "sound": plan_entry.get("sound", []), + "events": plan_entry.get("events", []), + "data": plan_entry.get("data", []) + } + + # Format identified opcodes for the prompt + identified_opcodes_str = "" + for category, opcodes in identified_opcodes.items(): + if opcodes: + identified_opcodes_str += f" - {category.capitalize()}: {', '.join(opcodes)}\n" + + # Determine the actual event opcode for the prompt. + # If event_type is already an opcode (like 'event_whenflagclicked'), use it directly. + # Otherwise, infer from the description. + actual_event_opcode = event_type if event_type.startswith("event_") or event_type.startswith("control_start_") else "event_whenflagclicked" # Default if not explicitly an opcode + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is for the event with opcode '{actual_event_opcode}'.\n" + f"The sequential logic to implement is:\n" + f" Logic: {logic_description}\n\n" + f"**Based on the planning, the following Scratch block opcodes are expected to be used to implement this logic. Focus on using these specific opcodes where applicable, and refer to the ALL_SCRATCH_BLOCKS_CATALOG for their full structure and required inputs/fields:**\n" + f"{identified_opcodes_str if identified_opcodes_str else ' No specific block types pre-identified (rely on logic description and ALL_SCRATCH_BLOCKS_CATALOG).'} \n\n" + f"Here is the comprehensive catalog of all available Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(ALL_SCRATCH_BLOCKS_CATALOG, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON (VERY IMPORTANT - FOLLOW PRECISELY):**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: {script_x_offset}, y: {script_y_offset[sprite_name]}`).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'abc123def4').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Handle `inputs` for parameters and substacks (CRITICAL DETAIL - READ CAREFULLY):**\n" + f" * The value for any key within the `inputs` dictionary MUST always be an **array** of two elements: `[type_code, value_or_block_id]`.\n" + f" * **Crucially, consult the `ALL_SCRATCH_BLOCKS_CATALOG` for the specific block and input field to determine the exact `value_or_block_id` format.**\n" + f" * **Do NOT embed an object with `shadow: true` or `fields` directly as an input value.** These properties belong to a separate, distinct shadow block definition, or are part of a directly embedded literal as shown in the catalog.\n" + f" * The `type_code` (first element of the array) indicates the nature of the input:\n" + f" * `1`: For a primitive value or a shadow block ID (e.g., a number, string, boolean, or reference to a shadow block).\n" + f" * `2`: For a block ID where another block is directly plugged into this input (e.g., an operator block connected to an `if` condition).\n" + f" * `3`: For a variable/list reference (less common in simple scripts).\n" + f" * **For inputs that take a literal number (like '10' for `motion_movesteps` or `motion_changeyby`):** If the `ALL_SCRATCH_BLOCKS_CATALOG` shows the input value as `[4, \"NUMBER_STRING\"]` (e.g., `\"STEPS\": [1, [4, \"10\"]]`), then embed it directly in that format. DO NOT create a separate `math_number` shadow block unless `math_number` is explicitly listed in your `ALL_SCRATCH_BLOCKS_CATALOG` as an opcode for literal numerical values, and the parent block's input points to it as a shadow.\n" + f" * **For inputs that take a literal string or dropdown selection (e.g., 'What's your name?' for `sensing_askandwait` or 'space' for `event_whenkeypressed`):** These are usually handled in one of two ways based on the `ALL_SCRATCH_BLOCKS_CATALOG`:\n" + f" * **Directly in `inputs`:** If the catalog shows `[1, [10, \"STRING_VALUE\"]]` for a string or `[1, [11, \"STRING_VALUE\", \"BROADCAST_ID\"]]` for a broadcast menu, embed it directly.\n" + f" * **Via a separate shadow block:** If the catalog shows an input like `\"TO\": [1, \"SHADOW_BLOCK_ID\"]` and there's a corresponding shadow block like `motion_goto_menu` (which exists in your catalog), then create that separate shadow block and reference its ID. Also, if a sensing menu (like `sensing_touchingobjectmenu` or `sensing_keyoptions`) is required, define it as a separate shadow block.\n" + f" * **For C-blocks (e.g., `control_forever`, `control_if`, `control_repeat`):** Use the `SUBSTACK` input. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_SUBSTACK_ID\"]`.\n" + f"5. **Define ALL Necessary Shadow Blocks Separately (ONLY when explicitly required by the parent block's input structure in the catalog):** Whenever a block's input requires a dropdown menu selection or a broadcast message *and the catalog shows its input pointing to a shadow block ID*, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow.\n" + f" * Each such shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f" * The `parent` field of the shadow block MUST point to the ID of the block that uses it as an input.\n" + f" * **Example for a dropdown menu selection (e.g., 'space' key for `when key pressed` if its `KEY_OPTION` input pointed to a shadow, or `motion_pointtowards_menu` for `motion_pointtowards`):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_MENU_ID\": {{\n" + f" \"opcode\": \"sensing_keyoptions\", // Or other appropriate menu opcode from catalog like event_whenkeypressed_keymenu\n" + f" \"fields\": {{\n" + f" \"KEY_OPTION\": [\"space\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + " \"parent\": \"PARENT_BLOCK_ID_THAT_USES_THIS_SHADOW\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent block's `inputs`, it would be: `\"KEY_OPTION\": [1, \"UNIQUE_SHADOW_MENU_ID\"]` (only if the parent's input expects a shadow ID).\n" + f" * **Example for a Broadcast Message (e.g., 'gameover' for `broadcast message` if it used a shadow):**\n" + f" ```json\n" + f" \"UNIQUE_SHADOW_BROADCAST_ID\": {{\n" + f" \"opcode\": \"event_broadcast_menu\",\n" + f" \"fields\": {{\n" + f" \"BROADCAST_OPTION\": [\"gameover\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + " \"parent\": \"PARENT_BROADCAST_BLOCK_ID_THAT_USES_THIS_SHADOW\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent `event_broadcast` or `event_broadcastandwait` block's `inputs`, it would be: `\"BROADCAST_INPUT\": [1, \"UNIQUE_SHADOW_BROADCAST_ID\"]` (only if the parent's input expects a shadow ID).\n" + f" * **Example for a Touching Object Menu (`sensing_touchingobjectmenu`):**\n" + f" ```json\n" + f" \"TOUCHING_OBJECT_MENU_ID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\",\n" + f" \"fields\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [\"_mouse_\", null] // or \"Sprite1\", \"soccer ball\", etc.\n" + f" }},\n" + f" \"shadow\": true,\n" + " \"parent\": \"PARENT_TOUCHING_BLOCK_ID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" Then, in the parent `sensing_touchingobject` block's `inputs`, it would be: `\"TOUCHINGOBJECTMENU\": [1, \"TOUCHING_OBJECT_MENU_ID\"]`.\n" + f"6. **`fields` for direct dropdown values/text (ONLY when NOT an input, or within a shadow block as defined in the catalog):** Use the `fields` dictionary ONLY IF the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., the `KEY_OPTION` field within the `event_whenkeypressed` hat block itself, or the `variable` field on a `data_setvariableto` block). Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Ensure Unique Block IDs:** Every block you generate (main blocks and shadow blocks) must have a unique ID within the entire script's block dictionary.\n" + f"9. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + print("prompt sent to LLM [BehaviorNodeBuilder]:", llm_block_generation_prompt) # Print the first 500 characters of the prompt for debugging + raw_response = response["messages"][-1].content + # Use actual_event_opcode for logging, as it's the more precise identifier for the script + print(f"Raw response from LLM [BehaviorNodeBuilder - {sprite_name} - {actual_event_opcode}]:", raw_response) + + generated_blocks = extract_json_from_llm_response(raw_response) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset # Use the fixed X offset + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment Y for the next script within this sprite + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Behavior blocks added for sprite '{sprite_name}', script '{actual_event_opcode}' by BehaviorNodeBuilder.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{actual_event_opcode}': {e}") + raise + + logger.info("Updated project JSON with behavior nodes.") + print("Updated project JSON with behavior nodes:", json.dumps(project_json, indent=2)) + state["project_json"] = project_json # Update the state with the modified project_json + return state + +# Node 4: Scoring Function +def add_scoring_function(state: GameState): + logger.info("--- Running ScoringNode ---") + # if not state['parsed_game_elements'].get("scoring_goals"): + # logger.info("No scoring goals identified, skipping scoring node.") + # return {} # Return an empty dictionary if no changes are made + + llm_scoring_prompt = ( + f"Based on the game description '{state['description']}' " + f"add a scoring mechanism to the Scratch project JSON. " + f"**Crucially, add a 'score' variable to the Stage's 'variables' dictionary.** The format for variables in `targets[].variables` is " + f"a dictionary where keys are unique variable IDs (e.g., 'scoreVarId123') and values are arrays `[\"variable_name\", initial_value]`. " + f"Example: `\"variables\": {{ \"scoreVarId123\": [\"score\", 0] }}`.\n" + f"Then, add blocks to update it (e.g., 'data_setvariableto', 'data_changevariableby') " + f"based on positive actions identified in 'scoring_goals' (e.g., jumping over an obstacle). " + f"**Strictly adhere to the Scratch 3.0 block schema rules for `inputs` and `fields`:**\n" + f"- For `data_setvariableto` or `data_changevariableby`, the variable name is a `field`: `\"VARIABLE\": [\"score\", null]`.\n" + f"- The value for `data_setvariableto` or `data_changevariableby` is an `input` linking to a shadow block: `\"VALUE\": [1, [\"num\", \"0\"]]` or `\"VALUE\": [1, [\"num\", \"10\"]]`.\n" + f"Ensure all block IDs are unique and relationships are correct. No nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_scoring_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Scoring function added by ScoringNode.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ScoringNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ScoringNode: {e}") + raise + +# Node 5: Reset Option +def add_reset_option(state: GameState): + logger.info("--- Running ResetNode ---") + # if not state['parsed_game_elements'].get("reset_triggers"): + # logger.info("No reset triggers identified, skipping reset node.") + # return {} # Return an empty dictionary if no changes are made + + llm_reset_prompt = ( + f"Based on the game description '{state['description']}'" + f"add a reset mechanism to the Scratch project JSON. " + f"This should typically involve an 'event_whenkeypressed' (e.g., 'r' key) or 'event_whenflagclicked' that resets sprite positions ('motion_gotoxy') " + f"and any variables (like score) to their initial states, potentially broadcasting a 'reset_game' message. " + f"**Ensure variable IDs and names are consistent with those in the Stage's 'variables' dictionary.**\n" + f"**Strictly adhere to the Scratch 3.0 block schema rules provided in the SYSTEM_PROMPT for `inputs` and `fields`:**\n" + f"- For `event_whenkeypressed`, the key is a `field`: `\"KEY\": [\"r\", null]`.\n" + f"- For `motion_gotoxy`, `X` and `Y` are `inputs` that link to number reporter shadow blocks: `\"X\": [1, [\"num\", \"-160\"]], \"Y\": [1, [\"num\", \"-110\"]]`.\n" + f"- For `data_setvariableto`, the variable name is a `field`: `\"VARIABLE\": [\"score\", null]`, and value is an `input` like `\"VALUE\": [1, [\"num\", \"0\"]]`.\n" + f"- For `event_broadcast`, the message is a `field`: `\"BROADCAST_OPTION\": [\"reset_game\", null]`.\n" + f"- Ensure all block IDs are unique and relationships are correct. No nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_reset_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Reset option added by ResetNode.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ResetNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ResetNode: {e}") + raise + +# Node 6: JSON Validation (and optional Refinement) +def validate_json(state: GameState): + logger.info("--- Running JSONValidatorNode ---") + validation_errors = [] + + try: + # Basic check if it's a valid JSON structure at least + json.dumps(state['project_json']) + + for i, target in enumerate(state['project_json'].get("targets", [])): + target_name = target.get('name', f'index {i}') + # Validate objName presence for all targets + if "objName" not in target: + validation_errors.append(f"Target '{target_name}' is missing 'objName'.") + + # Validate variables structure + if "variables" in target and isinstance(target["variables"], dict): + for var_id, var_data in target["variables"].items(): + if not isinstance(var_data, list) or len(var_data) != 2 or not isinstance(var_data[0], str): + validation_errors.append(f"Target '{target_name}' variable '{var_id}' has incorrect format: {var_data}. Expected [\"name\", initial_value].") + else: + validation_errors.append(f"Target '{target_name}' has missing or malformed 'variables' section.") + + + block_ids = set() + for block_id, block_data in target.get("blocks", {}).items(): + if block_id in block_ids: + validation_errors.append(f"Duplicate block ID found: '{block_id}' in target '{target_name}'.") + block_ids.add(block_id) + + # Validate inputs + for input_name, input_data in block_data.get("inputs", {}).items(): + if not isinstance(input_data, list): + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has non-list input '{input_name}': {input_data}. Should be a list.") + continue + + if len(input_data) < 2: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has malformed input '{input_name}': {input_data}. Expected at least 2 elements.") + continue + + if input_data[0] in [1, 2, 3] and isinstance(input_data[1], str) and input_data[1] not in block_ids and input_data[1] != "": + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent input block '{input_data[1]}' for input '{input_name}'.") + + # Validate fields + for field_name, field_data in block_data.get("fields", {}).items(): + if not isinstance(field_data, list) or len(field_data) < 1 or (len(field_data) > 1 and field_data[1] is not None): + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has malformed field '{field_name}': {field_data}. Should be [\"value\", null].") + + # Validate parent + if "parent" in block_data and block_data["parent"] is not None and block_data["parent"] not in block_ids: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent parent '{block_data['parent']}'.") + + # Validate next + if "next" in block_data and block_data["next"] is not None and block_data["next"] not in block_ids: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent next block '{block_data['next']}'.") + + # Check for nested 'blocks' (common LLM error) + if "blocks" in block_data and isinstance(block_data["blocks"], dict) and len(block_data["blocks"]) > 0: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has a nested 'blocks' dictionary. Blocks should only be at the target's top level and linked via inputs (e.g., SUBSTACK).") + + + except json.JSONDecodeError as e: + validation_errors.append(f"Structural JSON error (invalid format) during validation: {e}") + except Exception as e: + validation_errors.append(f"An unexpected error occurred during JSON validation: {e}") + + if validation_errors: + error_message = "; ".join(validation_errors) + logger.warning(f"JSON validation failed: {error_message}. Attempting refinement...") + + llm_refine_prompt = ( + f"The Scratch project JSON you generated failed validation due to the following issues: {error_message}. " + f"Please review and correct the JSON, ensuring all rules from your SYSTEM_PROMPT are followed, " + f"especially regarding unique block IDs, correct structure (parent/next/inputs/fields), and valid opcodes. " + f"Pay extremely close attention to the **exact array format for inputs and fields** (e.g., `\"FIELD_NAME\": [\"value\", null]` and `\"INPUT_NAME\": [1, [\"num\", \"value\"]]` or `[2, \"linked_block_id\"]`). " + f"Crucially, ensure the `variables` section within each target is formatted as a dictionary where keys are unique IDs and values are `[\"variable_name\", initial_value]`. " + f"Also, ensure no `blocks` dictionary is nested inside a block definition; use `SUBSTACK` input for C-blocks. " + f"The current problematic JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **corrected complete JSON**." + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_refine_prompt}]}) + raw_response = response["messages"][-1].content + refined_project_json = extract_json_from_llm_response(raw_response) + logger.info("JSON refined and updated by JSONValidatorNode.") + return {"project_json": refined_project_json} + except ValueError as e_refine: + logger.error(f"Refinement also failed to return valid JSON: {e_refine}") + raise ValueError(f"JSON validation and refinement failed: {e_refine}") + except Exception as e_refine: + logger.error(f"An unexpected error occurred during JSON refinement: {e_refine}") + raise ValueError(f"JSON validation and refinement failed due to unexpected error: {e_refine}") + else: + logger.info("JSON validation successful (all checks passed).") + return {} # Return an empty dictionary if no changes are needed or made in this pass + +# Build the LangGraph workflow +workflow = StateGraph(GameState) + +workflow.add_node("parse_query", parse_query_and_set_initial_positions) +workflow.add_node("plan_actions", plan_sprite_actions) +workflow.add_node("build_actions", build_action_nodes) +workflow.add_node("plan_behaviors", plan_behaviors_node) +workflow.add_node("build_behaviors", build_behavior_nodes) +workflow.add_node("add_scoring", add_scoring_function) +workflow.add_node("add_reset", add_reset_option) +workflow.add_node("validate_json", validate_json) + +# Define the flow (edges) +workflow.set_entry_point("parse_query") +workflow.add_edge("parse_query", "plan_actions") +workflow.add_edge("plan_actions", "build_actions") +workflow.add_edge("build_actions", "plan_behaviors") +workflow.add_edge("plan_behaviors", "build_behaviors") +workflow.add_edge("build_behaviors", "add_scoring") +workflow.add_edge("add_scoring", "add_reset") +workflow.add_edge("add_reset", "validate_json") +workflow.add_edge("validate_json", END) # End of the main flow + +app_graph = workflow.compile() + +from IPython.display import Image, display +png_bytes = app_graph.get_graph().draw_mermaid_png() +with open("langgraph_workflow.png", "wb") as f: + f.write(png_bytes) + +### Modified `generate_game` Endpoint + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description", "") + backdrops = payload.get("backdrops", []) + sprites = payload.get("sprites", []) + + logger.info(f"Starting game generation for description: '{desc}'") + + # 1) Initial skeleton generation + project_skeleton = { + "targets": [ + { + "isStage": True, + "name":"Stage", + "objName": "Stage", # ADDED THIS FOR THE STAGE + "variables":{}, # This must be a dict: { "var_id": ["var_name", value] } + "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + for idx, s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project_skeleton["targets"].append({ + "isStage": False, + "name": s["name"], + "objName": s["name"], # objName is also important for sprites + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + logger.info("Initial project skeleton created.") + + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + + try: + os.makedirs(project_folder, exist_ok=True) + # Save initial skeleton and copy assets + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project_skeleton, f, indent=2) + logger.info(f"Initial project skeleton saved to {project_json_path}") + + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source backdrop asset not found: {src_path}") + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source sprite asset not found: {src_path}") + logger.info("Assets copied to project folder.") + + # Initialize the state for LangGraph as a dictionary matching the TypedDict structure + initial_state_dict: GameState = { # Type hint for clarity + "project_json": project_skeleton, + "description": desc, + "project_id": project_id, + "sprite_initial_positions": {}, + "action_plan": {}, + "behavior_plan": {}, + } + + # Run the LangGraph workflow with the dictionary input. + final_state_dict: GameState = app_graph.invoke(initial_state_dict) + + # Access elements from the final_state_dict + final_project_json = final_state_dict['project_json'] + + # Save the *final* filled project JSON, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(final_project_json, f, indent=2) + logger.info(f"Final project JSON saved to {project_json_path}") + + return jsonify({"message": "Game generated successfully", "project_id": project_id}) + + except Exception as e: + logger.error(f"Error during game generation workflow for project ID {project_id}: {e}", exc_info=True) + return jsonify(error=f"Error generating game: {e}"), 500 + + +if __name__=="__main__": + logger.info("Starting Flask application...") + app.run(debug=True,port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/sys_scratch_3.py b/scratch_VLM/scratch_agent/sys_scratch_3.py new file mode 100644 index 0000000000000000000000000000000000000000..239a139d2e8dca2e42f5507e95980ae1cff411cf --- /dev/null +++ b/scratch_VLM/scratch_agent/sys_scratch_3.py @@ -0,0 +1,2061 @@ +from flask import Flask, request, jsonify, render_template, send_from_directory +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +#from langgraph.graph import draw +from langchain.chat_models import ChatOpenAI +from PIL import Image +import os, json, re +import shutil +import uuid +from langgraph.graph import StateGraph, END +import logging +from typing import Dict, TypedDict, Optional, Any, List + +# --- Configure logging --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +# --- LLM / Vision model setup --- +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") +os.environ["OPENROUTER_API_KEY"] = os.getenv("OPENROUTER_API_KEY", "default_key_or_placeholder") +os.environ["OPENROUTER_BASE_URL"] = os.getenv("OPENROUTER_BASE_URL", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + logger.critical("GROQ_API_KEY environment variable is not set or invalid. Please set it to proceed.") + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +#Main LLM for the SCRATCH 3.0 Agent +llm = ChatGroq( + #model="deepseek-r1-distill-llama-70b", + #model="llama-3.3-70b-versatile", + #model="meta-llama/llama-4-maverick-17b-128e-instruct", + model="meta-llama/llama-4-scout-17b-16e-instruct", + temperature=0.0, +) + +# Json debugger [temporary] +llm2 = ChatGroq( + model="deepseek-r1-distill-llama-70b", + #model="llama-3.3-70b-versatile", + #model="meta-llama/llama-4-maverick-17b-128e-instruct", + #model="meta-llama/llama-4-scout-17b-16e-instruct", + temperature=0.0, +) + +# llm = ChatOpenAI( +# openai_api_key=os.environ["OPENROUTER_API_KEY"], +# openai_api_base=os.environ["OPENROUTER_BASE_URL"], +# model_name="deepseek/deepseek-r1-0528:free", +# model_kwargs={ +# "headers": { +# "HTTP-Referer": "https://localhost:5000/", +# "X-Title": "agent_scratch", +# } +# }, +#) + + +# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema, informed by comprehensive reference materials. When generating or modifying the `blocks` section, pay extremely close attention to the following: + +**Scratch Target and Block Schema Rules:** +1. **Target Structure:** + * Every target (Stage or Sprite) must have an `objName` property. For the Stage, it's typically `"objName": "Stage"`. For sprites, it's their name (e.g., `"objName": "Cat"`). + * `variables` dictionary: This dictionary maps unique variable IDs to arrays `[variable_name, initial_value]`. + Example: `"variables": { "myVarId123": ["score", 0] }` + +2. **Block Structure:** Every block must have `opcode`, `blockType`, `parent`, `next`, `inputs`, `fields`, `shadow`, `topLevel`. + * `opcode`: Unique internal identifier for the block's specific functionality (e.g., `"motion_movesteps"`)[cite: 439, 452]. + * `blockType`: Specifies the visual shape and general behavior. Common types include `"command"` (Stack block), `"reporter"` (Reporter block), `"Boolean"` (Boolean block), and `"hat"` (Hat block)[cite: 453, 454]. + * `parent`: ID of the block it's stacked on (or `null` for top-level). + * `next`: ID of the block directly below it (or `null` for end of stack). + * `topLevel`: `true` if it's a hat block or a standalone block, `false` otherwise. + * `shadow`: `true` if it's a shadow block (e.g., a default number input), `false` otherwise. + +3. **`inputs` vs. `fields`:** This is critical. + * **`inputs`**: Used for blocks or values *plugged into* a block (e.g., condition for an `if` block, target for `go to`). Values are **arrays**. + * `[1, ]`: Represents a Reporter or Boolean block *plugged in* (e.g., `operator_equals`). + * `[1, [, ""]]`: For a shadow block with a literal value. The `type` specifies the data type and can be "num" (number), "str" (string), "bool" (Boolean), "colour", "angle", "date", "dropdown", "iconmenu", or "variable"[cite: 457, 461]. + Example: `[1, ["num", "10"]]` for a number input. + * `[2, ]`: For a C-block's substack (e.g., `SUBSTACK` in `control_if`). + * **`fields`**: Used for dropdown menus or direct internal values *within* a block (e.g., key selected in `when key pressed`, variable name in `set variable`). Values are always **arrays** `["", null]`. + * Example for `event_whenkeypressed`: `"KEY": ["space", null]` + * Example for `data_setvariableto`: `"VARIABLE": ["score", null]` + +4. **Unique IDs:** All block IDs, variable IDs, and list IDs must be unique strings (e.g., "myBlock123", "myVarId456"). Do NOT use placeholder strings like "block_id_here". + +5. **No Nested `blocks` Dictionary:** The `blocks` dictionary should only appear once per `target` (sprite/stage). Do NOT nest a `blocks` dictionary inside an individual block definition. Blocks that are part of a substack (like inside an `if` or `forever` or `repeat` loop and other) are linked via the `SUBSTACK` input, where the input value is `[2, "ID_of_first_block_in_substack"]`. + +6. **Asset Properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY` should be correct for costumes/sounds. + +**General Principles and Important Considerations:** +* **Backward Compatibility:** Adhere strictly to existing Scratch 3.0 opcodes and schema to ensure backward compatibility with older projects[cite: 439, 440, 446]. +* **Forgiving Inputs:** Recognize that Scratch is designed to be "forgiving in its interpretation of inputs"[cite: 442, 459]. While generating valid JSON, understand that the Scratch VM handles potentially "invalid" inputs gracefully (e.g., type coercion, returning default values like zero or empty strings, rather than crashing)[cite: 443, 459]. This implies that precise type matching for inputs might be handled internally by Scratch, allowing for some flexibility in how values are provided, but the agent should aim for the most common and logical type. + +**When responding, you must:** +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, field, and variable declaration strictly follows the Scratch 3.0 schema and the rules above. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" + +SYSTEM_PROMPT_JSON_CORRECTOR =""" +You are an assistant that outputs JSON responses strictly following the given schema. +If the JSON you produce has any formatting errors, missing required fields, or invalid structure, you must identify the problems and correct them. +Always return only valid JSON that fully conforms to the schema below, enclosed in triple backticks (```), without any extra text or explanation. + +If you receive an invalid or incomplete JSON response, fix it by: +- Adding any missing required fields with appropriate values. +- Correcting syntax errors such as missing commas, brackets, or quotes. +- Ensuring the JSON structure matches the schema exactly. + +Remember: Your output must be valid JSON only, ready to be parsed without errors. +""" + + +# Main agent of the system agent for Scratch 3.0 +agent = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT +) + +# debugger and resolver agent for Scratch 3.0 +agent_json_resolver = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT_JSON_CORRECTOR +) + +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + backdrops = [] + sprites = [] + try: + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + logger.info("Successfully listed static assets.") + except Exception as e: + logger.error(f"Error listing static assets: {e}") + return jsonify({"error": "Failed to list assets"}), 500 + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + logger.warning(f"Could not determine image dimensions for {filename}: {e}. Setting center to 0,0.") + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- New endpoint to fetch project.json --- +@app.route("/get_project/", methods=["GET"]) +def get_project(project_id): + project_folder = os.path.join("generated_projects", project_id) + project_json_path = os.path.join(project_folder, "project.json") + + try: + if os.path.exists(project_json_path): + logger.info(f"Serving project.json for project ID: {project_id}") + return send_from_directory(project_folder, "project.json", as_attachment=True, download_name=f"{project_id}.json") + else: + logger.warning(f"Project JSON not found for ID: {project_id}") + return jsonify({"error": "Project not found"}), 404 + except Exception as e: + logger.error(f"Error serving project.json for ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve project"}), 500 + +# --- New endpoint to fetch assets --- +@app.route("/get_asset//", methods=["GET"]) +def get_asset(project_id, filename): + project_folder = os.path.join("generated_projects", project_id) + asset_path = os.path.join(project_folder, filename) + + try: + if os.path.exists(asset_path): + logger.info(f"Serving asset '{filename}' for project ID: {project_id}") + return send_from_directory(project_folder, filename) + else: + logger.warning(f"Asset '{filename}' not found for project ID: {project_id}") + return jsonify({"error": "Asset not found"}), 404 + except Exception as e: + logger.error(f"Error serving asset '{filename}' for project ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve asset"}), 500 + + +### LangGraph Workflow Definition + +# Define a state for your graph using TypedDict +class GameState(TypedDict): + project_json: dict + description: str + project_id: str + sprite_initial_positions: dict # Add this as well, as it's part of your state + action_plan: Optional[Dict] + #behavior_plan: Optional[Dict] + improvement_plan: Optional[Dict] + needs_improvement: bool + plan_validation_feedback: Optional[Dict] + iteration_count: int # Track the number of iterations for improvements + review_block_feedback: Optional[Dict] # Feedback from the agent on the blocks after verification + +# Helper function to update project JSON with sprite positions +import copy +def update_project_with_sprite_positions(project_json: dict, sprite_positions: dict) -> dict: + """ + Update the 'x' and 'y' coordinates of sprites in the Scratch project JSON. + + Args: + project_json (dict): Original Scratch project JSON. + sprite_positions (dict): Dict mapping sprite names to {'x': int, 'y': int}. + + Returns: + dict: Updated project JSON with new sprite positions. + """ + updated_project = copy.deepcopy(project_json) + + for target in updated_project.get("targets", []): + if not target.get("isStage", False): + sprite_name = target.get("name") + if sprite_name in sprite_positions: + pos = sprite_positions[sprite_name] + if "x" in pos and "y" in pos: + target["x"] = pos["x"] + target["y"] = pos["y"] + + return updated_project + +# Helper function to load the block catalog from a JSON file +def _load_block_catalog(file_path: str) -> Dict: + """Loads the Scratch block catalog from a specified JSON file.""" + try: + with open(file_path, 'r') as f: + catalog = json.load(f) + logger.info(f"Successfully loaded block catalog from {file_path}") + return catalog + except FileNotFoundError: + logger.error(f"Error: Block catalog file not found at {file_path}") + # Return an empty dict or raise an error, depending on desired behavior + return {} + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON from {file_path}: {e}") + return {} + except Exception as e: + logger.error(f"An unexpected error occurred while loading {file_path}: {e}") + return {} + +# --- Global variable for the block catalog --- +ALL_SCRATCH_BLOCKS_CATALOG = {} +BLOCK_CATALOG_PATH = r"blocks\blocks.json" # Define the path to your JSON file +HAT_BLOCKS_PATH = r"blocks\hat_blocks.json" # Path to the hat blocks JSON file +STACK_BLOCKS_PATH = r"blocks\stack_blocks.json" # Path to the stack blocks JSON file +REPORTER_BLOCKS_PATH = r"blocks\reporter_blocks.json" # Path to the reporter blocks JSON file +BOOLEAN_BLOCKS_PATH = r"blocks\boolean_blocks.json" # Path to the boolean blocks JSON file +C_BLOCKS_PATH = r"blocks\c_blocks.json" # Path to the C blocks JSON file +CAP_BLOCKS_PATH = r"blocks\cap_blocks.json" # Path to the cap blocks JSON file + +# Load the block catalogs from their respective JSON files +hat_block_data = _load_block_catalog(HAT_BLOCKS_PATH) +hat_description = hat_block_data["description"] +hat_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in hat_block_data["blocks"]]) +print("Hat blocks loaded successfully.", hat_description) +boolean_block_data = _load_block_catalog(BOOLEAN_BLOCKS_PATH) +boolean_description = boolean_block_data["description"] +boolean_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in boolean_block_data["blocks"]]) + +c_block_data = _load_block_catalog(C_BLOCKS_PATH) +c_description = c_block_data["description"] +c_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in c_block_data["blocks"]]) + +cap_block_data = _load_block_catalog(CAP_BLOCKS_PATH) +cap_description = cap_block_data["description"] +cap_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in cap_block_data["blocks"]]) + +reporter_block_data = _load_block_catalog(REPORTER_BLOCKS_PATH) +reporter_description = reporter_block_data["description"] +reporter_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in reporter_block_data["blocks"]]) + +stack_block_data = _load_block_catalog(STACK_BLOCKS_PATH) +stack_description = stack_block_data["description"] +stack_opcodes_functionalities = "\n".join([f" - Opcode: {block['op_code']}, functionality: {block['functionality']}" for block in stack_block_data["blocks"]]) + +# This makes ALL_SCRATCH_BLOCKS_CATALOG available globally +ALL_SCRATCH_BLOCKS_CATALOG = _load_block_catalog(BLOCK_CATALOG_PATH) + +# Helper function to generate a unique block ID +def generate_block_id(): + """Generates a short, unique ID for a Scratch block.""" + return str(uuid.uuid4())[:10].replace('-', '') # Shorten for readability, ensure uniqueness + +# Placeholder for your extract_json_from_llm_response function +# # Helper function to extract JSON from LLM response +# def extract_json_from_llm_response(raw_response: str) -> dict: +# """ +# Extracts a JSON object from an LLM response string, robustly handling +# literal newlines inside string values by escaping them. +# """ +# # 1) Pull out the inner JSON block (```json … ```) +# json_block = raw_response +# m = re.search(r"```json\s*(\{.*\})\s*```", raw_response, re.DOTALL) +# if m: +# json_block = m.group(1) + +# # 2) Find the feedback field and escape its newlines +# # This pattern captures everything between "feedback": " … " +# feedback_pattern = r'("feedback"\s*:\s*")(.+?)("\s*,)' +# def _escape_feedback(m): +# prefix, val, suffix = m.groups() +# # replace each literal newline and carriage return with \n +# safe_val = val.replace("\r", "\\r").replace("\n", "\\n") +# return prefix + safe_val + suffix + +# json_sanitized = re.sub(feedback_pattern, _escape_feedback, json_block, flags=re.DOTALL) + +# # 3) Now try loading +# try: +# return json.loads(json_sanitized) +# except json.JSONDecodeError as e: +# logger.error(f"Failed to parse sanitized JSON: {e!r}\nContent was:\n{json_sanitized}") +# raise + +def extract_json_from_llm_response(raw_response: str) -> dict: + """ + Extracts a JSON object from an LLM response string, robustly handling + various common LLM output formats and parsing errors. + """ + json_string_to_parse = raw_response + + # --- Step 1: Try to extract JSON from markdown code block (most common and cleanest) --- + # This pattern is more robust: it matches "```json" or "```" (for general code blocks) + # and captures everything until the next "```" + markdown_match = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", raw_response) + if markdown_match: + json_string_to_parse = markdown_match.group(1).strip() + logger.debug("Extracted potential JSON from markdown block.") + else: + logger.debug("No markdown JSON block found. Attempting to parse raw response.") + + # --- Step 2: Pre-process for common LLM JSON generation errors --- + # 2.1) Escape newlines/carriage returns in string values + # This is a general fix for all string values, not just 'feedback' + # It looks for "key": "value" patterns and replaces newlines within 'value' + # This might be tricky to get right for all cases without a full parser. + # A safer bet is to hope the LLM formats strings correctly, or to rely on direct JSON parsing. + # For now, let's keep your feedback-specific one if you find LLM adds newlines there, + # but the primary error is structural. + + # Re-apply feedback escaping if you still suspect newlines in specific fields + feedback_pattern = r'("feedback"\s*:\s*")(.+?)("(?:\s*,|\s*\})?)' # Adjusted regex to handle end of object/array + def _escape_feedback(m): + prefix, val, suffix = m.groups() + safe_val = val.replace("\r", "\\r").replace("\n", "\\n") + return prefix + safe_val + suffix + json_string_to_parse = re.sub(feedback_pattern, _escape_feedback, json_string_to_parse, flags=re.DOTALL) + logger.debug("Applied feedback newline escaping.") + + + # --- Step 3: Attempt to parse the (sanitized) JSON string --- + try: + parsed_json = json.loads(json_string_to_parse) + logger.info("Successfully parsed JSON from LLM response.") + return parsed_json + except json.JSONDecodeError as original_error: + # If parsing fails, log the problematic string and the error for debugging + logger.error(f"Failed to parse JSON. Error: {original_error!r}") + logger.error(f"Problematic JSON string (start):\n{json_string_to_parse[:1000]}...") # Log first 1000 chars + logger.error(f"Problematic JSON string (full length: {len(json_string_to_parse)}):\n{json_string_to_parse}") # Log full for deep dive + + # Attempt a fallback for common structural issues (e.g., extra trailing commas) + # This is speculative and may not always work, but worth a try before failing. + try: + # Remove trailing commas from objects and arrays + # This regex is simplified and might not catch all cases, but handles common ones + # For example, {"a": 1,} -> {"a": 1} + # Or [1, 2,] -> [1, 2] + sanitized_for_trailing_commas = re.sub(r',\s*([}\]])', r'\1', json_string_to_parse) + logger.warning("Attempting to parse after removing potential trailing commas.") + return json.loads(sanitized_for_trailing_commas) + except json.JSONDecodeError as e_fallback: + logger.error(f"Fallback parsing also failed. Error: {e_fallback!r}") + # The original error is more informative for 'Expecting ,' delimiter, so raise that. + raise original_error # Re-raise the original, more specific JSONDecodeError + +# Node 1: Detailed description generator. +def game_description_node(state: GameState): + """ + Generates a detailed narrative description of the game based on the initial query. + """ + logger.info("--- Running GameDescriptionNode ---") + sprite_name = {} + initial_description = state.get("description", "A simple game.") + project_json = state["project_json"] + for target in project_json["targets"]: + sprite_name[target["name"]] = target["name"] + + description_prompt = ( + f"You are an AI assistant tasked with generating a detailed narrative description for a Scratch 3.0 game.\n" + f"The initial high-level description is: '{initial_description}'.\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Make sure you donot change Sprite and Stage name. Here are all the name: {sprite_name} \n" + f"Create a rich, engaging, and detailed description, including potential gameplay elements, objectives, and overall feel with the available resources\n" + f"The output should be a plain text description." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": description_prompt}]}) + detailed_game_description = response["messages"][-1].content + state["description"] = detailed_game_description + logger.info("Detailed game description generated by GameDescriptionNode.") + print(f"Detailed Game Description: {detailed_game_description}") + return state + except Exception as e: + logger.error(f"Error in GameDescriptionNode: {e}") + raise + +# Node 2: Analysis User Query and Initial Positions +def parse_query_and_set_initial_positions(state: GameState): + logger.info("--- Running ParseQueryNode ---") + + llm_query_prompt = ( + f"Based on the user's game description: '{state['description']}', " + f"and the current Scratch project JSON below, " + f"determine the most appropriate initial 'x' and 'y' coordinates for each sprite. " + f"Return ONLY a JSON object with a single key 'sprite_initial_positions' mapping sprite names to their {{'x': int, 'y': int}} coordinates.\n\n" + f"The current Scratch project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Example Json output:\n" + "```json\n" + "{\n" + " \"sprite_initial_positions\": {\n" + " \"Sprite1\": {\"x\": -160, \"y\": -110},\n" + " \"Sprite2\": {\"x\": 240, \"y\": -135}\n" + " }\n" + "}" + "```\n" + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_query_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM:", raw_response) + # json debugging and solving + try: + updated_data = extract_json_from_llm_response(raw_response) + sprite_positions = updated_data.get("sprite_initial_positions", {}) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT PARSER]: {correction_response}") + corrected_data = extract_json_from_llm_response(correction_response["messages"][-1].content) + sprite_positions = corrected_data.get("sprite_initial_positions", {}) + + new_project_json = update_project_with_sprite_positions(state["project_json"], sprite_positions) + state["project_json"]= new_project_json + print("Updated project JSON with sprite positions:", json.dumps(new_project_json, indent=2)) + return state + #return {"project_json": new_project_json} + except Exception as e: + logger.error(f"Error in ParseQueryNode: {e}") + raise + + +# Node 3: Sprite Action Plan Builder +def overall_planner_node(state: GameState): + """ + Generates a comprehensive action plan for sprites, including detailed Scratch block information. + This node acts as an overall planner, leveraging knowledge of all block shapes and categories. + """ + logger.info("--- Running OverallPlannerNode ---") + + description = state.get("description", "") + project_json = state["project_json"] + sprite_names = [t["name"] for t in project_json["targets"] if not t["isStage"]] + + # Get sprite positions from the project_json if available, or set defaults + sprite_positions = {} + for target in project_json["targets"]: + if not target["isStage"]: + sprite_positions[target["name"]] = {"x": target.get("x", 0), "y": target.get("y", 0)} + + planning_prompt = ( + "Generate a detailed action plan for the game's sprites based on the user query and sprite details.\n\n" + f"**Game Description:** '{description}'\n\n" + f"**Sprites in Game:** {', '.join(sprite_names)}\n" + f"**Current Sprite Positions:** {json.dumps(sprite_positions)}\n\n" + "--- Scratch 3.0 Block Reference ---\n" + "This section provides a comprehensive reference of Scratch 3.0 blocks, categorized by shape, including their opcodes and functional descriptions. Use this to accurately identify block types and behavior.\n\n" + f"### Hat Blocks\nDescription: {hat_description}\nBlocks:\n{hat_opcodes_functionalities}\n\n" + f"### Boolean Blocks\nDescription: {boolean_description}\nBlocks:\n{boolean_opcodes_functionalities}\n\n" + f"### C Blocks\nDescription: {c_description}\nBlocks:\n{c_opcodes_functionalities}\n\n" + f"### Cap Blocks\nDescription: {cap_description}\nBlocks:\n{cap_opcodes_functionalities}\n\n" + f"### Reporter Blocks\nDescription: {reporter_description}\nBlocks:\n{reporter_opcodes_functionalities}\n\n" + f"### Stack Blocks\nDescription: {stack_description}\nBlocks:\n{stack_opcodes_functionalities}\n\n" + "-----------------------------------\n\n" + "Your task is to define the primary actions and movements for each sprite.\n" + "The output should be a JSON object with a single key 'action_overall_flow'. Each key inside this object should be a sprite name (e.g., 'Player', 'Enemy'), and its value must include a 'description' and a list of 'plans'.\n" + "Each plan must begin with a **single Scratch Hat Block** (e.g., 'event_whenflagclicked') and should contain:\n" + "1. **'event'**: the exact `opcode` of the hat block that initiates the logic.\n" + "2. **'logic'**: a natural language breakdown of each step taken after the event. Separate each step with a semicolon ';'. Ensure clarity and granularity—each described action should map closely to a Scratch block or tight sequence.\n" + " - For example: for a jump: 'change y by N; wait M seconds; change y by -N'.\n" + " - Use 'forever: ...' to prefix repeating logic explicitly.\n" + " - Use Scratch-consistent verbs: 'move', 'change', 'wait', 'hide', 'say', etc.\n" + "3. **Opcode Lists**: include relevant Scratch opcodes grouped under `motion`, `control`, `operator`, `sensing`, `looks`, `sounds`, `events`, and `data`. List only the non-empty categories. Use exact opcodes including shadow/helper blocks (e.g., 'math_number').\n\n" + "Use sprite names exactly as listed in `sprite_names`. Do NOT rename or invent new sprites.\n" + "Ensure the plan reflects accurate opcode usage derived strictly from the block reference above.\n\n" + "Example structure for 'action_overall_flow':\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at starting point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"repeat(10): change y by 10; wait 0.1 seconds; change y by -10;\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_repeat\", \"control_wait\"],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement and interaction\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135; forever: glide 2 seconds to x:-240 y:-135; if x position < -235, then set x to 240; if touching Sprite1, then hide;\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\", \"motion_setx\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operator\": [\"operator_lt\"],\n" + " \"sensing\": [\"sensing_istouching\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [\"looks_hide\"],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Based on the provided context, generate the `action_overall_flow`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": planning_prompt}]}) + raw_response = response["messages"][-1].content + print("Raw response from LLM [OverallPlannerNode]:", raw_response) # Uncomment for debugging + # json debugging and solving + try: + overall_plan = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT OVERALLPLANNERNODE ]: {correction_response}") + overall_plan= extract_json_from_llm_response(correction_response["messages"][-1].content) + + state["action_plan"] = overall_plan + logger.info("Overall plan generated by OverallPlannerNode.") + return state + + except Exception as e: + logger.error(f"Error in OverallPlannerNode: {e}") + raise + +# Helper function to get a block by its opcode from a single catalog +def get_block_by_opcode(catalog_data: dict, opcode: str) -> dict | None: + """ + Search a single catalog (with keys "description" and "blocks": List[dict]) + for a block whose 'op_code' matches the given opcode. + Returns the block dict or None if not found. + """ + for block in catalog_data["blocks"]: + if block.get("op_code") == opcode: + return block + return None + +# Helper function to find a block in all catalogs by opcode +def find_block_in_all(opcode: str, all_catalogs: list[dict]) -> dict | None: + """ + Search across multiple catalogs for a given opcode. + Returns the first matching block dict or None. + """ + for catalog in all_catalogs: + blk = get_block_by_opcode(catalog, opcode) + if blk is not None: + return blk + return None + +# Node 4: Sprite Plan Verification Node +def plan_verification_node(state: GameState): + """ + Validates the generated action plan/blocks, identifies missing logic, + provides feedback, and determines if further improvements are needed. + Also manages the iteration count for the improvement loop. + """ + logger.info(f"--- Running VerificationNode (Iteration: {state.get('iteration_count', 0)}) ---") + + MAX_IMPROVEMENT_ITERATIONS = 1 # Set a sensible limit to prevent infinite loops + current_iteration = state.get("iteration_count", 0) + #project_json = state["project_json"] + action_plan = state.get("action_plan", {}) + print(f"[action_plan before verification] on ({current_iteration}): {json.dumps(action_plan, indent=2)}") + #improvement_plan = state.get("improvement_plan", {}) # May contain prior improvement guidance + + # Corrected validation_prompt + validation_prompt = ( + f"You are an AI validator for Scratch project plans and generated blocks. " + f"Your task is to review the current state of the game's action plan and block structure. " + f"Critically analyze if there are any missing logic, structural inconsistencies, or unclear intentions. " + f"Provide **precise** and **constructive** feedback for improvement.\n\n" + f"**Game Description:**\n" + f"{state.get('description', '')}\n\n" + f"**Current Action Plan (High-Level Logic):**\n" + f"```json\n{json.dumps(action_plan, indent=2)}\n```\n\n" + # Uncomment below if needed + # f"**Current Project JSON (Generated Blocks):**\n" + # f"```json\n{json.dumps(project_json, indent=2)}\n```\n\n" + f"**Previous Feedback (if any):**\n" + f"{state.get('plan_validation_feedback', 'None')}\n\n" + f"Based on the above, return a response strictly in the following JSON format:\n" + "```json\n" + "{\n" + " \"feedback\": \"Detailed comments on any missing logic, inconsistencies, or unclear intent. Be concise but specific. If everything is perfect, state that explicitly.\",\n" + " \"needs_improvement\": true,\n" + " \"suggested_description_updates\": \"Concise revision of the game description if needed. Use an empty string if no change is required.\"\n" + "}\n" + "```\n" + "**Important:**\n" + "- The `needs_improvement` field must be strictly `true` or `false` (boolean). Do **not** include any other text or explanation inside the JSON.\n" + "- Be strict in evaluation. If **any** part of the plan or block logic appears incomplete, ambiguous, or incorrect, set `needs_improvement` to `true`." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": validation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [VerificationNode]: {raw_response[:500]}...") + # json debugging and solving + try: + validation_result = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT PLANVERIFICATIONNODE ]: {correction_response}") + validation_result = extract_json_from_llm_response(correction_response["messages"][-1].content) + + # Update state with feedback and improvement flag + state["plan_validation_feedback"] = validation_result.get("feedback", "No specific feedback provided.") + state["needs_improvement"] = validation_result.get("needs_improvement", False) + + suggested_description_updates = validation_result.get("suggested_description_updates", "") + if suggested_description_updates: + # You might want to append or intelligently merge this with the existing detailed_game_description + # For simplicity, let's just append for now or update a specific field + #current_description = state.get("detailed_game_description", state.get("description", "")) + current_description = state.get(state.get("description", "")) + #state["detailed_game_description"] = f"{current_description}\n\nSuggested Update: {suggested_description_updates}"\ + state["description"] = f"{current_description}\n\nSuggested Update: {suggested_description_updates}" + logger.info("Updated detailed game description based on validation feedback.") + + + # Manage iteration count + if state["needs_improvement"]: + state["iteration_count"] = current_iteration + 1 + if state["iteration_count"] >= MAX_IMPROVEMENT_ITERATIONS: + logger.warning(f"Max improvement iterations ({MAX_IMPROVEMENT_ITERATIONS}) reached. Forcing 'needs_improvement' to False.") + state["needs_improvement"] = False + state["plan_validation_feedback"] += "\n(Note: Max iterations reached, stopping further improvements.)" + else: + state["iteration_count"] = 0 # Reset if no more improvement needed + + logger.info(f"Verification completed. Needs Improvement: {state['needs_improvement']}. Feedback: {state['plan_validation_feedback'][:100]}...") + print(f"[updated action_plan after verification] on ({current_iteration}): {json.dumps(state.get("action_plan", {}), indent=2)}") + return state + except Exception as e: + logger.error(f"Error in VerificationNode: {e}") + state["needs_improvement"] = False # Force end loop on error + state["plan_validation_feedback"] = f"Validation error: {e}" + raise + +# Node 5: Refined Planner Node +def refined_planner_node(state: GameState): + """ + Refines the action plan based on validation feedback and game description. + """ + logger.info("--- Running RefinedPlannerNode ---") + + #detailed_game_description = state.get("detailed_game_description", state.get("description", "A game.")) + detailed_game_description = state.get("description","") + current_action_plan = state.get("action_plan", {}) + print(f"[current_action_plan before refinement] on ({state.get('iteration_count', 0)}): {json.dumps(current_action_plan, indent=2)}") + plan_validation_feedback = state.get("plan_validation_feedback", "No specific feedback provided. Assume general refinement is needed.") + project_json = state["project_json"] + sprite_names = [t["name"] for t in project_json["targets"] if not t["isStage"]] + + # Get sprite positions from the project_json if available, or set defaults + sprite_positions = {} + for target in project_json["targets"]: + if not target["isStage"]: + sprite_positions[target["name"]] = {"x": target.get("x", 0), "y": target.get("y", 0)} + + refinement_prompt = ( + "Refine the existing action plan for the game's sprites based on the detailed game description and the validation feedback provided.\n\n" + f"**Detailed Game Description:** '{detailed_game_description}'\n\n" + f"**Sprites in Game:** {', '.join(sprite_names)}\n" + f"**Current Sprite Positions:** {json.dumps(sprite_positions)}\n\n" + f"**Current Action Plan (to be refined):**\n" + f"```json\n{json.dumps(current_action_plan, indent=2)}\n```\n\n" + f"**Validation Feedback:**\n" + f"'{plan_validation_feedback}'\n\n" + "--- Scratch 3.0 Block Reference ---\n" + f"### Hat Blocks\nDescription: {hat_description}\nBlocks:\n{hat_opcodes_functionalities}\n\n" + f"### Boolean Blocks\nDescription: {boolean_description}\nBlocks:\n{boolean_opcodes_functionalities}\n\n" + f"### C Blocks\nDescription: {c_description}\nBlocks:\n{c_opcodes_functionalities}\n\n" + f"### Cap Blocks\nDescription: {cap_description}\nBlocks:\n{cap_opcodes_functionalities}\n\n" + f"### Reporter Blocks\nDescription: {reporter_description}\nBlocks:\n{reporter_opcodes_functionalities}\n\n" + f"### Stack Blocks\nDescription: {stack_description}\nBlocks:\n{stack_opcodes_functionalities}\n\n" + "-----------------------------------\n\n" + "Your task is to refine the JSON object 'action_overall_flow'.\n" + "Use sprite names exactly as provided in `sprite_names` (e.g., 'Sprite1', 'soccer ball'); do **NOT** rename them.\n\n" + "Follow this exact format for the output (example):\n" + "Example structure for 'action_overall_flow':\n" + "```json\n" + "{\n" + " \"action_overall_flow\": {\n" + " \"Sprite1\": {\n" + " \"description\": \"Main character (cat) actions\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to initial position at starting point.\",\n" + " \"motion\": [\"motion_gotoxy\"],\n" + " \"control\": [],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " },\n" + " {\n" + " \"event\": \"event_whenkeypressed\",\n" + " \"logic\": \"repeat(10): change y by 10 → wait 0.1 sec → change y by -10\",\n" + " \"motion\": [\"motion_changeyby\"],\n" + " \"control\": [\"control_repeat\", \"control_wait\"],\n" + " \"operator\": [],\n" + " \"sensing\": [],\n" + " \"looks\": [],\n" + " \"sounds\": [],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " },\n" + " \"soccer ball\": {\n" + " \"description\": \"Obstacle movement and interaction\",\n" + " \"plans\": [\n" + " {\n" + " \"event\": \"event_whenflagclicked\",\n" + " \"logic\": \"go to x:240 y:-135 → forever glide to x:-240 y:-135 → if x position < -235 then set x to 240 → if touching Sprite1 then hide\",\n" + " \"motion\": [\"motion_gotoxy\", \"motion_glidesecstoxy\", \"motion_xposition\", \"motion_setx\"],\n" + " \"control\": [\"control_forever\", \"control_if\"],\n" + " \"operator\": [\"operator_lt\"],\n" + " \"sensing\": [\"sensing_istouching\", \"sensing_touchingobjectmenu\"],\n" + " \"looks\": [\"looks_hide\"],\n" + " \"data\": []\n" + " }\n" + " ]\n" + " }\n" + " }\n" + "}\n" + "```\n" + "Use the validation feedback to address errors, fill in missing logic, or enhance clarity.\n" + "example of few possible improvements: 1.event_whenflagclicked is used to control sprite but its used for actual start scratch project and reset scratch. 2. looping like forever used where we should use iterative. 3. missing of for variable we used in the block\n" + "- Maintain the **exact JSON structure** shown above.\n" + "- All `logic` fields must be **clear and granular**.\n" + "- Only include opcode categories that contain relevant opcodes.\n" + "- Ensure that each opcode matches its intended Scratch functionality.\n" + "- If feedback suggests major change, **rethink the entire plan** for the affected sprite(s).\n" + "- If feedback is minor, make precise, minimal improvements only." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": refinement_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [RefinedPlannerNode]: {raw_response[:500]}...") + # json debugging and solving + try: + refined_plan = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT REFINEPLANNER ]: {correction_response}") + refined_plan = extract_json_from_llm_response(correction_response["messages"][-1].content) + logger.info("Refined plan corrected by JSON resolver agent.") + + if refined_plan: + #state["action_plan"] = refined_plan.get("action_overall_flow", {}) # Update to the key 'action_overall_flow' [error] + state["action_plan"] = refined_plan.get("action_overall_flow", {}) # Update the main the prompt includes updated only + logger.info("Action plan refined by RefinedPlannerNode.") + else: + logger.warning("RefinedPlannerNode did not return a valid 'action_overall_flow' structure. Keeping previous plan.") + print("[Refined Action Plan]:", json.dumps(state["action_plan"], indent=2)) + print("[current state after refinement]:", json.dumps(state, indent=2)) + return state + except Exception as e: + logger.error(f"Error in RefinedPlannerNode: {e}") + raise + +# Node 4: Overall Block Builder Node +def overall_block_builder_node(state: GameState): + logger.info("--- Running OverallBlockBuilderNode ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + # Pre-load all block-catalog JSONs once + all_catalogs = [ + hat_block_data, + boolean_block_data, + c_block_data, + cap_block_data, + reporter_block_data, + stack_block_data + ] + action_plan = state.get("action_plan", {}) + print("[Overall Action Plan received at the block generator]:", json.dumps(action_plan, indent=2)) + if not action_plan: + logger.warning("No action plan found in state. Skipping OverallBlockBuilderNode.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + # This is the handler which ensure if somehow json response changed it handle it.[DONOT REMOVE BELOW LOGIC] + if action_plan.get("action_overall_flow", {})=={}: + plan_data = action_plan.items() + else: + plan_data= action_plan.get("action_overall_flow", {}).items() + + for sprite_name, sprite_actions_data in plan_data: + #for sprite_name, sprite_actions_data in action_plan.items(): + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_actions_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operator_opcodes = plan_entry.get("operator", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + # Create a string representation of the identified opcodes for the prompt + identified_opcodes_str = "" + if motion_opcodes: + identified_opcodes_str += f" Motion Blocks (opcodes): {', '.join(motion_opcodes)}\n" + if control_opcodes: + identified_opcodes_str += f" Control Blocks (opcodes): {', '.join(control_opcodes)}\n" + if operator_opcodes: + identified_opcodes_str += f" Operator Blocks (opcodes): {', '.join(operator_opcodes)}\n" + if sensing_opcodes: + identified_opcodes_str += f" Sensing Blocks (opcodes): {', '.join(sensing_opcodes)}\n" + if looks_opcodes: + identified_opcodes_str += f" Looks Blocks (opcodes): {', '.join(looks_opcodes)}\n" + if sound_opcodes: + identified_opcodes_str += f" Sound Blocks (opcodes): {', '.join(sound_opcodes)}\n" + if events_opcodes: + identified_opcodes_str += f" Event Blocks (opcodes): {', '.join(events_opcodes)}\n" + if data_opcodes: + identified_opcodes_str += f" Data Blocks (opcodes): {', '.join(data_opcodes)}\n" + + needed_opcodes = motion_opcodes + control_opcodes + operator_opcodes + sensing_opcodes + looks_opcodes + sound_opcodes + events_opcodes + data_opcodes + needed_opcodes = list(set(needed_opcodes)) + + + # 2) build filtered runtime catalog (if you still need it) + filtered_catalog = { + op: ALL_SCRATCH_BLOCKS_CATALOG[op] + for op in needed_opcodes + if op in ALL_SCRATCH_BLOCKS_CATALOG + } + + # 3) merge human‑written catalog + runtime entry for each opcode + combined_blocks = {} + for op in needed_opcodes: + catalog_def = find_block_in_all(op, all_catalogs) or {} + runtime_def = ALL_SCRATCH_BLOCKS_CATALOG.get(op, {}) + # merge: catalog fields first, then runtime overrides/adds + merged = {**catalog_def, **runtime_def} + combined_blocks[op] = merged + print("[Combined blocks for this script]:", json.dumps(combined_blocks, indent=2)) + + llm_block_generation_prompt = ( + f"You are an AI assistant generating a **single complete Scratch 3.0 script** in JSON format.\n" + f"The current sprite is '{sprite_name}'.\n" + f"This script must start with the event block (Hat Block) for opcode '{event_opcode}'.\n" + f"The sequential logic to implement for this script is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Required Block Opcodes & Catalog:**\n" + f"Based on the planning, the following specific Scratch block opcodes are expected to be used. You MUST use these opcodes where applicable.\n" + f"Here is the comprehensive catalog for these blocks, including their structure and required inputs/fields:\n" + f"```json\n{json.dumps(combined_blocks, indent=2)}\n```\n\n" + f"Current Scratch project JSON for this sprite (provided for context; you are generating a NEW, complete script):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**CRITICAL INSTRUCTIONS FOR GENERATING THE BLOCK JSON (READ CAREFULLY AND FOLLOW PRECISELY):**\n" + f"1. **Unique Block IDs:** Generate a **globally unique ID** for EVERY block (main and shadow blocks) within the entire JSON output for this script. Example: 'myBlockID123'.\n" + f"2. **Script Initiation (Hat Block - VERY IMPORTANT):**\n" + f" * The **first block** of the script (the Hat block, opcode '{event_opcode}') MUST have `\"topLevel\": true` and `\"parent\": null`.\n" + f" * ONLY this Hat block should have `\"topLevel\": true`. All other blocks in the script MUST have `\"topLevel\": false`.\n" + f" * Set its `x` and `y` coordinates (e.g., `x: 0, y: 0` or similar for clear placement).\n" + f"3. **Strict Block Chaining (`next` and `parent`):**\n" + f" * Use the `next` field to point to the ID of the block DIRECTLY BELOW it in the stack. If a block has a `next`, its `next` block's `parent` MUST point back to the current block's ID.\n" + f" * Use the `parent` field to point to the ID of the block DIRECTLY ABOVE it in the stack. If a block has a `parent`, then that `parent` block's `next` MUST point to the current block's ID.\n" + f" * The **last block** in a linear stack (e.g., a Stack block not containing other blocks, or a Cap block) MUST have `\"next\": null`.\n" + f" * Blocks plugged into inputs (like Boolean reporters or Reporter blocks) or substacks (like inside C-blocks) do NOT use `next` to connect to the block holding them. Their connection is solely via the `inputs` field of their parent.\n" + f"4. **`inputs` Field Structure (ABSOLUTELY CRITICAL - ADHERE TO THIS RIGIDLY):**\n" + f" * The value for ANY key within the `inputs` dictionary MUST be an **array of EXACTLY two elements**: `[type_code, value_or_block_id]`.\n" + f" * **NEVER embed direct primitive values or arrays within `inputs` without a `type_code`**: E.g., `\"INPUT_NAME\": [\"value\", null]` or `\"INPUT_NAME\": [\"nestedArray\"]` are **STRICTLY FORBIDDEN and will cause errors**.\n" + f" * **`type_code` (first element):**\n" + f" * `1`: Use when `value_or_block_id` refers to a **primitive value** (number, string, boolean) or the ID of a **shadow block** (e.g., `math_number`, `text`, menu blocks).\n" + f" * `2`: Use when `value_or_block_id` refers to the ID of a **non-shadow block** that is PLUGGED IN (e.g., a Boolean block in a condition, or the first block of a C-block's `SUBSTACK`).\n" + f" * **Correct Examples (Re-emphasized):**\n" + f" * **Numerical Input (`motion_movesteps`):** To input `10` steps, define a separate `math_number` shadow block and reference its ID:\n" + f" ```json\n" + f" // Main block\n" + f" \"moveStepsID\": {{\n" + f" \"opcode\": \"motion_movesteps\", \"inputs\": {{ \"STEPS\": [1, \"shadowNumID\"] }},\n" + f" \"parent\": \"parentBlockID\", \"next\": \"nextBlockID\", \"topLevel\": false, \"shadow\": false\n" + f" }},\n" + f" // Separate shadow block for '10'\n" + f" \"shadowNumID\": {{\n" + f" \"opcode\": \"math_number\", \"fields\": {{ \"NUM\": [\"10\", null] }},\n" + f" \"parent\": \"moveStepsID\", \"shadow\": true, \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Dropdown/Menu Input (`sensing_touchingobject` with 'edge'):** Define a separate menu shadow block and reference its ID:\n" + f" ```json\n" + f" // Main block\n" + f" \"touchingBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", \"inputs\": {{ \"TOUCHINGOBJECTMENU\": [1, \"shadowEdgeMenuID\"] }},\n" + f" \"parent\": \"parentBlockID\", \"next\": null, \"topLevel\": false, \"shadow\": false\n" + f" }},\n" + f" // Separate shadow block for '_edge_'\n" + f" \"shadowEdgeMenuID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\", \"fields\": {{ \"TOUCHINGOBJECTMENU\": [\"_edge_\", null] }},\n" + f" \"parent\": \"touchingBlockID\", \"shadow\": true, \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **C-block (`control_forever`):** Its `SUBSTACK` input MUST point to the first block inside its loop using `type_code: 2`.\n" + f" ```json\n" + f" \"foreverBlockID\": {{\n" + f" \"opcode\": \"control_forever\", \"inputs\": {{ \"SUBSTACK\": [2, \"firstBlockInsideForeverID\"] }},\n" + f" \"next\": null, \"parent\": \"blockAboveForeverID\", \"shadow\": false, \"topLevel\": false\n" + f" }},\n" + f" \"firstBlockInsideForeverID\": {{\n" + f" \"opcode\": \"motion_movesteps\", \"parent\": \"foreverBlockID\", \"next\": null, \"topLevel\": false, \"shadow\": false\n" + f" }}\n" + f" ```\n" + f" * **C-block with Condition (`control_if`):** `CONDITION` input uses `type_code: 1` (referencing a Boolean reporter), and `SUBSTACK` uses `type_code: 2` (referencing the first block inside the if-body).\n" + f" ```json\n" + f" \"ifBlockID\": {{\n" + f" \"opcode\": \"control_if\",\n" + f" \"inputs\": {{\n" + f" \"CONDITION\": [1, \"conditionBlockID\"],\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideIfID\"]\n" + f" }},\n" + f" \"next\": \"blockAfterIfID\", \"parent\": \"blockAboveIfID\", \"shadow\": false, \"topLevel\": false\n" + f" }},\n" + f" \"conditionBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", \"parent\": \"ifBlockID\", \"shadow\": false, \"topLevel\": false // ... and its own inputs/fields\n" + f" }},\n" + f" \"firstBlockInsideIfID\": {{\n" + f" \"opcode\": \"looks_sayforsecs\", \"parent\": \"ifBlockID\", \"next\": null, \"topLevel\": false, \"shadow\": false\n" + f" }}\n" + f" ```\n" + f"5. **Separate Shadow Blocks (MANDATORY):** Every time a block's input requires a numeric value, string literal, or a selection from a dropdown/menu, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow. Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f"6. **`fields` for Direct Values:** Use the `fields` dictionary ONLY if the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., `NUM` field in `math_number`, `KEY_OPTION` in `event_whenkeypressed`, `VARIABLE` field in `data_setvariableto`). Example: `\"fields\": {{\"NUM\": [\"10\", null]}}`.\n" + f"7. **Strictly Use Catalog Opcodes:** You MUST only use `opcode` values that are present in the provided `combined_blocks` catalog. Do NOT use unlisted or hypothetical opcodes (e.g., `motion_jump`).\n" + f"8. **Output Format:** Return **ONLY the JSON object** representing all the blocks for THIS SINGLE SCRIPT. Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [OverallBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + print(f"Raw response from LLM [OverallBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response}") # Uncomment for debugging + try: + generated_blocks = extract_json_from_llm_response(raw_response) + + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT OVERALLBLOCKBUILDER ]: {correction_response}") + generated_blocks = extract_json_from_llm_response(correction_response["messages"][-1].content) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + + # setting the iteration count for the script + state["iteration_count"] = 0 + + logger.info(f"Action blocks added for sprite '{sprite_name}', script '{event_opcode}' by OverallBlockBuilderNode.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with action nodes.") + print("Updated project JSON with action nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + + +#helper function to identify the shape of block utilized by the block verifier +def get_block_type(opcode: str) -> str: + """Determines the general type of a Scratch block based on its opcode.""" + if not opcode: + return "unknown" + if opcode.startswith("event_when") or opcode == "control_start_as_clone": + return "hat" + elif opcode.startswith("control_") and ("if" in opcode or "repeat" in opcode or "forever" in opcode): + return "c_block" + elif opcode in ["operator_equals", "operator_gt", "operator_lt", "operator_and", "operator_or", "operator_not"] or \ + (opcode.startswith("sensing_") and ("mousedown" in opcode or "keypressed" in opcode or "touching" in opcode)): + return "boolean" + elif opcode.endswith("menu"): # For dropdown shadow blocks, treat as reporter for type checking + return "reporter" + elif any(s in opcode for s in ["position", "direction", "size", "volume", "costume", "backdrop", "random", "add", "subtract", "multiply", "divide", "length", "item", "of"]): + # A more comprehensive check for reporters + return "reporter" + elif "stop_all" in opcode or "delete_this_clone" in opcode or "procedures_definition" in opcode: + return "cap" + # Default to stack for most command blocks if not explicitly defined + return ALL_SCRATCH_BLOCKS_CATALOG.get(opcode, {}).get("blockType", "stack") + +#helper function to identify the shape of block utilized by the block verifier +def filter_script_blocks(all_blocks: dict, hat_block_id: str) -> dict: + """ + Filters and returns only the blocks that are part of a specific script + starting from the given hat_block_id, including connected reporters/shadows. + """ + script_blocks = {} + q = [hat_block_id] + visited = set() + + while q: + current_block_id = q.pop(0) + if current_block_id in visited: + continue + visited.add(current_block_id) + + block_data = all_blocks.get(current_block_id) + if not block_data: + continue + + script_blocks[current_block_id] = block_data + + # Add next block in sequence + next_id = block_data.get("next") + if next_id and next_id in all_blocks: + q.append(next_id) + + # Add blocks connected via inputs (e.g., reporters, shadow blocks, substacks) + if "inputs" in block_data: + for input_name, input_value in block_data["inputs"].items(): + if isinstance(input_value, list) and len(input_value) >= 2: + value_or_block_id = input_value[1] + if isinstance(value_or_block_id, str) and value_or_block_id in all_blocks: + q.append(value_or_block_id) + # For type code 3 (reporter with default value), the third element might be a connected block + if len(input_value) >= 3 and isinstance(input_value[2], str) and input_value[2] in all_blocks: + q.append(input_value[2]) + + # For C-blocks, add blocks in substacks (if present) + # SUBSTACKs are inputs that have type code 2 and contain the block ID of the first block in the substack + if get_block_type(block_data.get("opcode")) == "c_block": + for input_key, input_val in block_data.get("inputs", {}).items(): + if input_key.startswith("SUBSTACK") and isinstance(input_val, list) and len(input_val) >= 2 and input_val[0] == 2 and isinstance(input_val[1], str) and input_val[1] in all_blocks: + q.append(input_val[1]) + + return script_blocks + + +def analyze_script_structure(script_blocks: dict, hat_block_id: str, sprite_name: str) -> list: + """ + Analyzes the structure of a single Scratch script for common errors. + Returns a list of issue strings. + """ + issues = [] + + # 1. Validate the hat block + hat_block = script_blocks.get(hat_block_id) + if not hat_block: + issues.append(f"Script for sprite '{sprite_name}' (hat ID: {hat_block_id}) has no hat block data.") + return issues # Cannot proceed without a hat block + + if not hat_block.get("topLevel") or hat_block.get("parent") is not None: + issues.append(f"Hat block '{hat_block_id}' for sprite '{sprite_name}' is not marked as topLevel or has a parent.") + + # 2. Check all blocks within the script + for block_id, block_data in script_blocks.items(): + opcode = block_data.get("opcode") + if not opcode: + issues.append(f"Block '{block_id}' for sprite '{sprite_name}' is missing an opcode.") + continue + + # Check if opcode exists in the catalog (simplified check for this example) + if opcode not in ALL_SCRATCH_BLOCKS_CATALOG: + issues.append(f"Block '{block_id}' for sprite '{sprite_name}' has unknown opcode '{opcode}'.") + + # Parent-Child and Next-Previous Linkage + parent_id = block_data.get("parent") + next_id = block_data.get("next") + + if parent_id: + parent_block = script_blocks.get(parent_id) + if not parent_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' references non-existent parent '{parent_id}'.") + else: + parent_block_type = get_block_type(parent_block.get("opcode")) + current_block_type = get_block_type(opcode) + + if parent_block_type in ["stack", "hat"]: + # For stack and hat blocks, the parent's 'next' should point to this block + if parent_block.get("next") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode})'s parent '{parent_id}' does not link back to it via 'next'.") + elif parent_block_type == "c_block": + # For C-blocks, this block should be in a substack input + found_in_substack = False + for input_key, input_val in parent_block.get("inputs", {}).items(): + if isinstance(input_val, list) and len(input_val) >= 2 and input_val[1] == block_id: + if input_val[0] == 2 and input_key.startswith("SUBSTACK"): # Type code 2 for substacks + found_in_substack = True + break + # Shadow blocks and reporter blocks can also be inputs to C-blocks but not as substacks + if not found_in_substack and not block_data.get("shadow") and current_block_type not in ["reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) has parent '{parent_id}' but is not a substack, shadow, or reporter/boolean connection to a C-block.") + elif parent_block_type in ["reporter", "boolean"]: + # Reporter/Boolean blocks can be parents if they are input to another block + found_in_input = False + for input_key, input_val in parent_block.get("inputs", {}).items(): + if isinstance(input_val, list) and len(input_val) >= 2 and input_val[1] == block_id: + found_in_input = True + break + if not found_in_input: + issues.append(f"Block '{block_id}' (opcode: {opcode}) has reporter/boolean parent '{parent_id}' but is not linked via an input.") + else: # e.g., cap blocks cannot be parents to other blocks in a sequence + issues.append(f"Block '{block_id}' (opcode: {opcode}) has an unexpected parent block type '{parent_block_type}' for parent '{parent_id}'.") + + + if next_id: + next_block = script_blocks.get(next_id) + if not next_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' references non-existent next block '{next_id}'.") + elif next_block.get("parent") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode})'s next block '{next_id}' does not link back to it via 'parent'.") + elif block_data.get("shadow"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection.") + elif get_block_type(opcode) == "hat": # Hat blocks should not generally have a 'next' in a linear script sequence + issues.append(f"Hat block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection as a sequential block.") + elif get_block_type(opcode) == "cap": + issues.append(f"Cap block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection as it signifies script end.") + + + # Input validation + if "inputs" in block_data: + for input_name, input_value in block_data["inputs"].items(): + if not isinstance(input_value, list) or len(input_value) < 2: + issues.append(f"Block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' has malformed input '{input_name}': {input_value}.") + continue + + type_code = input_value[0] + value_or_block_id = input_value[1] + + # Type code 1: number/string/boolean literal or block ID (reporter/boolean) + # Type code 2: block ID (substack or reporter/boolean plugged in) + # Type code 3: number/string/boolean literal with shadow, or reporter with default value if nothing plugged in + if type_code not in [1, 2, 3]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has invalid type code: {type_code}. Expected 1, 2, or 3.") + + if isinstance(value_or_block_id, str): + # It's a block ID, check if it exists and its parent link is correct + connected_block = script_blocks.get(value_or_block_id) + if not connected_block: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' references non-existent block ID '{value_or_block_id}'.") + elif connected_block.get("parent") != block_id: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' connects to '{value_or_block_id}', but '{value_or_block_id}'s parent is not '{block_id}'.") + # Check for type code consistency with connected block type + elif type_code == 2 and get_block_type(connected_block.get("opcode")) not in ["stack", "hat", "c_block", "cap", "reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has type code 2 but connected block '{value_or_block_id}' is not a valid block type for substack/input.") + elif type_code == 1 and get_block_type(connected_block.get("opcode")) not in ["reporter", "boolean"]: + issues.append(f"Block '{block_id}' (opcode: {opcode}) input '{input_name}' has type code 1 but connected block '{value_or_block_id}' is not a reporter or boolean (expected for direct value input).") + + # Specific checks for C-blocks' SUBSTACK + if block_data.get("blockType") == "c_block" and input_name.startswith("SUBSTACK"): # Changed to startswith as SUBSTACK1, SUBSTACK2 might exist + if not (isinstance(value_or_block_id, str) and script_blocks.get(value_or_block_id) and type_code == 2): + issues.append(f"C-block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' has an invalid or missing SUBSTACK input configuration.") + + + # Shadow block specific checks + if block_data.get("shadow"): + if block_data.get("topLevel"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' is incorrectly marked as topLevel.") + if not block_data.get("parent"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' is missing a parent.") + if block_data.get("next"): + issues.append(f"Shadow block '{block_id}' (opcode: {opcode}) for sprite '{sprite_name}' should not have a 'next' connection.") + # Check fields for math_number and menu blocks + if opcode == "math_number" and not (isinstance(block_data.get("fields", {}).get("NUM"), list) and len(block_data["fields"]["NUM"]) >= 1 and isinstance(block_data["fields"]["NUM"][0], (str, int, float))): + issues.append(f"Math_number shadow block '{block_id}' (opcode: {opcode}) has malformed 'NUM' field.") + # This logic for menu shadow blocks assumes the field name is the opcode in uppercase without _MENU + # Example: for "looks_costumemenu", it expects a field "COSTUME" + if opcode.endswith("menu"): + expected_field = opcode.upper().replace("_MENU", "") + if not (isinstance(block_data.get("fields", {}).get(expected_field), list) and len(block_data["fields"][expected_field]) >= 1 and isinstance(block_data["fields"][expected_field][0], str)): + issues.append(f"Menu shadow block '{block_id}' (opcode: {opcode}) has malformed field '{expected_field}' for its specific menu option.") + + return issues + +# Node 5: Verification Node +def block_verification_node(state: dict) -> dict: + """ + The block verifier check for the if any improvement need if any through logical if else and then add it to improvement_plan. + aftet improvement plan the llm reviewer node also check for other error or issues if any and at last give the review as feedback. + + Args: + state (dict): _description_ + + Returns: + dict: _description_ + """ + logger.info(f"--- Running BlockVerificationNode (Iteration: {state.get('iteration_count', 0)}) ---") + + MAX_IMPROVEMENT_ITERATIONS = 1 # Set a sensible limit to prevent infinite loops + current_iteration = state.get("iteration_count", 0) + project_json = state["project_json"] + targets = project_json["targets"] + + # Initialize needs_improvement for the current run + state["needs_improvement"] = False + block_validation_feedback_overall = [] + + improvement_plan = {"sprite_issues": {}} + + + for target in targets: + sprite_name = target["name"] + all_blocks_for_sprite = target.get("blocks", {}) + + if not all_blocks_for_sprite: + logger.info(f"Sprite '{sprite_name}' has no blocks. Skipping verification.") + continue + + sprite_issues = [] + hat_block_ids = [ + block_id for block_id, block_data in all_blocks_for_sprite.items() + if block_data.get("topLevel") and get_block_type(block_data.get("opcode")) == "hat" + ] + + processed_script_blocks = set() + + if not hat_block_ids: + sprite_issues.append("No top-level hat blocks found for this sprite. Scripts may not run.") + for block_id, block_data in all_blocks_for_sprite.items(): + if block_data.get("topLevel") and not get_block_type(block_data.get("opcode")) == "hat": + sprite_issues.append(f"Top-level block '{block_id}' (opcode: {block_data.get('opcode')}) is not a hat block, so it will not run automatically.") + if not block_data.get("topLevel") and not block_data.get("parent") and not block_data.get("shadow"): + sprite_issues.append(f"Orphaned block '{block_id}' (opcode: {block_data.get('opcode')}) is not top-level, has no parent, and is not a shadow block.") + + for hat_id in hat_block_ids: + logger.info(f"Verifying script starting with hat block '{hat_id}' for sprite '{sprite_name}'.") + + current_script_blocks = filter_script_blocks(all_blocks_for_sprite, hat_id) + processed_script_blocks.update(current_script_blocks.keys()) + + script_issues = analyze_script_structure(current_script_blocks, hat_id, sprite_name) + if script_issues: + sprite_issues.append(f"Issues in script starting with '{hat_id}':") + sprite_issues.extend([f" - {issue}" for issue in script_issues]) + else: + logger.info(f"Script starting with '{hat_id}' for sprite '{sprite_name}' passed basic verification.") + + orphaned_blocks_overall = { + block_id for block_id in all_blocks_for_sprite.keys() + if block_id not in processed_script_blocks + and not all_blocks_for_sprite[block_id].get("topLevel") + and not all_blocks_for_sprite[block_id].get("parent") + } + + if orphaned_blocks_overall: + sprite_issues.append(f"Found {len(orphaned_blocks_overall)} truly orphaned blocks not connected to any valid script: {', '.join(list(orphaned_blocks_overall)[:5])}{'...' if len(orphaned_blocks_overall) > 5 else ''}.") + + if sprite_issues: + improvement_plan["sprite_issues"][sprite_name] = sprite_issues + logger.warning(f"Verification found issues for sprite '{sprite_name}'.") + block_validation_feedback_overall.append(f"Issues for {sprite_name}:\n" + "\n".join([f"- {issue}" for issue in sprite_issues])) + state["needs_improvement"] = True + print(f"\n--- Verification Report (Issues Found for {sprite_name}) ---") + print(json.dumps({sprite_name: sprite_issues}, indent=2)) + else: + logger.info(f"Sprite '{sprite_name}' passed all verification checks.") + + # Consolidate feedback for the LLM + state["block_validation_feedback"] = "\n\n".join(block_validation_feedback_overall) + + print(F"[OVERALL IMPROVEMENT PLAN ON ITERATION {current_iteration}]: {improvement_plan}") + + if state["needs_improvement"]: + + llm_reviewer_prompt = ( + "You are an expert Scratch project reviewer. Your task is to analyze the provided " + "structural issues found in a Scratch project's sprites and suggest improvements or " + "further insights. Focus on clarity, accuracy, and actionable advice.\n\n" + "Here are the detected structural issues:\n" + f"{json.dumps(improvement_plan['sprite_issues'], indent=2)}\n\n" + "Here are the block validation feedback:\n" + f"{json.dumps(state["block_validation_feedback"], indent=2)}\n\n" + + "Please review these issues and provide a consolidated report with potential causes " + "and recommendations for fixing them. If an issue is minor or expected in certain " + "scenarios (e.g., hidden blocks for backward compatibility), please note that." + "Structure your response as a JSON object with 'review_summary' as the key, " + "containing a dictionary where keys are sprite names and values are lists of suggested improvements." + "Example:\n" + "```json\n" + "{\n" + " \"review_summary\": {\n" + " \"Sprite1\": [\n" + " \"Issue: Hat block 'abc' is not topLevel. Recommendation: Ensure all scripts start with a top-level hat block that has no parent.\",\n" + " \"Issue: Block 'xyz' has unknown opcode 'motion_nonexistent'. Recommendation: Verify the opcode against the Scratch 3.0 block reference. This might be a typo or a deprecated block.\"\n" + " ],\n" + " \"Sprite2\": [\n" + " \"Issue: Found 3 orphaned blocks. Recommendation: Reconnect these blocks to existing scripts or remove them if no longer needed.\"\n" + " ]\n" + " }\n" + "}\n" + "```" + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_reviewer_prompt}]}) + raw_review_response = response["messages"][-1].content + try: + state["review_block_feedback"] = extract_json_from_llm_response(raw_review_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_review_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT BLOCKVERFIER ]: {correction_response}") + state["review_block_feedback"] = extract_json_from_llm_response(correction_response["messages"][-1].content) + + logger.info("Agent review feedback added to the state.") + print("\n--- Agent Review Feedback ---") + print(json.dumps(state["review_block_feedback"], indent=2)) + + except Exception as e: + logger.error(f"Error invoking agent for review in BlockVerificationNode: {e}") + state["review_block_feedback"] = {"review_summary": {"Overall": [f"Error during LLM review: {e}"]}} + else: + logger.info("BlockVerificationNode completed: No issues found in any sprite blocks.") + print("\n--- Verification Report (No Issues Found) ---") + state["block_validation_feedback"] = "No issues found in sprite blocks." + state["review_block_feedback"] = {"review_summary": {"Overall": ["No issues found in sprite blocks. All good!"]}} + + # Manage iteration count based on overall needs_improvement flag + if state["needs_improvement"]: + state["iteration_count"] = current_iteration + 1 + if state["iteration_count"] >= MAX_IMPROVEMENT_ITERATIONS: + logger.warning(f"Max improvement iterations ({MAX_IMPROVEMENT_ITERATIONS}) reached for block verification. Forcing 'needs_improvement' to False.") + state["needs_improvement"] = False + state["block_validation_feedback"] += "\n(Note: Max iterations reached for block verification, stopping further improvements.)" + state["improvement_plan"] = improvement_plan + state["review_block_feedback"] = {"review_summary": {"Overall": ["No issues found in sprite blocks. All good!"]}} + logger.info("BlockVerificationNode found issues and added an improvement plan to the state.") + print("\n--- Overall Verification Report (Issues Found) ---") + print(json.dumps(improvement_plan, indent=2)) + else: + state["iteration_count"] = 0 # Reset if no more improvement needed for blocks + + logger.info(f"Block verification completed. Needs Improvement: {state['needs_improvement']}. Feedback: {state['block_validation_feedback'][:100]}...") + print("===========================================================================") + print(f"[BLOCK VERIFICATION NODE: (improvement_plan)]:{state["improvement_plan"]}") + print(f"[BLOCK VERIFICATION NODE: (review_block_feedback)]:{state["review_block_feedback"]}") + return state + +def improvement_block_builder_node(state: GameState): + logger.info("--- Running ImprovementBlockBuilderNode ---") + + project_json = state["project_json"] + targets = project_json["targets"] + + sprite_map = {target["name"]: target for target in targets if not target["isStage"]} + # Also get the Stage target + stage_target = next((target for target in targets if target["isStage"]), None) + if stage_target: + sprite_map[stage_target["name"]] = stage_target + + # Pre-load all block-catalog JSONs once + all_catalogs = [ + hat_block_data, + boolean_block_data, + c_block_data, + cap_block_data, + reporter_block_data, + stack_block_data + ] + + improvement_plan = state.get("improvement_plan", {}) + block_verification_feedback = state.get("block_validation_feedback", "no feedback") + + if not improvement_plan: + logger.warning("No improvement plan found in state. Skipping ImprovementBlockBuilderNode.") + return state + + script_y_offset = {} + script_x_offset_per_sprite = {name: 0 for name in sprite_map.keys()} + + # This is the handler which ensure if somehow json response changed it handle it.[DONOT REMOVE BELOW LOGIC] + if improvement_plan.get("improvement_overall_flow", {})=={}: + plan_data = improvement_plan.items() + else: + plan_data= improvement_plan.get("action_overall_flow", {}).items() + + for sprite_name, sprite_improvements_data in plan_data: + if sprite_name in sprite_map: + current_sprite_target = sprite_map[sprite_name] + if "blocks" not in current_sprite_target: + current_sprite_target["blocks"] = {} + + if sprite_name not in script_y_offset: + script_y_offset[sprite_name] = 0 + + for plan_entry in sprite_improvements_data.get("plans", []): + event_opcode = plan_entry["event"] # This is now expected to be an opcode + logic_sequence = plan_entry["logic"] # This is the semicolon-separated string + + # Extract the new opcode lists from the plan_entry + motion_opcodes = plan_entry.get("motion", []) + control_opcodes = plan_entry.get("control", []) + operator_opcodes = plan_entry.get("operator", []) + sensing_opcodes = plan_entry.get("sensing", []) + looks_opcodes = plan_entry.get("looks", []) + sound_opcodes = plan_entry.get("sound", []) + events_opcodes = plan_entry.get("events", []) + data_opcodes = plan_entry.get("data", []) + + needed_opcodes = ( + motion_opcodes + control_opcodes + operator_opcodes + + sensing_opcodes + looks_opcodes + sound_opcodes + + events_opcodes + data_opcodes + ) + needed_opcodes = list(set(needed_opcodes)) + + # 2) build filtered runtime catalog (if you still need it) + filtered_catalog = { + op: ALL_SCRATCH_BLOCKS_CATALOG[op] + for op in needed_opcodes + if op in ALL_SCRATCH_BLOCKS_CATALOG + } + + # 3) merge human-written catalog + runtime entry for each opcode + combined_blocks = {} + for op in needed_opcodes: + catalog_def = find_block_in_all(op, all_catalogs) or {} + runtime_def = ALL_SCRATCH_BLOCKS_CATALOG.get(op, {}) + # merge: catalog fields first, then runtime overrides/adds + merged = {**catalog_def, **runtime_def} + combined_blocks[op] = merged + print("Combined blocks for this script:", json.dumps(combined_blocks, indent=2)) + + llm_block_generation_prompt = ( + f"You are an AI assistant generating Scratch 3.0 block JSON for a single script based on an improvement plan.\n" + f"The current sprite is '{sprite_name}'.\n" + f"The specific script to generate blocks for is for the event with opcode '{event_opcode}'.\n" + f"The sequential logic to implement is:\n" + f" Logic: {logic_sequence}\n\n" + f"**Based on the planning, the following Scratch block opcodes are expected to be used to implement this logic. Focus on using these specific opcodes where applicable, and refer to the ALL_SCRATCH_BLOCKS_CATALOG for their full structure and required inputs/fields:**\n" + f"Here is the comprehensive catalog of required Scratch 3.0 blocks:\n" + f"```json\n{json.dumps(combined_blocks, indent=2)}\n```\n\n" + f"Here is feedback ans suggetion you should take care of:\n" + f" suggestion:{block_verification_feedback}\n\n" + f"Current Scratch project JSON for this sprite (for context, its existing blocks if any, though you should generate a new script entirely if this is a hat block):\n" + f"```json\n{json.dumps(current_sprite_target, indent=2)}\n```\n\n" + f"**Instructions for generating the block JSON (EXTREMELY IMPORTANT - FOLLOW THESE EXAMPLES PRECISELY):**\n" + f"1. **Start with the event block (Hat Block):** This block's `topLevel` should be `true` and `parent` should be `null`. Its `x` and `y` coordinates should be set (e.g., `x: 0, y: 0` or reasonable offsets for multiple scripts).\n" + f"2. **Generate a sequence of connected blocks:** For each block, generate a **unique block ID** (e.g., 'myBlockID123').\n" + f"3. **Link blocks correctly:**\n" + f" * Use the `next` field to point to the ID of the block directly below it in the stack.\n" + f" * Use the `parent` field to point to the ID of the block directly above it in the stack.\n" + f" * For the last block in a stack, `next` should be `null`.\n" + f"4. **Handle `inputs` for parameters and substacks (CRITICAL DETAIL - PAY CLOSE ATTENTION TO EXAMPLES):**\n" + f" * The value for any key within the `inputs` dictionary MUST always be an **array** of two elements: `[type_code, value_or_block_id]`.\n" + f" * **STRICTLY FORBIDDEN MISTAKE:** DO NOT put an array like `[\"num\", \"value\"]` or `[\"_edge_\", null]` directly as the `value_or_block_id`. This is the source of past errors.\n" + f" * The `type_code` (first element of the array) indicates the nature of the input:\n" + f" * `1`: For a primitive value or a shadow block ID (e.g., a number, string, boolean, or reference to a shadow block). This is the most common type for direct values.\n" + f" * `2`: For a block ID where another block is directly plugged into this input (e.g., an operator block connected to an `if` condition, or the first block of a C-block's substack).\n" + f" * **Correct Example for Numerical Input (e.g., for `motion_movesteps` STEPS, or `motion_gotoxy` X/Y):**\n" + f" If you need to input the number `10` into a block, you MUST create a separate `math_number` shadow block for it, and then reference its ID.\n" + f" ```json\n" + f" // Main block using the number\n" + f" \"mainBlockID\": {{\n" + f" \"opcode\": \"motion_movesteps\",\n" + f" \"inputs\": {{\n" + f" \"STEPS\": [1, \"shadowNumID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the number '10'\n" + f" \"shadowNumID\": {{\n" + f" \"opcode\": \"math_number\",\n" + f" \"fields\": {{\n" + f" \"NUM\": [\"10\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"mainBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for Dropdown/Menu Input (e.g., `sensing_touchingobject` with 'edge'):**\n" + f" If you need to select 'edge' for the `touching ()?` block, you MUST create a separate `sensing_touchingobjectmenu` shadow block and reference its ID.\n" + f" ```json\n" + f" // Main block using the dropdown selection\n" + f" \"touchingBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\",\n" + f" \"inputs\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [1, \"shadowEdgeMenuID\"]\n" + f" }},\n" + f" // ... other fields\n" + f" }},\n" + f"\n" + f" // The separate shadow block for the 'edge' menu option\n" + f" \"shadowEdgeMenuID\": {{\n" + f" \"opcode\": \"sensing_touchingobjectmenu\",\n" + f" \"fields\": {{\n" + f" \"TOUCHINGOBJECTMENU\": [\"_edge_\", null]\n" + f" }},\n" + f" \"shadow\": true,\n" + f" \"parent\": \"touchingBlockID\",\n" + f" \"topLevel\": false\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block (e.g., `control_forever`):**\n" + f" The `control_forever` block MUST have a `SUBSTACK` input pointing to the first block inside its loop. The value for `SUBSTACK` must be an array: `[2, \"FIRST_BLOCK_IN_FOREVER_LOOP_ID\"]`.\n" + f" ```json\n" + f" \"foreverBlockID\": {{\n" + f" \"opcode\": \"control_forever\",\n" + f" \"inputs\": {{\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideForeverID\"]\n" + f" }},\n" + f" \"next\": null,\n" + f" \"parent\": \"blockAboveForeverID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"firstBlockInsideForeverID\": {{\n" + f" // ... definition of the first block inside the forever loop\n" + f" \"parent\": \"foreverBlockID\",\n" + f" \"next\": \"secondBlockInsideForeverID\" // if there's another block\n" + f" }}\n" + f" ```\n" + f" * **Correct Example for C-block with condition (e.g., `control_if`):**\n" + f" The `control_if` block MUST have a `CONDITION` input (typically `type_code: 1` referencing a boolean reporter block) and a `SUBSTACK` input (`type_code: 2` referencing the first block inside the if-body).\n" + f" ```json\n" + f" \"ifBlockID\": {{\n" + f" \"opcode\": \"control_if\",\n" + f" \"inputs\": {{\n" + f" \"CONDITION\": [1, \"conditionBlockID\"],\n" + f" \"SUBSTACK\": [2, \"firstBlockInsideIfID\"]\n" + f" }},\n" + f" \"next\": \"blockAfterIfID\",\n" + f" \"parent\": \"blockAboveIfID\",\n" + f" \"shadow\": false,\n" + f" \"topLevel\": false\n" + f" }},\n" + f" \"conditionBlockID\": {{\n" + f" \"opcode\": \"sensing_touchingobject\", // Example condition block\n" + f" // ... definition for condition block, parent should be \"ifBlockID\"\n" + f" }},\n" + f" \"firstBlockInsideIfID\": {{\n" + f" // ... definition of the first block inside the if body\n" + f" \"parent\": \"ifBlockID\",\n" + f" \"next\": null // or next block if more\n" + f" }}\n" + f" ```\n" + f"5. **Define ALL Shadow Blocks Separately (THIS IS ESSENTIAL):** Every time a block's input requires a number, string literal, or a selection from a dropdown/menu, you MUST define a **separate block entry** in the top-level blocks dictionary for that shadow. Each shadow block MUST have `\"shadow\": true` and `\"topLevel\": false`.\n" + f"6. **`fields` for direct dropdown values/text:** Use the `fields` dictionary ONLY IF the block directly embeds a dropdown value or text field without an `inputs` connection (e.g., the `KEY_OPTION` field within the `event_whenkeypressed` shadow block itself, or the `variable` field on a `data_setvariableto` block). Example: `\"fields\": {{\"KEY_OPTION\": [\"space\", null]}}`.\n" + f"7. **`topLevel: true` for hat blocks:** Only the starting (hat) block of a script should have `\"topLevel\": true`.\n" + f"8. **Ensure Unique Block IDs:** Every block you generate (main blocks and shadow blocks) must have a unique ID within the entire script's block dictionary.\n" + f"9. **Strictly Use Catalog Opcodes:** You MUST only use `opcode` values that are present in the provided `ALL_SCRATCH_BLOCKS_CATALOG`. Do NOT use unlisted opcodes like `motion_jump`.\n" + f"10. **Return ONLY the JSON object representing all the blocks for THIS SINGLE SCRIPT.** Do NOT wrap it in a 'blocks' key or the full project JSON. The output should be a dictionary where keys are block IDs and values are block definitions." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_block_generation_prompt}]}) + raw_response = response["messages"][-1].content + logger.info(f"Raw response from LLM [ImprovementBlockBuilderNode - {sprite_name} - {event_opcode}]: {raw_response[:500]}...") + try: + generated_blocks = extract_json_from_llm_response(raw_response) + except json.JSONDecodeError: + logger.error("Failed to extract JSON from LLM response. Attempting to correct the response.") + # Use the JSON resolver agent to fix the response + correction_prompt = ( + "Correct the following JSON respose to ensure it follows proper schema rules:\n\n" + "Take care of following instruction carefully:\n" + "1. Do **NOT** put anything inside the json just correct and resolve the issues inside the json.\n" + "2. Do **NOT** add any additional text, comments, or explanations.\n" + "3. Just response correct the json where you find and put the same content as it is.\n" + "Here is the raw response:\n\n" + f"raw_response: {raw_response}\n\n" + ) + correction_response = agent_json_resolver.invoke({"messages": [{"role": "user", "content": correction_prompt}]}) + print(f"[JSON CORRECTOR RESPONSE AT IMPROVEMENTBLOCKBUILDER ]: {correction_response}") + generated_blocks = extract_json_from_llm_response(correction_response["messages"][-1].content) + + if "blocks" in generated_blocks and isinstance(generated_blocks["blocks"], dict): + logger.warning(f"LLM returned nested 'blocks' key for {sprite_name}. Unwrapping.") + generated_blocks = generated_blocks["blocks"] + + # Update block positions for top-level script + for block_id, block_data in generated_blocks.items(): + if block_data.get("topLevel"): + block_data["x"] = script_x_offset_per_sprite.get(sprite_name, 0) + block_data["y"] = script_y_offset[sprite_name] + script_y_offset[sprite_name] += 150 # Increment for next script + + current_sprite_target["blocks"].update(generated_blocks) + logger.info(f"Improvement blocks added for sprite '{sprite_name}', script '{event_opcode}' by ImprovementBlockBuilderNode.") + except Exception as e: + logger.error(f"Error generating blocks for sprite '{sprite_name}', script '{event_opcode}': {e}") + raise + + state["project_json"] = project_json # Update the state with the modified project_json + logger.info("Updated project JSON with improvement nodes.") + print("Updated project JSON with improvement nodes:", json.dumps(project_json, indent=2)) # Print for direct visibility + return state # Return the state object + +#temporarry time delay for handling TPM issue +import time +def delay_for_tpm_node(state: GameState): + logger.info("--- Running DelayForTPMNode ---") + time.sleep(80) # Adjust the delay as needed + logger.info("Delay completed.") + return state + + +# Build the LangGraph workflow +workflow = StateGraph(GameState) + +# Add all nodes to the workflow +workflow.add_node("parse_query", parse_query_and_set_initial_positions) +workflow.add_node("game_description", game_description_node) +workflow.add_node("time_delay_1", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_2", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_3", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_4", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("time_delay_5", delay_for_tpm_node) # this is a temporary node to handle TPM issues +workflow.add_node("initial_plan_build", overall_planner_node) # High-level planning node +workflow.add_node("plan_verifier", plan_verification_node) # Verifies the high-level plan +workflow.add_node("refined_planner", refined_planner_node) # Refines the action plan +workflow.add_node("block_builder", overall_block_builder_node) # Builds blocks from a plan +workflow.add_node("block_verifier", block_verification_node) # Verifies the generated blocks +workflow.add_node("improved_block_builder", improvement_block_builder_node) # For specific block-level improvements + +# Set the entry point +workflow.set_entry_point("game_description") + +# Define the standard initial flow +workflow.add_edge("game_description", "parse_query") +workflow.add_edge("parse_query", "time_delay_1") +workflow.add_edge("time_delay_1", "initial_plan_build") +workflow.add_edge("initial_plan_build", "time_delay_5") +workflow.add_edge("time_delay_5", "plan_verifier") +# Define the conditional logic after plan_verifier (for high-level plan issues) +def decide_next_step_after_plan_verification(state: GameState): + if state.get("needs_improvement", False): + # If the plan needs refinement, go to the refined_planner + return "refined_planner" + else: + # If the plan is good, proceed to building blocks from this plan + return "block_builder" + +workflow.add_conditional_edges( + "plan_verifier", + decide_next_step_after_plan_verification, + { + "refined_planner": "refined_planner", # Path if plan needs refinement + "block_builder": "block_builder" # Path if plan is approved, proceeds to block building + } +) + +# --- CRITICAL CHANGE FOR THE PLAN REFINEMENT LOOP --- +# After refining the plan, it should go back to plan_verifier for re-verification. +workflow.add_edge("refined_planner", "time_delay_2") +workflow.add_edge("time_delay_2", "plan_verifier") # This closes the loop for plan refinement and re-verification. +# Note: The original code had workflow.add_edge("time_delay", "block_builder") here, +# but after refined_planner -> time_delay -> plan_verifier, the decision is made by plan_verifier. +# So, this edge might be redundant or incorrect depending on the desired flow. +# Assuming the intent is for plan_verifier to always decide the next step. + +# After blocks are built, they need to be verified +#workflow.add_edge("time_delay_3", "block_builder") +workflow.add_edge("block_builder", "time_delay_3") +workflow.add_edge("time_delay_3", "block_verifier") + +# Define the conditional logic after block_verifier (for generated blocks issues) +def decide_after_block_verification(state: GameState): + if state.get("needs_improvement", False): + # If blocks need improvement, go to improved_block_builder. + # This assumes improved_block_builder handles specific block-level fixes. + return "improved_block_builder" + else: + # If blocks are good, end the workflow + return "END" + +workflow.add_conditional_edges( + "block_verifier", + decide_after_block_verification, + { + "improved_block_builder": "improved_block_builder", # Path if blocks need improvement + "END": END # Path if blocks are good + } +) + +# Create the loop: If blocks improved, re-verify them +#workflow.add_edge("time_delay_4", "improved_block_builder") +workflow.add_edge("improved_block_builder", "time_delay_4") +workflow.add_edge("time_delay_4", "block_verifier") + +# Compile the workflow graph +app_graph = workflow.compile() + +# # Build the LangGraph workflow +# workflow = StateGraph(GameState) + +# # Add all nodes to the workflow +# workflow.add_node("parse_query", parse_query_and_set_initial_positions) +# workflow.add_node("game_description", game_description_node) +# workflow.add_node("initial_plan_build", overall_planner_node) # High-level planning node +# workflow.add_node("plan_verifier", plan_verification_node) # Verifies the high-level plan +# workflow.add_node("refined_planner", refined_planner_node) # Refines the action plan +# workflow.add_node("block_builder", overall_block_builder_node) # Builds blocks from a plan +# workflow.add_node("block_verifier", block_verification_node) # Verifies the generated blocks +# workflow.add_node("improved_block_builder", improvement_block_builder_node) # For specific block-level improvements + +# # Set the entry point +# workflow.set_entry_point("game_description") + +# # Define the standard initial flow +# workflow.add_edge("game_description", "parse_query") +# workflow.add_edge("parse_query", "initial_plan_build") +# workflow.add_edge("initial_plan_build", "plan_verifier") + +# # Define the conditional logic after plan_verifier (for high-level plan issues) +# def decide_next_step_after_plan_verification(state: GameState): +# if state.get("needs_improvement", False): +# # If the plan needs refinement, go to the refined_planner +# return "refined_planner" +# else: +# # If the plan is good, proceed to building blocks from this plan +# return "block_builder" + +# workflow.add_conditional_edges( +# "plan_verifier", +# decide_next_step_after_plan_verification, +# { +# "refined_planner": "refined_planner", # Path if plan needs refinement +# "block_builder": "block_builder" # Path if plan is approved, proceeds to block building +# } +# ) + +# # --- CRITICAL CHANGE FOR THE PLAN REFINEMENT LOOP --- +# # After refining the plan, it should go back to plan_verifier for re-verification. +# workflow.add_edge("refined_planner", "plan_verifier") # This closes the loop for plan refinement and re-verification. + +# # After blocks are built, they need to be verified +# workflow.add_edge("block_builder", "block_verifier") + +# # Define the conditional logic after block_verifier (for generated blocks issues) +# def decide_after_block_verification(state: GameState): +# if state.get("needs_improvement", False): +# # If blocks need improvement, go to improved_block_builder. +# # This assumes improved_block_builder handles specific block-level fixes. +# return "improved_block_builder" +# else: +# # If blocks are good, end the workflow +# return "END" + +# workflow.add_conditional_edges( +# "block_verifier", +# decide_after_block_verification, +# { +# "improved_block_builder": "improved_block_builder", # Path if blocks need improvement +# "END": END # Path if blocks are good +# } +# ) + +# # Create the loop: If blocks improved, re-verify them +# workflow.add_edge("improved_block_builder", "block_verifier") + +# # Compile the workflow graph +# app_graph = workflow.compile() + + +from IPython.display import Image, display +png_bytes = app_graph.get_graph().draw_mermaid_png() +with open("langgraph_workflow.png", "wb") as f: + f.write(png_bytes) + +### Modified `generate_game` Endpoint + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description", "") + backdrops = payload.get("backdrops", []) + sprites = payload.get("sprites", []) + + logger.info(f"Starting game generation for description: '{desc}'") + + # 1) Initial skeleton generation + project_skeleton = { + "targets": [ + { + "isStage": True, + "name":"Stage", + "objName": "Stage", # ADDED THIS FOR THE STAGE + "variables":{}, # This must be a dict: { "var_id": ["var_name", value] } + "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + for idx, s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project_skeleton["targets"].append({ + "isStage": False, + "name": s["name"], + "objName": s["name"], # objName is also important for sprites + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + logger.info("Initial project skeleton created.") + + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + + try: + os.makedirs(project_folder, exist_ok=True) + # Save initial skeleton and copy assets + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project_skeleton, f, indent=2) + logger.info(f"Initial project skeleton saved to {project_json_path}") + + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source backdrop asset not found: {src_path}") + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source sprite asset not found: {src_path}") + logger.info("Assets copied to project folder.") + + # Initialize the state for LangGraph as a dictionary matching the TypedDict structure + initial_state_dict: GameState = { # Type hint for clarity + "project_json": project_skeleton, + "description": desc, + "project_id": project_id, + "sprite_initial_positions": {}, + "action_plan": {}, + #"behavior_plan": {}, + "improvement_plan": {}, + "needs_improvement": False, + "plan_validation_feedback": {}, + "iteration_count": 0, + "review_block_feedback": {} + } + + # Run the LangGraph workflow with the dictionary input. + final_state_dict: GameState = app_graph.invoke(initial_state_dict) + + # Access elements from the final_state_dict + final_project_json = final_state_dict['project_json'] + + # Save the *final* filled project JSON, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(final_project_json, f, indent=2) + logger.info(f"Final project JSON saved to {project_json_path}") + + return jsonify({"message": "Game generated successfully", "project_id": project_id}) + + except Exception as e: + logger.error(f"Error during game generation workflow for project ID {project_id}: {e}", exc_info=True) + return jsonify(error=f"Error generating game: {e}"), 500 + + +if __name__=="__main__": + logger.info("Starting Flask application...") + app.run(debug=True,port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/sys_scratch_app.py b/scratch_VLM/scratch_agent/sys_scratch_app.py new file mode 100644 index 0000000000000000000000000000000000000000..50a6064aa8a97e4df4a508af7fb2921a222a98ec --- /dev/null +++ b/scratch_VLM/scratch_agent/sys_scratch_app.py @@ -0,0 +1,591 @@ +from flask import Flask, request, jsonify, render_template, send_from_directory +from langgraph.prebuilt import create_react_agent +from langchain_groq import ChatGroq +from PIL import Image +import os, json, re +import shutil +import uuid +from langgraph.graph import StateGraph, END +import logging +from typing import TypedDict # Import TypedDict + +# --- Configure logging --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +# --- LLM / Vision model setup --- +os.environ["GROQ_API_KEY"] = os.getenv("GROQ_API_KEY", "default_key_or_placeholder") + +groq_key = os.environ["GROQ_API_KEY"] +if not groq_key or groq_key == "default_key_or_placeholder": + logger.critical("GROQ_API_KEY environment variable is not set or invalid. Please set it to proceed.") + raise ValueError("GROQ_API_KEY environment variable is not set or invalid.") + +llm = ChatGroq( + model="deepseek-r1-distill-llama-70b", + temperature=0.0, +) + +# Refined SYSTEM_PROMPT with more explicit Scratch JSON rules, especially for variables +SYSTEM_PROMPT = """ +You are an expert AI assistant named GameScratchAgent, specialized in generating and modifying Scratch-VM 3.x game project JSON. +Your core task is to process game descriptions and existing Scratch JSON structures, then produce or update JSON segments accurately. +You possess deep knowledge of Scratch 3.0 project schema. When generating or modifying the `blocks` section, pay extremely close attention to the following: + +**Scratch Target and Block Schema Rules:** +1. **Target Structure:** + * Every target (Stage or Sprite) must have an `objName` property. For the Stage, it's typically `"objName": "Stage"`. For sprites, it's their name (e.g., `"objName": "Cat"`). + * `variables` dictionary: This dictionary maps unique variable IDs to arrays `[variable_name, initial_value]`. + Example: `"variables": { "myVarId123": ["score", 0] }` + +2. **Block Structure:** Every block must have `opcode`, `parent`, `next`, `inputs`, `fields`, `shadow`, `topLevel`. + * `parent`: ID of the block it's stacked on (or `null` for top-level). + * `next`: ID of the block directly below it (or `null` for end of stack). + * `topLevel`: `true` if it's a hat block or a standalone block, `false` otherwise. + * `shadow`: `true` if it's a shadow block (e.g., a default number input), `false` otherwise. + +3. **`inputs` vs. `fields`:** This is critical. + * **`inputs`**: Used for blocks or values *plugged into* a block (e.g., condition for an `if` block, target for `go to`). Values are **arrays**. + * `[1, ]`: Represents a Reporter or Boolean block *plugged in* (e.g., `operator_equals`). + * `[1, [, ""]]`: For a shadow block with a literal value (e.g., `[1, ["num", "10"]]` for a number input). The `type` can be "num", "str", "bool", "colour", etc. + * `[2, ]`: For a C-block's substack (e.g., `SUBSTACK` in `control_if`). + * **`fields`**: Used for dropdown menus or direct internal values *within* a block (e.g., key selected in `when key pressed`, variable name in `set variable`). Values are always **arrays** `["", null]`. + * Example for `event_whenkeypressed`: `"KEY": ["space", null]` + * Example for `data_setvariableto`: `"VARIABLE": ["score", null]` + +4. **Unique IDs:** All block IDs, variable IDs, and list IDs must be unique strings (e.g., "myBlock123", "myVarId456"). Do NOT use placeholder strings like "block_id_here". + +5. **No Nested `blocks` Dictionary:** The `blocks` dictionary should only appear once per `target` (sprite/stage). Do NOT nest a `blocks` dictionary inside an individual block definition. Blocks that are part of a substack (like inside an `if` or `forever` loop) are linked via the `SUBSTACK` input, where the input value is `[2, "ID_of_first_block_in_substack"]`. + +6. **Asset Properties:** `assetId`, `md5ext`, `bitmapResolution`, `rotationCenterX`/`rotationCenterY` should be correct for costumes/sounds. + +**When responding, you must:** +1. **Output ONLY the complete, valid JSON object.** Do not include markdown, commentary, or conversational text. +2. Ensure every generated block ID, input, field, and variable declaration strictly follows the Scratch 3.0 schema and the rules above. +3. If presented with partial or problematic JSON, aim to complete or correct it based on the given instructions and your schema knowledge. +""" + +agent = create_react_agent( + model=llm, + tools=[], # No specific tools are defined here, but could be added later + prompt=SYSTEM_PROMPT +) + +# --- Serve the form --- +@app.route("/", methods=["GET"]) +def index(): + return render_template("index4.html") + +# --- List static assets for the front-end --- +@app.route("/list_assets", methods=["GET"]) +def list_assets(): + bdir = os.path.join(app.static_folder, "assets", "backdrops") + sdir = os.path.join(app.static_folder, "assets", "sprites") + backdrops = [] + sprites = [] + try: + if os.path.isdir(bdir): + backdrops = [f for f in os.listdir(bdir) if f.lower().endswith(".svg")] + if os.path.isdir(sdir): + sprites = [f for f in os.listdir(sdir) if f.lower().endswith(".svg")] + logger.info("Successfully listed static assets.") + except Exception as e: + logger.error(f"Error listing static assets: {e}") + return jsonify({"error": "Failed to list assets"}), 500 + return jsonify(backdrops=backdrops, sprites=sprites) + +# --- Helper: build costume entries --- +def make_costume_entry(folder, filename, name): + asset_id = filename.rsplit(".", 1)[0] + entry = { + "name": name, + "bitmapResolution": 1, + "dataFormat": filename.rsplit(".", 1)[1], + "assetId": asset_id, + "md5ext": filename, + } + + path = os.path.join(app.static_folder, "assets", folder, filename) + ext = filename.lower().endswith(".svg") + + if ext: + if folder == "backdrops": + entry["rotationCenterX"] = 240 + entry["rotationCenterY"] = 180 + else: + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + else: + try: + img = Image.open(path) + w, h = img.size + entry["rotationCenterX"] = w // 2 + entry["rotationCenterY"] = h // 2 + except Exception as e: + logger.warning(f"Could not determine image dimensions for {filename}: {e}. Setting center to 0,0.") + entry["rotationCenterX"] = 0 + entry["rotationCenterY"] = 0 + + return entry + +# --- New endpoint to fetch project.json --- +@app.route("/get_project/", methods=["GET"]) +def get_project(project_id): + project_folder = os.path.join("generated_projects", project_id) + project_json_path = os.path.join(project_folder, "project.json") + + try: + if os.path.exists(project_json_path): + logger.info(f"Serving project.json for project ID: {project_id}") + return send_from_directory(project_folder, "project.json", as_attachment=True, download_name=f"{project_id}.json") + else: + logger.warning(f"Project JSON not found for ID: {project_id}") + return jsonify({"error": "Project not found"}), 404 + except Exception as e: + logger.error(f"Error serving project.json for ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve project"}), 500 + +# --- New endpoint to fetch assets --- +@app.route("/get_asset//", methods=["GET"]) +def get_asset(project_id, filename): + project_folder = os.path.join("generated_projects", project_id) + asset_path = os.path.join(project_folder, filename) + + try: + if os.path.exists(asset_path): + logger.info(f"Serving asset '{filename}' for project ID: {project_id}") + return send_from_directory(project_folder, filename) + else: + logger.warning(f"Asset '{filename}' not found for project ID: {project_id}") + return jsonify({"error": "Asset not found"}), 404 + except Exception as e: + logger.error(f"Error serving asset '{filename}' for project ID {project_id}: {e}") + return jsonify({"error": "Failed to retrieve asset"}), 500 + + +### LangGraph Workflow Definition + +# Define a state for your graph using TypedDict +class GameState(TypedDict): + project_json: dict + description: str + project_id: str + parsed_game_elements: dict # Add this as well, as it's part of your state + +# Helper function to extract JSON from LLM response +def extract_json_from_llm_response(raw_response: str) -> dict: + m = re.search(r"```json\n(.+?)```", raw_response, re.S) + body = m.group(1) if m else raw_response + try: + json_data = json.loads(body) + logger.debug("Successfully extracted and parsed JSON from LLM response.") + return json_data + except json.JSONDecodeError as e: + logger.error(f"LLM did not return valid JSON. Error: {e}\nRaw response: {raw_response}") + raise ValueError(f"LLM did not return valid JSON: {e}\nRaw response: {raw_response}") + except Exception as e: + logger.error(f"An unexpected error occurred during JSON extraction: {e}\nRaw response: {raw_response}") + raise + +# Node 1: Analysis User Query and Initial Positions +def parse_query_and_set_initial_positions(state: GameState): + logger.info("--- Running ParseQueryNode ---") + llm_query_prompt = ( + f"Analyze the user's game description: '{state['description']}'. " + f"Determine appropriate initial x, y positions for each sprite based on the game concept. " + f"Identify key game elements and actions (e.g., 'jumps', 'obstacle', 'score', 'reset'). " + f"Return the **full updated Scratch project JSON** where 'x' and 'y' properties for each sprite are set under its target object. " + f"Additionally, include a new top-level key 'parsed_game_elements' with details like " + f"'sprite_initial_positions' (a dictionary mapping sprite name to {{'x': int, 'y': int}}), " + f"'main_actions' (a dictionary mapping sprite name to its primary action, e.g., 'jumping'), " + f"'behaviors' (e.g., 'collision_detection', 'movement'), " + f"'scoring_goals' (e.g., 'jump_over_obstacle'), and 'reset_triggers' (e.g., 'key_r').\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"**Example for 'jumping cat game over obstacle':**\n" + f"The 'parsed_game_elements' should look like: " + f"{{ \"sprite_initial_positions\": {{ \"Cat\": {{ \"x\": -160, \"y\": -110 }}, \"Obstacle\": {{ \"x\": 240, \"y\": -135 }} }}, \"main_actions\": {{ \"Cat\": \"jumping\", \"Obstacle\": \"gliding towards cat\" }}, \"behaviors\": \"collision_detection\", \"scoring_goals\": \"jump_over_obstacle\", \"reset_triggers\": \"key_r\" }}\n" + f"Output **ONLY the complete, valid JSON object**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_query_prompt}]}) + raw_response = response["messages"][-1].content + updated_data = extract_json_from_llm_response(raw_response) + + new_project_json = {k: v for k, v in updated_data.items() if k != "parsed_game_elements"} + new_parsed_game_elements = updated_data.get("parsed_game_elements", {}) + + logger.info("Initial positions and parsed elements updated by ParseQueryNode.") + return {"project_json": new_project_json, "parsed_game_elements": new_parsed_game_elements} + except ValueError as e: + logger.error(f"ValueError in ParseQueryNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ParseQueryNode: {e}") + raise + +# Node 2: Action Node Builder +def build_action_nodes(state: GameState): + logger.info("--- Running ActionNodeBuilder ---") + llm_action_prompt = ( + f"Given the game description '{state['description']}' and parsed elements: {json.dumps(state['parsed_game_elements'], indent=2)}, " + f"your task is to populate the 'blocks' section for each relevant sprite in the Scratch project JSON. " + f"Focus on implementing the **main actions** identified (e.g., cat jumping, obstacle gliding). " + f"**Strictly follow these Scratch 3.0 block schema rules for `inputs` and `fields`:**\n" + f"- For `event_whenkeypressed`, the key is a `field`: `\"KEY\": [\"space\", null]`. (e.g., for 'space' key)\n" + f"- For `motion_changeyby` or `motion_changexby`, the value is a `field`: `\"DY\": [\"10\", null]` or `\"DX\": [\"-5\", null]`.\n" + f"- For `sound_play`, the sound name is a `field`: `\"SOUND_MENU\": [\"pop\", null]`.\n" + f"- For `control_waituntil` or `control_if`, the `CONDITION` is an `input` that links to a boolean reporter block: `\"CONDITION\": [2, \"\"]`.\n" + f"- For `motion_gotoxy`, `X` and `Y` are `inputs` that link to number reporter shadow blocks: `\"X\": [1, [\"num\", \"-160\"]], \"Y\": [1, [\"num\", \"-110\"]]`.\n" + f"- For `motion_goto`, if going to a sprite, `TO` is a `field`: `\"TO\": [\"Cat\", null]`. If going to specific coordinates, use `motion_gotoxy`.\n" + f"- For any other block input that takes a numerical or string value directly (not a linked block), use `[1, [\"num\", \"value\"]]` or `[1, [\"str\", \"value\"]]` for shadow blocks within inputs, or `[\"value\", null]` for fields.\n" + f"- Ensure block IDs are unique (e.g., generate new unique IDs for each block like 'jumpBlock1', 'glideBlockA').\n" + f"- Correctly link `parent` and `next` properties. `topLevel` should be `true` for hat blocks.\n" + f"- **Crucially, the `blocks` dictionary should ONLY exist at the target level. Do NOT nest `blocks` inside individual block definitions. For C-blocks (like `control_if`, `control_forever`), use the `SUBSTACK` input: `\"SUBSTACK\": [2, \"ID_of_first_block_inside_substack\"]`.\n" + f"The current project JSON to modify is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output **ONLY the complete, valid and updated JSON object**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_action_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Action blocks added by ActionNodeBuilder.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ActionNodeBuilder: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ActionNodeBuilder: {e}") + raise + +# Node 3: Behaviour Collector (Sensing/Collision) +def collect_behaviors(state: GameState): + logger.info("--- Running BehaviorCollector ---") + llm_behavior_prompt = ( + f"Based on the game description '{state['description']}' and parsed elements: {json.dumps(state['parsed_game_elements'], indent=2)}, " + f"add collision detection and other behavioral logic to the 'blocks' section of the Scratch project JSON. " + f"This includes blocks like 'sensing_touchingobject', 'control_if', 'control_waituntil', 'control_stop' or 'event_broadcast'. " + f"**Strictly follow the Scratch 3.0 block schema rules provided in the SYSTEM_PROMPT for `inputs` and `fields`, especially for linking boolean conditions and specifying fields.**\n" + f"Example for `sensing_touchingobject`: `\"TOUCHINGOBJECTMENU\": [\"Cat\", null]` (field for sprite name).\n" + f"Example for `control_if`: `\"CONDITION\": [2, \"\"]`, and `\"SUBSTACK\": [2, \"\"]`.\n" + f"Example for `control_stop`: `\"STOP_OPTION\": [\"all\", null]`.\n" + f"Ensure all block IDs are unique and parent/next relationships are correct. Remember, no nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_behavior_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Behavior blocks added by BehaviorCollector.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in BehaviorCollector: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in BehaviorCollector: {e}") + raise + +# Node 4: Scoring Function +def add_scoring_function(state: GameState): + logger.info("--- Running ScoringNode ---") + if not state['parsed_game_elements'].get("scoring_goals"): + logger.info("No scoring goals identified, skipping scoring node.") + return {} # Return an empty dictionary if no changes are made + + llm_scoring_prompt = ( + f"Based on the game description '{state['description']}' and parsed elements: {json.dumps(state['parsed_game_elements'], indent=2)}, " + f"add a scoring mechanism to the Scratch project JSON. " + f"**Crucially, add a 'score' variable to the Stage's 'variables' dictionary.** The format for variables in `targets[].variables` is " + f"a dictionary where keys are unique variable IDs (e.g., 'scoreVarId123') and values are arrays `[\"variable_name\", initial_value]`. " + f"Example: `\"variables\": {{ \"scoreVarId123\": [\"score\", 0] }}`.\n" + f"Then, add blocks to update it (e.g., 'data_setvariableto', 'data_changevariableby') " + f"based on positive actions identified in 'scoring_goals' (e.g., jumping over an obstacle). " + f"**Strictly adhere to the Scratch 3.0 block schema rules for `inputs` and `fields`:**\n" + f"- For `data_setvariableto` or `data_changevariableby`, the variable name is a `field`: `\"VARIABLE\": [\"score\", null]`.\n" + f"- The value for `data_setvariableto` or `data_changevariableby` is an `input` linking to a shadow block: `\"VALUE\": [1, [\"num\", \"0\"]]` or `\"VALUE\": [1, [\"num\", \"10\"]]`.\n" + f"Ensure all block IDs are unique and relationships are correct. No nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_scoring_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Scoring function added by ScoringNode.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ScoringNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ScoringNode: {e}") + raise + +# Node 5: Reset Option +def add_reset_option(state: GameState): + logger.info("--- Running ResetNode ---") + if not state['parsed_game_elements'].get("reset_triggers"): + logger.info("No reset triggers identified, skipping reset node.") + return {} # Return an empty dictionary if no changes are made + + llm_reset_prompt = ( + f"Based on the game description '{state['description']}' and parsed elements: {json.dumps(state['parsed_game_elements'], indent=2)}, " + f"add a reset mechanism to the Scratch project JSON. " + f"This should typically involve an 'event_whenkeypressed' (e.g., 'r' key) or 'event_whenflagclicked' that resets sprite positions ('motion_gotoxy') " + f"and any variables (like score) to their initial states, potentially broadcasting a 'reset_game' message. " + f"**Ensure variable IDs and names are consistent with those in the Stage's 'variables' dictionary.**\n" + f"**Strictly adhere to the Scratch 3.0 block schema rules provided in the SYSTEM_PROMPT for `inputs` and `fields`:**\n" + f"- For `event_whenkeypressed`, the key is a `field`: `\"KEY\": [\"r\", null]`.\n" + f"- For `motion_gotoxy`, `X` and `Y` are `inputs` that link to number reporter shadow blocks: `\"X\": [1, [\"num\", \"-160\"]], \"Y\": [1, [\"num\", \"-110\"]]`.\n" + f"- For `data_setvariableto`, the variable name is a `field`: `\"VARIABLE\": [\"score\", null]`, and value is an `input` like `\"VALUE\": [1, [\"num\", \"0\"]]`.\n" + f"- For `event_broadcast`, the message is a `field`: `\"BROADCAST_OPTION\": [\"reset_game\", null]`.\n" + f"- Ensure all block IDs are unique and relationships are correct. No nested `blocks` dictionaries.\n" + f"The current project JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **complete, valid and updated JSON**." + ) + + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_reset_prompt}]}) + raw_response = response["messages"][-1].content + updated_project = extract_json_from_llm_response(raw_response) + logger.info("Reset option added by ResetNode.") + return {"project_json": updated_project} + except ValueError as e: + logger.error(f"ValueError in ResetNode: {e}") + raise + except Exception as e: + logger.error(f"An unexpected error occurred in ResetNode: {e}") + raise + +# Node 6: JSON Validation (and optional Refinement) +def validate_json(state: GameState): + logger.info("--- Running JSONValidatorNode ---") + validation_errors = [] + + try: + # Basic check if it's a valid JSON structure at least + json.dumps(state['project_json']) + + for i, target in enumerate(state['project_json'].get("targets", [])): + target_name = target.get('name', f'index {i}') + # Validate objName presence for all targets + if "objName" not in target: + validation_errors.append(f"Target '{target_name}' is missing 'objName'.") + + # Validate variables structure + if "variables" in target and isinstance(target["variables"], dict): + for var_id, var_data in target["variables"].items(): + if not isinstance(var_data, list) or len(var_data) != 2 or not isinstance(var_data[0], str): + validation_errors.append(f"Target '{target_name}' variable '{var_id}' has incorrect format: {var_data}. Expected [\"name\", initial_value].") + else: + validation_errors.append(f"Target '{target_name}' has missing or malformed 'variables' section.") + + + block_ids = set() + for block_id, block_data in target.get("blocks", {}).items(): + if block_id in block_ids: + validation_errors.append(f"Duplicate block ID found: '{block_id}' in target '{target_name}'.") + block_ids.add(block_id) + + # Validate inputs + for input_name, input_data in block_data.get("inputs", {}).items(): + if not isinstance(input_data, list): + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has non-list input '{input_name}': {input_data}. Should be a list.") + continue + + if len(input_data) < 2: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has malformed input '{input_name}': {input_data}. Expected at least 2 elements.") + continue + + if input_data[0] in [1, 2, 3] and isinstance(input_data[1], str) and input_data[1] not in block_ids and input_data[1] != "": + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent input block '{input_data[1]}' for input '{input_name}'.") + + # Validate fields + for field_name, field_data in block_data.get("fields", {}).items(): + if not isinstance(field_data, list) or len(field_data) < 1 or (len(field_data) > 1 and field_data[1] is not None): + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has malformed field '{field_name}': {field_data}. Should be [\"value\", null].") + + # Validate parent + if "parent" in block_data and block_data["parent"] is not None and block_data["parent"] not in block_ids: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent parent '{block_data['parent']}'.") + + # Validate next + if "next" in block_data and block_data["next"] is not None and block_data["next"] not in block_ids: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' references non-existent next block '{block_data['next']}'.") + + # Check for nested 'blocks' (common LLM error) + if "blocks" in block_data and isinstance(block_data["blocks"], dict) and len(block_data["blocks"]) > 0: + validation_errors.append(f"Block '{block_id}' in target '{target_name}' has a nested 'blocks' dictionary. Blocks should only be at the target's top level and linked via inputs (e.g., SUBSTACK).") + + + except json.JSONDecodeError as e: + validation_errors.append(f"Structural JSON error (invalid format) during validation: {e}") + except Exception as e: + validation_errors.append(f"An unexpected error occurred during JSON validation: {e}") + + if validation_errors: + error_message = "; ".join(validation_errors) + logger.warning(f"JSON validation failed: {error_message}. Attempting refinement...") + + llm_refine_prompt = ( + f"The Scratch project JSON you generated failed validation due to the following issues: {error_message}. " + f"Please review and correct the JSON, ensuring all rules from your SYSTEM_PROMPT are followed, " + f"especially regarding unique block IDs, correct structure (parent/next/inputs/fields), and valid opcodes. " + f"Pay extremely close attention to the **exact array format for inputs and fields** (e.g., `\"FIELD_NAME\": [\"value\", null]` and `\"INPUT_NAME\": [1, [\"num\", \"value\"]]` or `[2, \"linked_block_id\"]`). " + f"Crucially, ensure the `variables` section within each target is formatted as a dictionary where keys are unique IDs and values are `[\"variable_name\", initial_value]`. " + f"Also, ensure no `blocks` dictionary is nested inside a block definition; use `SUBSTACK` input for C-blocks. " + f"The current problematic JSON is:\n```json\n{json.dumps(state['project_json'], indent=2)}\n```\n" + f"Output the **corrected complete JSON**." + ) + try: + response = agent.invoke({"messages": [{"role": "user", "content": llm_refine_prompt}]}) + raw_response = response["messages"][-1].content + refined_project_json = extract_json_from_llm_response(raw_response) + logger.info("JSON refined and updated by JSONValidatorNode.") + return {"project_json": refined_project_json} + except ValueError as e_refine: + logger.error(f"Refinement also failed to return valid JSON: {e_refine}") + raise ValueError(f"JSON validation and refinement failed: {e_refine}") + except Exception as e_refine: + logger.error(f"An unexpected error occurred during JSON refinement: {e_refine}") + raise ValueError(f"JSON validation and refinement failed due to unexpected error: {e_refine}") + else: + logger.info("JSON validation successful (all checks passed).") + return {} # Return an empty dictionary if no changes are needed or made in this pass + +# Build the LangGraph workflow +workflow = StateGraph(GameState) + +workflow.add_node("parse_query", parse_query_and_set_initial_positions) +workflow.add_node("build_actions", build_action_nodes) +workflow.add_node("collect_behaviors", collect_behaviors) +workflow.add_node("add_scoring", add_scoring_function) +workflow.add_node("add_reset", add_reset_option) +workflow.add_node("validate_json", validate_json) + +# Define the flow (edges) +workflow.set_entry_point("parse_query") +workflow.add_edge("parse_query", "build_actions") +workflow.add_edge("build_actions", "collect_behaviors") +workflow.add_edge("collect_behaviors", "add_scoring") +workflow.add_edge("add_scoring", "add_reset") +workflow.add_edge("add_reset", "validate_json") +workflow.add_edge("validate_json", END) # End of the main flow + +app_graph = workflow.compile() + +### Modified `generate_game` Endpoint + +@app.route("/generate_game", methods=["POST"]) +def generate_game(): + payload = request.json + desc = payload.get("description", "") + backdrops = payload.get("backdrops", []) + sprites = payload.get("sprites", []) + + logger.info(f"Starting game generation for description: '{desc}'") + + # 1) Initial skeleton generation + project_skeleton = { + "targets": [ + { + "isStage": True, + "name":"Stage", + "objName": "Stage", # ADDED THIS FOR THE STAGE + "variables":{}, # This must be a dict: { "var_id": ["var_name", value] } + "lists":{}, "broadcasts":{}, + "blocks":{}, "comments":{}, + "currentCostume": len(backdrops)-1 if backdrops else 0, + "costumes": [make_costume_entry("backdrops",b["filename"],b["name"]) + for b in backdrops], + "sounds": [], "volume":100,"layerOrder":0, + "tempo":60,"videoTransparency":50,"videoState":"on", + "textToSpeechLanguage": None + } + ], + "monitors": [], "extensions": [], "meta":{ + "semver":"3.0.0","vm":"11.1.0", + "agent": request.headers.get("User-Agent","") + } + } + + for idx, s in enumerate(sprites, start=1): + costume = make_costume_entry("sprites", s["filename"], s["name"]) + project_skeleton["targets"].append({ + "isStage": False, + "name": s["name"], + "objName": s["name"], # objName is also important for sprites + "variables":{}, "lists":{}, "broadcasts":{}, + "blocks":{}, + "comments":{}, + "currentCostume":0, + "costumes":[costume], + "sounds":[], "volume":100, + "layerOrder": idx+1, + "visible":True, "x":0,"y":0,"size":100,"direction":90, + "draggable":False, "rotationStyle":"all around" + }) + + logger.info("Initial project skeleton created.") + + project_id = str(uuid.uuid4()) + project_folder = os.path.join("generated_projects", project_id) + + try: + os.makedirs(project_folder, exist_ok=True) + # Save initial skeleton and copy assets + project_json_path = os.path.join(project_folder, "project.json") + with open(project_json_path, "w") as f: + json.dump(project_skeleton, f, indent=2) + logger.info(f"Initial project skeleton saved to {project_json_path}") + + for b in backdrops: + src_path = os.path.join(app.static_folder, "assets", "backdrops", b["filename"]) + dst_path = os.path.join(project_folder, b["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source backdrop asset not found: {src_path}") + for s in sprites: + src_path = os.path.join(app.static_folder, "assets", "sprites", s["filename"]) + dst_path = os.path.join(project_folder, s["filename"]) + if os.path.exists(src_path): + shutil.copy(src_path, dst_path) + else: + logger.warning(f"Source sprite asset not found: {src_path}") + logger.info("Assets copied to project folder.") + + # Initialize the state for LangGraph as a dictionary matching the TypedDict structure + initial_state_dict: GameState = { # Type hint for clarity + "project_json": project_skeleton, + "description": desc, + "project_id": project_id, + "parsed_game_elements": {} # Initialize this as well + } + + # Run the LangGraph workflow with the dictionary input. + final_state_dict: GameState = app_graph.invoke(initial_state_dict) + + # Access elements from the final_state_dict + final_project_json = final_state_dict['project_json'] + + # Save the *final* filled project JSON, overwriting the skeleton + with open(project_json_path, "w") as f: + json.dump(final_project_json, f, indent=2) + logger.info(f"Final project JSON saved to {project_json_path}") + + return jsonify({"message": "Game generated successfully", "project_id": project_id}) + + except Exception as e: + logger.error(f"Error during game generation workflow for project ID {project_id}: {e}", exc_info=True) + return jsonify(error=f"Error generating game: {e}"), 500 + + +if __name__=="__main__": + logger.info("Starting Flask application...") + app.run(debug=True,port=5000) \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/templates/index.html b/scratch_VLM/scratch_agent/templates/index.html new file mode 100644 index 0000000000000000000000000000000000000000..3edf9aae07989a8eff7392b38c1ef8e01b89ad11 --- /dev/null +++ b/scratch_VLM/scratch_agent/templates/index.html @@ -0,0 +1,775 @@ + + + + + + + Agent Chat + + + + + + + + + + +
+
+ + {% for chat in chat_history %} +
+
+
{{ chat.user }}
+
+
+
+
{{ chat.assistant | safe }}
+
+ {% endfor %} +
+ + + +
+
+ +
+ + +
+
+
+ +
+ +
+
Agent Chat v1.0 · © 2023
+
+ + +
+
+

Logs

+ +
+
+ +
+
+ + + + + + + diff --git a/scratch_VLM/scratch_agent/templates/index2.html b/scratch_VLM/scratch_agent/templates/index2.html new file mode 100644 index 0000000000000000000000000000000000000000..aa0e28a480d0fa4fdab23421ea6b694efe77a1bd --- /dev/null +++ b/scratch_VLM/scratch_agent/templates/index2.html @@ -0,0 +1,289 @@ + + + + + + Agent Chat - Idea Generator + + + + +
+

Agent Chat - Idea Generator

+
+ +
+ +
+ +
+ +
+ +
+ + + + + +
+
+
+ + + + diff --git a/scratch_VLM/scratch_agent/templates/index3.html b/scratch_VLM/scratch_agent/templates/index3.html new file mode 100644 index 0000000000000000000000000000000000000000..17ba54511be998ff2edc5d9083b2b79161aff88e --- /dev/null +++ b/scratch_VLM/scratch_agent/templates/index3.html @@ -0,0 +1,50 @@ + + + + + +Scratch Game JSON Generator + + + +

Scratch Game JSON Generator

+
+
+ +

Output JSON:

+
Waiting for input...
+ + + + diff --git a/scratch_VLM/scratch_agent/templates/index4.html b/scratch_VLM/scratch_agent/templates/index4.html new file mode 100644 index 0000000000000000000000000000000000000000..253f0ea6c42836dc720f5c8adfb9267abfea9515 --- /dev/null +++ b/scratch_VLM/scratch_agent/templates/index4.html @@ -0,0 +1,106 @@ + + + + + + Scratch Game JSON Generator + + + +

Scratch Game JSON Generator

+ +
+
+ +

Backdrops

+
+ + +

Sprites

+
+
+ + + +

Output JSON:

+ 
Waiting for input...
+ + + + diff --git a/scratch_VLM/scratch_agent/templates/upload.html b/scratch_VLM/scratch_agent/templates/upload.html new file mode 100644 index 0000000000000000000000000000000000000000..e39e2d1bd1d06b22168194c97743b3ee125ae61e --- /dev/null +++ b/scratch_VLM/scratch_agent/templates/upload.html @@ -0,0 +1,332 @@ + + + + + + Upload Database + + + + + + +
+
+
+
+ +
+

Upload File

+

Select file to upload

+
+ +
+
+
+ + +
+
+
+ +
+
+
+
+ +
+
Agent Chat v1.0 · © 2023
+
+ + + + + + \ No newline at end of file diff --git a/scratch_VLM/scratch_agent/test_files/sample.docx b/scratch_VLM/scratch_agent/test_files/sample.docx new file mode 100644 index 0000000000000000000000000000000000000000..da085724bbc90b277a42df2d3bf92bf5086e0cd9 --- /dev/null +++ b/scratch_VLM/scratch_agent/test_files/sample.docx @@ -0,0 +1,1574 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GitHub · Where software is built + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ Skip to content + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+
+ +
+ + + + / + + test_files + + + Public +
+ + +
+
+ +
+
+ +

This repository has been disabled

+ +

+ Access to this repository has been disabled by GitHub Staff due to a violation of + GitHub's Terms of Service. + If you are the owner of the repository, you may reach out to GitHub Support for more + information. +

+ +
+
+ +
+ +
+

Footer

+ + + + +
+
+ + + + + © 2025 GitHub, Inc. + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + +
+ +
+
+ + + diff --git a/scratch_VLM/scratch_agent/test_files/sample.jpg b/scratch_VLM/scratch_agent/test_files/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e948041a1b19c9366d99a48501db8e608653041e Binary files /dev/null and b/scratch_VLM/scratch_agent/test_files/sample.jpg differ diff --git a/scratch_VLM/scratch_agent/test_files/sample.mp3 b/scratch_VLM/scratch_agent/test_files/sample.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..ea7e6d2432fb910719108ef1a9b830f123c63bea Binary files /dev/null and b/scratch_VLM/scratch_agent/test_files/sample.mp3 differ diff --git a/scratch_VLM/scratch_agent/test_files/sample.pdf b/scratch_VLM/scratch_agent/test_files/sample.pdf new file mode 100644 index 0000000000000000000000000000000000000000..774c2ea70c55104973794121eae56bcad918da97 Binary files /dev/null and b/scratch_VLM/scratch_agent/test_files/sample.pdf differ diff --git a/scratch_VLM/scratch_agent/test_files/sample.xlsx b/scratch_VLM/scratch_agent/test_files/sample.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..08fd872c2217a41f7b98d667f9dc258c099e68e7 --- /dev/null +++ b/scratch_VLM/scratch_agent/test_files/sample.xlsx @@ -0,0 +1,1574 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GitHub · Where software is built + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ Skip to content + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+
+ +
+ + + + / + + test_files + + + Public +
+ + +
+
+ +
+
+ +

This repository has been disabled

+ +

+ Access to this repository has been disabled by GitHub Staff due to a violation of + GitHub's Terms of Service. + If you are the owner of the repository, you may reach out to GitHub Support for more + information. +

+ +
+
+ +
+ +
+

Footer

+ + + + +
+
+ + + + + © 2025 GitHub, Inc. + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + +
+ +
+
+ + + diff --git a/scratch_VLM/scratch_agent/test_files/test_code.py b/scratch_VLM/scratch_agent/test_files/test_code.py new file mode 100644 index 0000000000000000000000000000000000000000..22036d36a59a3ba3760092d62bedd2da2d14b757 --- /dev/null +++ b/scratch_VLM/scratch_agent/test_files/test_code.py @@ -0,0 +1,19 @@ + +def greet(name): + """Say hello to someone + + Args: + name: The person's name + + Returns: + A greeting message + """ + return f"Hello, {name}!" + +class Person: + def __init__(self, name, age): + self.name = name + self.age = age + + def is_adult(self): + return self.age >= 18 diff --git a/scratch_VLM/scratch_agent/uploads/databases/employee.db b/scratch_VLM/scratch_agent/uploads/databases/employee.db new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/scratch_VLM/scratch_agent/uploads/documents/LLM_based_QA_chatbot_builder.pdf b/scratch_VLM/scratch_agent/uploads/documents/LLM_based_QA_chatbot_builder.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8a134504e0acff8a78e78f1a0c56671c6041b307 --- /dev/null +++ b/scratch_VLM/scratch_agent/uploads/documents/LLM_based_QA_chatbot_builder.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:633e305a9a7fcdc69769e77d96db1fa715ab387c9f8745ebe77fa75590772848 +size 1887485 diff --git a/scratch_VLM/scratch_agent/uploads/images/how-ML-pipeline-Work.png b/scratch_VLM/scratch_agent/uploads/images/how-ML-pipeline-Work.png new file mode 100644 index 0000000000000000000000000000000000000000..fdb2dfba26d782ec6228703f6366fdd73ddeab56 Binary files /dev/null and b/scratch_VLM/scratch_agent/uploads/images/how-ML-pipeline-Work.png differ