From 16a28ab7604fb195bb5b8e26f7003e4eb75e89f6 Mon Sep 17 00:00:00 2001 From: "kaosat.dev" Date: Mon, 22 Jul 2024 00:29:41 +0200 Subject: [PATCH] feat(Blenvy:Blender): * added a fix for BlueprintInfo path issues on non posix platform by replacing specific os.path.join calls with posixpath.join ones (blueprint info paths, export_path, material_path etc) should hopefully solve spawning issues on Windows * restructured & improved materials export: now uses same logic as blueprints, with seperate determining of what materials changed & the export itself * minor tweaks --- TODO.md | 24 +++++--- .../add_ons/auto_export/common/auto_export.py | 20 ++++--- .../add_ons/auto_export/materials/__init__.py | 0 .../{common => materials}/export_materials.py | 55 ++++++++++--------- .../materials/get_materials_to_export.py | 31 +++++++++++ tools/blenvy/add_ons/auto_export/utils.py | 9 +-- tools/blenvy/assets/assets_scan.py | 11 ++-- tools/blenvy/blueprints/blueprint_helpers.py | 7 ++- tools/blenvy/materials/materials_helpers.py | 20 ++++++- 9 files changed, 121 insertions(+), 56 deletions(-) create mode 100644 tools/blenvy/add_ons/auto_export/materials/__init__.py rename tools/blenvy/add_ons/auto_export/{common => materials}/export_materials.py (62%) create mode 100644 tools/blenvy/add_ons/auto_export/materials/get_materials_to_export.py diff --git a/TODO.md b/TODO.md index 29920d3..833f3b0 100644 --- a/TODO.md +++ b/TODO.md @@ -62,7 +62,7 @@ Components: - [x] Hashmap Support - [x] fix parsing of keys's type either on Bevy side (prefered) or on the Blender side - [x] fix weird issue with missing "0" property when adding new entry in empty hashmap => happens only if the values for the "setter" have never been set - - [ ] handle missing types in registry for keys & values + - [x] handle missing types in registry for keys & values - [x] adding a hashmap nukes every existing component ?? - [x] Add correct upgrade handling from individual component to bevy_components - [x] Settings handling: @@ -218,10 +218,15 @@ Blender side: - [ ] add option to 'split out' meshes from blueprints ? - [ ] ie considering meshletts etc , it would make sense to keep blueprints seperate from purely mesh gltfs -- [ ] persist exported materials path in blueprints so that it can be read from library file users - - [ ] just like "export_path" write it into each blueprint's collection - - [ ] scan for used materials per blueprint ! - - [ ] for scenes, scan for used materials of all non instance objects (TODO: what about overrides ?) + +- [ ] materials fixes & upgrades + - [x] materials do not get exported again if the files are missing, until you change a material + - [x] materials do not get exported when a material is added ? + - [ ] material assets seem to be added to list regardless of whether material exports are enabled or not + - [ ] persist exported materials path in blueprints so that it can be read from library file users + - [ ] just like "export_path" write it into each blueprint's collection + - [ ] scan for used materials per blueprint ! + - [ ] for scenes, scan for used materials of all non instance objects (TODO: what about overrides ?) - [ ] add a way of visualizing per blueprint instances ? - [ ] display export path of blueprints (mostly external) ? @@ -284,7 +289,7 @@ Bevy Side: - [x] for sub blueprint tracking: do not propagate/ deal with parent blueprints if they are not themselves Spawning (ie filter out by "BlueprintSpawning") - [x] cleanup internals - [ ] analyse what is off with blueprint level components - - [ ] add the root blueprint itself to the assets either on the blender side or on the bevy side programatically + - [x] add the root blueprint itself to the assets either on the blender side or on the bevy side programatically - [ ] invalidate despawned entity & parent entities AABB - [ ] add unloading/cache removal of materials @@ -316,6 +321,11 @@ Bevy Side: - [x] replace all references to the old 2 add-ons with those to Blenvy - [x] rename repo to "Blenvy" - [x] do a deprecation release of all bevy_gltf_xxx crates to point at the new Blenvy crate -- [ ] material assets seem to be added to list regardless of whether material exports are enabled or not +- [ ] consider finding a way of having unique ids for all objects & collections in Blender (not trivial, if not impossible) + this would allow things such as + - [ ] mapping uuids to blueprint paths + - [ ] solving problems with scene renames + - [ ] the ability to map external TEXT files to data in BLender (git-able, hand editable) +- [x] make aabbs calculation non configurable, getting rid of the last setting (for now) clear && pytest -svv --blender-template ../../testing/bevy_example/art/testing_library.blend --blender-executable /home/ckaos/tools/blender/blender-4.1.0-linux-x64/blender tests/test_bevy_integration_prepare.py && pytest -svv --blender-executable /home/ckaos/tools/blender/blender-4.1.0-linux-x64/blender tests/test_bevy_integration.py \ No newline at end of file diff --git a/tools/blenvy/add_ons/auto_export/common/auto_export.py b/tools/blenvy/add_ons/auto_export/common/auto_export.py index ea92bb7..157711b 100644 --- a/tools/blenvy/add_ons/auto_export/common/auto_export.py +++ b/tools/blenvy/add_ons/auto_export/common/auto_export.py @@ -12,7 +12,9 @@ from .export_gltf import get_standard_exporter_settings from ..levels.export_levels import export_level_scene from ..blueprints.export_blueprints import export_blueprints -from .export_materials import cleanup_materials, export_materials + +from ..materials.get_materials_to_export import get_materials_to_export +from ..materials.export_materials import cleanup_materials, export_materials from ..levels.bevy_scene_components import remove_scene_components, upsert_scene_components @@ -70,10 +72,8 @@ def auto_export(changes_per_scene, changes_per_collection, changes_per_material, # since materials export adds components we need to call this before blueprints are exported # export materials & inject materials components into relevant objects - # FIXME: improve change detection, perhaps even add "material changes" - if export_materials_library and (changed_export_parameters or len(changes_per_material.keys()) > 0 ): - export_materials(blueprints_data.blueprint_names, settings.library_scenes, settings) - + materials_to_export = get_materials_to_export(changes_per_material, changed_export_parameters, blueprints_data, settings) + # update the list of tracked exports exports_total = len(blueprints_to_export) + len(level_scenes_to_export) + (1 if export_materials_library else 0) bpy.context.window_manager.auto_export_tracker.exports_total = exports_total @@ -90,9 +90,11 @@ def auto_export(changes_per_scene, changes_per_collection, changes_per_material, print("BLUEPRINTS: external:", external_blueprints) print("BLUEPRINTS: per_scene:", blueprints_per_scene) print("-------------------------------") - print("BLUEPRINTS: to export:", [blueprint.name for blueprint in blueprints_to_export]) + print("BLUEPRINTS: to export:", [blueprint.name for blueprint in blueprints_to_export]) print("-------------------------------") - print("MAIN SCENES: to export:", level_scenes_to_export) + print("LEVELS: to export:", level_scenes_to_export) + print("-------------------------------") + print("MATERIALS: to export:", materials_to_export) print("-------------------------------") # backup current active scene old_current_scene = bpy.context.scene @@ -112,6 +114,10 @@ def auto_export(changes_per_scene, changes_per_collection, changes_per_material, print("export LIBRARY") export_blueprints(blueprints_to_export, settings, blueprints_data) + # then deal with materials + if export_materials_library: + export_materials(materials_to_export, settings, blueprints_data)#blueprints_data.blueprint_names, settings.library_scenes, settings) + # reset current scene from backup bpy.context.window.scene = old_current_scene diff --git a/tools/blenvy/add_ons/auto_export/materials/__init__.py b/tools/blenvy/add_ons/auto_export/materials/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/blenvy/add_ons/auto_export/common/export_materials.py b/tools/blenvy/add_ons/auto_export/materials/export_materials.py similarity index 62% rename from tools/blenvy/add_ons/auto_export/common/export_materials.py rename to tools/blenvy/add_ons/auto_export/materials/export_materials.py index 2577e00..67927de 100644 --- a/tools/blenvy/add_ons/auto_export/common/export_materials.py +++ b/tools/blenvy/add_ons/auto_export/materials/export_materials.py @@ -5,8 +5,8 @@ from pathlib import Path from blenvy.core.helpers_collections import (traverse_tree) from blenvy.core.object_makers import make_cube from blenvy.materials.materials_helpers import add_material_info_to_objects, get_all_materials -from .generate_temporary_scene_and_export import generate_temporary_scene_and_export -from .export_gltf import (generate_gltf_export_settings) +from ..common.generate_temporary_scene_and_export import generate_temporary_scene_and_export +from ..common.export_gltf import (generate_gltf_export_settings) # material library logic # To avoid redundant materials (can be very costly, mostly when using high res textures) @@ -65,35 +65,38 @@ def clear_materials_scene(temp_scene): # exports the materials used inside the current project: # the name of the output path is /_materials_library.gltf/glb -def export_materials(collections, library_scenes, settings): - gltf_export_settings = generate_gltf_export_settings(settings) - materials_path_full = getattr(settings,"materials_path_full") +def export_materials(materials_to_export, settings, blueprints_data): + if len(materials_to_export) >0: + gltf_export_settings = generate_gltf_export_settings(settings) + materials_path_full = getattr(settings,"materials_path_full") - (used_material_names, materials_per_object) = get_all_materials(collections, library_scenes) - add_material_info_to_objects(materials_per_object, settings) + (used_material_names, materials_per_object) = get_all_materials(blueprints_data.blueprint_names, settings.library_scenes) + add_material_info_to_objects(materials_per_object, settings) - gltf_export_settings = { **gltf_export_settings, - 'use_active_scene': True, - 'use_active_collection':True, - 'use_active_collection_with_nested':True, - 'use_visible': False, - 'use_renderable': False, - 'export_apply':True - } + gltf_export_settings = { **gltf_export_settings, + 'use_active_scene': True, + 'use_active_collection':True, + 'use_active_collection_with_nested':True, + 'use_visible': False, + 'use_renderable': False, + 'export_apply':True + } - current_project_name = Path(bpy.context.blend_data.filepath).stem - gltf_output_path = os.path.join(materials_path_full, current_project_name + "_materials") + current_project_name = Path(bpy.context.blend_data.filepath).stem + gltf_output_path = os.path.join(materials_path_full, current_project_name + "_materials") - print(" exporting Materials to", gltf_output_path, ".gltf/glb") + print(" exporting Materials to", gltf_output_path, ".gltf/glb") - generate_temporary_scene_and_export( - settings=settings, - gltf_export_settings=gltf_export_settings, - temp_scene_name="__materials_scene", - gltf_output_path=gltf_output_path, - tempScene_filler= lambda temp_collection: generate_materials_scene_content(temp_collection, used_material_names), - tempScene_cleaner= lambda temp_scene, params: clear_materials_scene(temp_scene=temp_scene) - ) + generate_temporary_scene_and_export( + settings=settings, + gltf_export_settings=gltf_export_settings, + temp_scene_name="__materials_scene", + gltf_output_path=gltf_output_path, + tempScene_filler= lambda temp_collection: generate_materials_scene_content(temp_collection, used_material_names), + tempScene_cleaner= lambda temp_scene, params: clear_materials_scene(temp_scene=temp_scene) + ) + + def cleanup_materials(collections, library_scenes): diff --git a/tools/blenvy/add_ons/auto_export/materials/get_materials_to_export.py b/tools/blenvy/add_ons/auto_export/materials/get_materials_to_export.py new file mode 100644 index 0000000..7118607 --- /dev/null +++ b/tools/blenvy/add_ons/auto_export/materials/get_materials_to_export.py @@ -0,0 +1,31 @@ + +import bpy +from blenvy.materials.materials_helpers import find_materials_not_on_disk + +def get_materials_to_export(changes_per_material, changed_export_parameters, blueprints_data, settings): + export_gltf_extension = getattr(settings, "export_gltf_extension", ".glb") + blueprints_path_full = getattr(settings,"blueprints_path_full", "") + materials_path_full = getattr(settings,"materials_path_full", "") + + change_detection = getattr(settings.auto_export, "change_detection") + collection_instances_combine_mode = getattr(settings.auto_export, "collection_instances_combine_mode") + + all_materials = bpy.data.materials + local_materials = [material for material in all_materials if material.library is None] + #and (changed_export_parameters or len(changes_per_material.keys()) > 0 ) + + if change_detection and not changed_export_parameters: + changed_materials = [] + + # first check if all materials have already been exported before (if this is the first time the exporter is run + # in your current Blender session for example) + materials_not_on_disk = find_materials_not_on_disk(local_materials, materials_path_full, export_gltf_extension) + + # also deal with blueprints that are always marked as "always_export" + #materials_always_export = [material for material in internal_materials if is_material_always_export(material)] + materials_always_export = [] + materials_to_export = list(set(changed_materials + materials_not_on_disk + materials_always_export)) + + + + return materials_to_export diff --git a/tools/blenvy/add_ons/auto_export/utils.py b/tools/blenvy/add_ons/auto_export/utils.py index ed4f566..1581061 100644 --- a/tools/blenvy/add_ons/auto_export/utils.py +++ b/tools/blenvy/add_ons/auto_export/utils.py @@ -1,5 +1,5 @@ +import posixpath import bpy -import os from pathlib import Path from blenvy.assets.assets_scan import get_blueprint_asset_tree, get_level_scene_assets_tree2 @@ -10,9 +10,6 @@ def assets_to_fake_ron(list_like): return f"(assets: {result})".replace("'", '') - return f"({result})".replace("'", '') - - # TODO : move to assets def upsert_scene_assets(scene, blueprints_data, settings): """print("level scene", scene) @@ -33,7 +30,7 @@ def upsert_scene_assets(scene, blueprints_data, settings): blueprints_path = getattr(settings, "blueprints_path") for blueprint in blueprints_in_scene: if blueprint.local: - blueprint_exported_path = os.path.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") + blueprint_exported_path = posixpath.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints blueprint_exported_path = blueprint.collection['export_path'] if 'export_path' in blueprint.collection else None @@ -56,7 +53,7 @@ def upsert_scene_assets(scene, blueprints_data, settings): materials_path = getattr(settings, "materials_path") current_project_name = Path(bpy.context.blend_data.filepath).stem materials_library_name = f"{current_project_name}_materials" - materials_exported_path = os.path.join(materials_path, f"{materials_library_name}{export_gltf_extension}") + materials_exported_path = posixpath.join(materials_path, f"{materials_library_name}{export_gltf_extension}") material_assets = [{"name": materials_library_name, "path": materials_exported_path}] # we also add the material library as an asset print("material_assets", material_assets, "extension", export_gltf_extension) diff --git a/tools/blenvy/assets/assets_scan.py b/tools/blenvy/assets/assets_scan.py index 16b6191..d24f64b 100644 --- a/tools/blenvy/assets/assets_scan.py +++ b/tools/blenvy/assets/assets_scan.py @@ -1,5 +1,6 @@ import os import json +import posixpath import bpy from .asset_helpers import does_asset_exist, get_user_assets, get_user_assets_as_list @@ -22,7 +23,7 @@ def scan_assets(scene, blueprints_data, settings): #print("BLUEPRINT", blueprint) blueprint_exported_path = None if blueprint.local: - blueprint_exported_path = os.path.join(relative_blueprints_path, f"{blueprint.name}{export_gltf_extension}") + blueprint_exported_path = posixpath.join(relative_blueprints_path, f"{blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints blueprint_exported_path = blueprint.collection['Export_path'] if 'Export_path' in blueprint.collection else None @@ -71,7 +72,7 @@ def get_blueprint_assets_tree(blueprint, blueprints_data, parent, settings): if child_blueprint: blueprint_exported_path = None if blueprint.local: - blueprint_exported_path = os.path.join(blueprints_path, f"{child_blueprint.name}{export_gltf_extension}") + blueprint_exported_path = posixpath.join(blueprints_path, f"{child_blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints blueprint_exported_path = child_blueprint.collection['export_path'] if 'export_path' in child_blueprint.collection else None @@ -101,7 +102,7 @@ def get_level_scene_assets_tree(level_scene, blueprints_data, settings): if blueprint is not None: blueprint_exported_path = None if blueprint.local: - blueprint_exported_path = os.path.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") + blueprint_exported_path = posixpath.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints blueprint_exported_path = blueprint.collection['export_path'] if 'export_path' in blueprint.collection else None @@ -134,7 +135,7 @@ def get_level_scene_assets_tree2(level_scene, blueprints_data, settings): if blueprint is not None: blueprint_exported_path = None if blueprint.local: - blueprint_exported_path = os.path.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") + blueprint_exported_path = posixpath.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints blueprint_exported_path = blueprint.collection['export_path'] if 'export_path' in blueprint.collection else None @@ -156,7 +157,7 @@ def get_blueprint_asset_tree(blueprint, blueprints_data, settings): if sub_blueprint is not None: sub_blueprint_exported_path = None if sub_blueprint.local: - sub_blueprint_exported_path = os.path.join(blueprints_path, f"{sub_blueprint.name}{export_gltf_extension}") + sub_blueprint_exported_path = posixpath.join(blueprints_path, f"{sub_blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints sub_blueprint_exported_path = sub_blueprint.collection['export_path'] if 'export_path' in sub_blueprint.collection else None diff --git a/tools/blenvy/blueprints/blueprint_helpers.py b/tools/blenvy/blueprints/blueprint_helpers.py index 9ac08df..92e2f3a 100644 --- a/tools/blenvy/blueprints/blueprint_helpers.py +++ b/tools/blenvy/blueprints/blueprint_helpers.py @@ -3,6 +3,7 @@ import os import json import bpy from pathlib import Path +import posixpath from ..core.scene_helpers import add_scene_property @@ -28,10 +29,10 @@ def inject_export_path_into_internal_blueprints(internal_blueprints, blueprints_ materials_path = getattr(settings, "materials_path") current_project_name = Path(bpy.context.blend_data.filepath).stem materials_library_name = f"{current_project_name}_materials" - materials_exported_path = os.path.join(materials_path, f"{materials_library_name}{export_gltf_extension}") + materials_exported_path = posixpath.join(materials_path, f"{materials_library_name}{export_gltf_extension}") for blueprint in internal_blueprints: - blueprint_exported_path = os.path.join(blueprints_path, f"{blueprint.name}{gltf_extension}") + blueprint_exported_path = posixpath.join(blueprints_path, f"{blueprint.name}{gltf_extension}") # print("injecting blueprint path", blueprint_exported_path, "for", blueprint.name) blueprint.collection["export_path"] = blueprint_exported_path if export_materials_library: @@ -58,7 +59,7 @@ def inject_blueprints_list_into_level_scene(scene, blueprints_data, settings): #print("BLUEPRINT", blueprint) blueprint_exported_path = None if blueprint.local: - blueprint_exported_path = os.path.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") + blueprint_exported_path = posixpath.join(blueprints_path, f"{blueprint.name}{export_gltf_extension}") else: # get the injected path of the external blueprints blueprint_exported_path = blueprint.collection['Export_path'] if 'Export_path' in blueprint.collection else None diff --git a/tools/blenvy/materials/materials_helpers.py b/tools/blenvy/materials/materials_helpers.py index 4c2c2ab..79d7ecf 100644 --- a/tools/blenvy/materials/materials_helpers.py +++ b/tools/blenvy/materials/materials_helpers.py @@ -1,8 +1,24 @@ import os +import posixpath import bpy from pathlib import Path from ..core.helpers_collections import (traverse_tree) +def find_materials_not_on_disk(materials, folder_path, extension): + not_found_materials = [] + for material in materials: + gltf_output_path = os.path.join(folder_path, material.name + extension) + # print("gltf_output_path", gltf_output_path) + found = os.path.exists(gltf_output_path) and os.path.isfile(gltf_output_path) + if not found: + not_found_materials.append(material) + return not_found_materials + +def check_if_material_on_disk(scene_name, folder_path, extension): + gltf_output_path = os.path.join(folder_path, scene_name + extension) + found = os.path.exists(gltf_output_path) and os.path.isfile(gltf_output_path) + return found + # get materials per object, and injects the materialInfo component def get_materials(object, materials_per_object): material_slots = object.material_slots @@ -38,11 +54,11 @@ def add_material_info_to_objects(materials_per_object, settings): current_project_name = Path(bpy.context.blend_data.filepath).stem materials_library_name = f"{current_project_name}_materials" - materials_exported_path = os.path.join(materials_path, f"{materials_library_name}{export_gltf_extension}") + materials_exported_path = posixpath.join(materials_path, f"{materials_library_name}{export_gltf_extension}") for object in materials_per_object.keys(): material = materials_per_object[object] # TODO: switch to using actual components ? - materials_exported_path = os.path.join(materials_path, f"{materials_library_name}{export_gltf_extension}") + materials_exported_path = posixpath.join(materials_path, f"{materials_library_name}{export_gltf_extension}") object['MaterialInfo'] = f'(name: "{material.name}", path: "{materials_exported_path}")'