diff --git a/tools/blenvy/TODO.md b/tools/blenvy/TODO.md index f20c279..fe54982 100644 --- a/tools/blenvy/TODO.md +++ b/tools/blenvy/TODO.md @@ -15,7 +15,7 @@ Auto export - [x] main/ library scene names - [x] paths -Data storage: +Data storage for custom properties: - for scenes (main scenes) - at scene level - for blueprints @@ -23,7 +23,7 @@ Data storage: - Note: these should be COPIED to the scene level when exporting, into the temp_scene's properties > NOTE: UP until we manage to create a PR for Bevy to directly support the scene level gltf_extras, the auto exporter should automatically create (& remove) - an additional object with scene__components to copy that data to + any additional object with scene__components to copy that data to Assets: - blueprint assets should be auto_generated & inserted into the list of assets : these assets are NOT removable by the user @@ -103,3 +103,5 @@ General issues: - [x] fix auto export workflow - [ ] should we write the previous _xxx data only AFTER a sucessfull export only ? - [ ] add hashing of modifiers/ geometry nodes in serialize scene +- [ ] add ability to FORCE export specific blueprints & levels +- [ ] undo after a save removes any saved "serialized scene" data ? DIG into this \ No newline at end of file diff --git a/tools/blenvy/add_ons/auto_export/common/prepare_and_export.py b/tools/blenvy/add_ons/auto_export/common/prepare_and_export.py index ffe1c4c..51b6d39 100644 --- a/tools/blenvy/add_ons/auto_export/common/prepare_and_export.py +++ b/tools/blenvy/add_ons/auto_export/common/prepare_and_export.py @@ -7,7 +7,7 @@ from .settings_diff import get_setting_changes # prepare export by gather the changes to the scenes & settings def prepare_and_export(): print("prepare and export") - bpy.context.window_manager.auto_export_tracker.disable_change_detection() + #bpy.context.window_manager.auto_export_tracker.disable_change_detection() blenvy = bpy.context.window_manager.blenvy auto_export_settings = blenvy.auto_export if auto_export_settings.auto_export: # only do the actual exporting if auto export is actually enabled @@ -22,6 +22,6 @@ def prepare_and_export(): # cleanup # TODO: these are likely obsolete # reset the list of changes in the tracker - bpy.context.window_manager.auto_export_tracker.clear_changes() + #bpy.context.window_manager.auto_export_tracker.clear_changes() print("AUTO EXPORT DONE") - bpy.app.timers.register(bpy.context.window_manager.auto_export_tracker.enable_change_detection, first_interval=0.1) + #bpy.app.timers.register(bpy.context.window_manager.auto_export_tracker.enable_change_detection, first_interval=0.1) diff --git a/tools/blenvy/add_ons/auto_export/common/serialize_scene.py b/tools/blenvy/add_ons/auto_export/common/serialize_scene.py index 69d62cd..f5f0ceb 100644 --- a/tools/blenvy/add_ons/auto_export/common/serialize_scene.py +++ b/tools/blenvy/add_ons/auto_export/common/serialize_scene.py @@ -1,3 +1,4 @@ +import inspect import json from mathutils import Color import numpy as np @@ -13,6 +14,71 @@ fields_to_ignore_generic = [ 'session_uid', 'copy', 'id_type', 'is_embedded_data', 'is_evaluated', 'is_library_indirect', 'is_missing', 'is_runtime_data' ] + +def generic_fields_hasher(data, fields_to_ignore): + all_field_names = dir(data) + field_values = [getattr(data, prop, None) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show") and not callable(getattr(data, prop, None)) ] + return str(field_values) + +def peel_value( value ): + try: + len( value ) + return [ peel_value( x ) for x in value ] + except TypeError: + return value + +def _lookup_color(data): + return peel_value(data) + +def _lookup_array(data): + return peel_value(data) + +def _lookup_prop_group(data): + bla = generic_fields_hasher_evolved(data, fields_to_ignore=fields_to_ignore_generic) + print("PROPGROUP", bla) + return bla + +def _lookup_collection(data): + return [generic_fields_hasher_evolved(item, fields_to_ignore=fields_to_ignore_generic) for item in data] + +def _lookup_materialLineArt(data): + return generic_fields_hasher_evolved(data, fields_to_ignore=fields_to_ignore_generic) + +type_lookups = { + Color: _lookup_color,#lambda input: print("dsf")', + bpy.types.bpy_prop_array: _lookup_array, + bpy.types.PropertyGroup: _lookup_prop_group, + bpy.types.bpy_prop_collection: _lookup_collection, + bpy.types.MaterialLineArt: _lookup_materialLineArt +} + +# TODO: replace the first one with this once if its done +def generic_fields_hasher_evolved(data, fields_to_ignore): + all_field_names = dir(data) + field_values = [] + for field_name in all_field_names: + if not field_name.startswith("__") and not field_name in fields_to_ignore and not field_name.startswith("show") and not callable(getattr(data, field_name, None)): + raw_value = getattr(data, field_name, None) + #print("raw value", raw_value, "type", type(raw_value), isinstance(raw_value, Color), isinstance(raw_value, bpy.types.bpy_prop_array)) + conversion_lookup = None # type_lookups.get(type(raw_value), None) + all_types = inspect.getmro(type(raw_value)) + for s_type in all_types: + if type_lookups.get(s_type, None) is not None: + conversion_lookup = type_lookups[s_type] + break + + field_value = None + if conversion_lookup is not None: + field_value = conversion_lookup(raw_value) + print("field_name",field_name,"conv value", field_value) + else: + print("field_name",field_name,"raw value", raw_value) + field_value = raw_value + + field_values.append(str(field_value)) + + return str(field_values) + # possible alternatives https://blender.stackexchange.com/questions/286010/bpy-detect-modified-mesh-data-vertices-edges-loops-or-polygons-for-cachin def mesh_hash(obj): # this is incomplete, how about edges ? @@ -62,24 +128,23 @@ def animation_hash(obj): return compact_result -def camera_hash(obj): - camera_fields = ["angle", "angle_x", "angle_y", "animation_data", "background_images", "clip_end", "clip_start", "display_size", "dof", "fisheye_fov"] - camera_data = obj.data - fields_to_ignore= fields_to_ignore_generic +# TODO : we should also check for custom props on scenes, meshes, materials +# TODO: also how about our new "assets" custom properties ? those need to be check too +def custom_properties_hash(obj): + custom_properties = {} + for property_name in obj.keys(): + if property_name not in '_RNA_UI' and property_name != 'components_meta': + custom_properties[property_name] = obj[property_name] + return str(hash(str(custom_properties))) - all_field_names = dir(camera_data) - fields = [getattr(camera_data, prop, None) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] +def camera_hash(obj): + camera_data = obj.data # TODO: the above is not enough, certain fields are left as bpy.data.xx - #print("camera", obj, fields) - return str(fields) + return str(generic_fields_hasher(camera_data, fields_to_ignore_generic)) def light_hash(obj): light_data = obj.data - fields_to_ignore = fields_to_ignore_generic - - all_field_names = dir(light_data) - fields = [getattr(light_data, prop, None) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - return str(fields) + return str(generic_fields_hasher(light_data, fields_to_ignore_generic)) def bones_hash(bones): fields_to_ignore = fields_to_ignore_generic + ['AxisRollFromMatrix', 'MatrixFromAxisRoll', 'evaluate_envelope', 'convert_local_to_pose', 'foreach_get', 'foreach_set', 'get', 'set', 'find', 'items', 'keys', 'values'] @@ -100,25 +165,11 @@ def armature_hash(obj): all_field_names = dir(armature_data) fields = [getattr(armature_data, prop, None) if not prop in fields_to_convert.keys() else fields_to_convert[prop](getattr(armature_data, prop)) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - #print("ARMATURE", fields) """for bone in armature_data.bones: print("bone", bone, bone_hash(bone))""" return str(fields) -def field_value(data): - pass - -def color(color_data): - # print("color", color_data, type(color_data)) - return str(peel_value(color_data)) - -def lineart(lineart_data): - fields_to_ignore = fields_to_ignore_generic - - all_field_names = dir(lineart_data) - fields = [getattr(lineart_data, prop, None) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - return str(fields) def node_tree(nodetree_data): print("SCANNING NODE TREE", nodetree_data) @@ -126,8 +177,6 @@ def node_tree(nodetree_data): output = nodetree_data.get_output_node("ALL") print("output", output) - - fields_to_ignore = fields_to_ignore_generic+ ['contains_tree','get_output_node', 'interface_update', 'override_template_create'] all_field_names = dir(nodetree_data) fields = [getattr(nodetree_data, prop, None) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] @@ -135,33 +184,15 @@ def node_tree(nodetree_data): # print("node tree", fields) return str(fields) - -def peel_value( value ): - try: - len( value ) - return [ peel_value( x ) for x in value ] - except TypeError: - return value - -def material_hash(material): - fields_to_ignore = fields_to_ignore_generic - fields_to_convert = {'diffuse_color': color, 'line_color': color, 'lineart': lineart, 'node_tree': node_tree} # TODO: perhaps use types rather than names - all_field_names = dir(material) - fields = [getattr(material, prop, None) if not prop in fields_to_convert.keys() else fields_to_convert[prop](getattr(material, prop)) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - - type_of = [type(getattr(material, prop, None)) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - names = [prop for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - tutu = [t == Color for t in type_of] # bpy.types.MaterialLineArt bpy.types.ShaderNodeTree - #print("fields", type_of) - - """for prop in [prop for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")]: - bla = getattr(material, prop, None) - if hasattr(bla, "rna_type"): - print("YOLO", prop, bla, peel_value(bla), "type", type(bla), bla.rna_type, bla.rna_type == bpy.types.FloatProperty, type(bla) == bpy.types.bpy_prop_collection) - print("types", type(bla) == bpy.types.bpy_prop_collection, type(bla) == bpy.types.FloatColorAttributeValue)""" - - # print("oooooh", material, material.bl_rna.properties.items()) - return str(fields)#str(hash(str(fields))) +def material_hash(material, settings): + print("material_hash", material) + hashed_material = generic_fields_hasher_evolved(material, fields_to_ignore_generic + ['node_tree']) # we want to handle the node tree seperatly + print("HASH", hashed_material) + """if node_group is not None and settings.auto_export.materials_in_depth_scan: + pass + else: + generic_fields_hasher(material, fields_to_ignore_generic)""" + return str(hashed_material) # TODO: this is partially taken from export_materials utilities, perhaps we could avoid having to fetch things multiple times ? def materials_hash(obj, cache, settings): @@ -169,60 +200,92 @@ def materials_hash(obj, cache, settings): materials = [] for material_slot in obj.material_slots: material = material_slot.material - cached_hash = cache['materials'].get(material.name, None) + """cached_hash = cache['materials'].get(material.name, None) if cached_hash: - # print("CACHHHHHED", cached_hash) materials.append(cached_hash) + print("CAACHED") else: - mat = material_hash(material) + mat = material_hash(material, settings) cache['materials'][material.name] = mat - materials.append(mat) - # print("NOT CACHHH", mat) + materials.append(mat)""" + mat = material_hash(material, settings) + cache['materials'][material.name] = mat + materials.append(mat) + return str(hash(str(materials))) -# TODO : we should also check for custom props on scenes, meshes, materials -def custom_properties_hash(obj): - custom_properties = {} - for property_name in obj.keys(): - if property_name not in '_RNA_UI' and property_name != 'components_meta': - custom_properties[property_name] = obj[property_name] - return str(hash(str(custom_properties))) - def modifier_hash(modifier_data, settings): - fields_to_ignore = fields_to_ignore_generic - all_field_names = dir(modifier_data) - fields = [getattr(modifier_data, prop, None) for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - - filtered_field_names = [prop for prop in all_field_names if not prop.startswith("__") and not prop in fields_to_ignore and not prop.startswith("show_")] - print("fields", fields, "field names", filtered_field_names) node_group = getattr(modifier_data, "node_group", None) - if node_group is not None: - print("THIS IS A GEOMETRY NODE") + + if node_group is not None and settings.auto_export.modifiers_in_depth_scan: + #print("THIS IS A GEOMETRY NODE") + + # storage for hashing + links_hashes = [] + nodes_hashes = [] + modifier_inputs = dict(modifier_data) + for node in node_group.nodes: - print("node", node) - print("node type", node.type) - try: - print("node value", node.values()) - except:pass + #print("node", node, node.type, node.name, node.label) + #print("node info", dir(node)) + + input_hashes = [] for input in node.inputs: - print(" input", input, input.name, input.label) - if hasattr(input, "default_value"): - print("YOHO", dict(input), input.default_value) + #print(" input", input, "label", input.label, "name", input.name) + input_hash = f"{getattr(input, 'default_value', None)}" + input_hashes.append(input_hash) + """if hasattr(input, "default_value"): + print("YOHO", dict(input), input.default_value)""" - + output_hashes = [] + # IF the node itself is a group input, its outputs are the inputs of the geometry node (yes, not easy) + node_in_use = True + for (index, output) in enumerate(node.outputs): + # print(" output", output, "label", output.label, "name", output.name, "generated name", f"Socket_{index+1}") + output_hash = f"{getattr(output, 'default_value', None)}" + output_hashes.append(output_hash) + """if hasattr(output, "default_value"): + print("YOHO", output.default_value)""" + node_in_use = node_in_use and hasattr(output, "default_value") + #print("NODE IN USE", node_in_use) - return str(fields) + node_fields_to_ignore = fields_to_ignore_generic + ['internal_links', 'inputs', 'outputs'] + + node_hash = f"{generic_fields_hasher(node, node_fields_to_ignore)}_{str(input_hashes)}_{str(output_hashes)}" + #print("node hash", node_hash) + nodes_hashes.append(node_hash) + #print(" ") + + for link in node_group.links: + """print("LINK", link) #dir(link) + print("FROM", link.from_node, link.from_socket) + print("TO", link.to_node, link.to_socket)""" + + + from_socket_default = link.from_socket.default_value if hasattr(link.from_socket, "default_value") else None + to_socket_default = link.to_socket.default_value if hasattr(link.to_socket, "default_value") else None + + link_hash = f"{link.from_node.name}_{link.from_socket.name}_{from_socket_default}+{link.to_node.name}_{link.to_socket.name}_{to_socket_default}" + + """if hasattr(link.from_socket, "default_value"): + print("[FROM SOCKET]", link.from_socket.default_value) + if hasattr(link.to_socket, "default_value"): + print("[TO SOCKET]", link.to_socket.default_value)""" + + links_hashes.append(link_hash) + #print("link_hash", link_hash) + + return f"{str(modifier_inputs)}_{str(nodes_hashes)}_{str(links_hashes)}" + else: + return generic_fields_hasher(modifier_data, fields_to_ignore_generic) + def modifiers_hash(object, settings): - print("modifiers", object.modifiers) - modifiers = [] for modifier in object.modifiers: print("modifier", modifier )# modifier.node_group) - try: - print("MODIFIER FIEEEEEEELD", modifier.ratio) # apparently this only works for non geometry nodes ?? - except: pass modifiers.append(modifier_hash(modifier, settings)) + print(" ") return str(hash(str(modifiers))) def serialize_scene(settings): diff --git a/tools/blenvy/add_ons/auto_export/settings.py b/tools/blenvy/add_ons/auto_export/settings.py index e2fe3da..1b4237e 100644 --- a/tools/blenvy/add_ons/auto_export/settings.py +++ b/tools/blenvy/add_ons/auto_export/settings.py @@ -34,8 +34,15 @@ class AutoExportSettings(PropertyGroup): materials_in_depth_scan : BoolProperty( name='In depth scan of materials (could be slow)', - description='serializes more details of materials in order to detect changes (slower, but more accurate in detecting changes)', - default=False, + description='serializes more details of materials in order to detect changes (could be slower, but much more accurate in detecting changes)', + default=True, + update=save_settings + ) # type: ignore + + modifiers_in_depth_scan : BoolProperty( + name='In depth scan of modifiers (could be slow)', + description='serializes more details of modifiers (particularly geometry nodes) in order to detect changes (could be slower, but much more accurate in detecting changes)', + default=True, update=save_settings ) # type: ignore diff --git a/tools/blenvy/add_ons/auto_export/ui.py b/tools/blenvy/add_ons/auto_export/ui.py index 185e672..6c84d1a 100644 --- a/tools/blenvy/add_ons/auto_export/ui.py +++ b/tools/blenvy/add_ons/auto_export/ui.py @@ -32,8 +32,12 @@ def draw_settings_ui(layout, auto_export_settings): section.enabled = controls_enabled section.prop(auto_export_settings, "change_detection", text="Use change detection") + + section = section.box() + section.enabled = controls_enabled and auto_export_settings.change_detection + section.prop(auto_export_settings, "materials_in_depth_scan", text="Detailed materials scan") - + section.prop(auto_export_settings, "modifiers_in_depth_scan", text="Detailed modifiers scan") header, panel = layout.panel("Blueprints", default_closed=False) header.label(text="Blueprints")