feat(Blenvy): fixed all broken/ missing auto export elements

* fixed overhauled, cleaned up and improved settings change detection
 * serialize_scene now generate a hash per object instead of keeping the details (useless)
 * changed handling of gltf settings, so that only the actual non default, changed settings are saved
 * a ton of minor changes & cleanups to get export working correctly: it works !
 * started adding materials_in_depth_scan (wip) to be able to control more detailed (but costly) materials diffing for change detection

tools/blenvy/TODO.md

@@ -90,7 +90,7 @@ General issues:
 
 
  - [x] addon-prefs => settings
- - [x] generate_gltf_export_preferences => should not use add-on prefs at all ? since we are not overriding gltf settings that way anymore ?
+ - [x] generate_gltf_export_settings => should not use add-on prefs at all ? since we are not overriding gltf settings that way anymore ?
  - [x] remove hard coded path for standard gltf settings
  - [x] load settings on file load
     - [x] auto_export
@@ -100,4 +100,5 @@ General issues:
 
 - [ ] force overwrite of settings files instead of partial updates ?
 - [ ] add tests for disabled components 
-- [ ]  fix auto export workflow
+- [x]  fix auto export workflow
+- [ ] should we write the previous _xxx data only AFTER a sucessfull export only ?

tools/blenvy/add_ons/auto_export/__init__.py

@@ -39,7 +39,7 @@ def gltf_post_export_callback(data):
             gltf_export_settings.clear()
 
             settings = dict(settings)
-            current_gltf_settings = generate_complete_settings_dict(settings, presets=ExportGLTF2_Base, ignore_list=["use_active_collection", "use_active_collection_with_nested", "use_active_scene", "use_selection", "will_save_settings", "gltf_export_id"], preset_defaults=True)
+            current_gltf_settings = generate_complete_settings_dict(settings, presets=ExportGLTF2_Base, ignore_list=["use_active_collection", "use_active_collection_with_nested", "use_active_scene", "use_selection", "will_save_settings", "gltf_export_id"], preset_defaults=False)
             gltf_export_settings.write(json.dumps(current_gltf_settings))
         # now reset the original gltf_settings
         if gltf_settings_backup != "":

tools/blenvy/add_ons/auto_export/export/auto_export.py

@@ -41,8 +41,9 @@ def auto_export(changes_per_scene, changed_export_parameters, settings):
 
         [main_scene_names, level_scenes, library_scene_names, library_scenes] = get_main_and_library_scenes(settings)
 
-        bpy.context.window_manager.blueprints_registry.refresh_blueprints()
-        blueprints_data = bpy.context.window_manager.blueprints_registry.blueprints_data
+        blueprints_data = bpy.context.window_manager.blueprints_registry.refresh_blueprints()
+        #blueprints_data = bpy.context.window_manager.blueprints_registry.blueprints_data
+        #print("blueprints_data", blueprints_data)
         blueprints_per_scene = blueprints_data.blueprints_per_scenes
         internal_blueprints = [blueprint.name for blueprint in blueprints_data.internal_blueprints]
         external_blueprints = [blueprint.name for blueprint in blueprints_data.external_blueprints]

tools/blenvy/add_ons/auto_export/export/export_blueprints.py

@@ -1,15 +1,13 @@
 import os
 import bpy
-
 from ..constants import TEMPSCENE_PREFIX
 from ..helpers.generate_and_export import generate_and_export
-from .export_gltf import (generate_gltf_export_preferences)
 from ..helpers.helpers_scenes import clear_hollow_scene, copy_hollowed_collection_into
-
+from .export_gltf import generate_gltf_export_settings
 
 def export_blueprints(blueprints, settings, blueprints_data):
     blueprints_path_full = getattr(settings, "blueprints_path_full")
-    gltf_export_preferences = generate_gltf_export_preferences(settings)
+    gltf_export_settings = generate_gltf_export_settings(settings)
     
     try:
         # save current active collection
@@ -19,7 +17,7 @@ def export_blueprints(blueprints, settings, blueprints_data):
         for blueprint in blueprints:
             print("exporting collection", blueprint.name)
             gltf_output_path = os.path.join(blueprints_path_full, blueprint.name)
-            gltf_export_settings = { **gltf_export_preferences, 'use_active_scene': True, 'use_active_collection': True, 'use_active_collection_with_nested':True}
+            gltf_export_settings = { **gltf_export_settings, 'use_active_scene': True, 'use_active_collection': True, 'use_active_collection_with_nested':True}
             
             # if we are using the material library option, do not export materials, use placeholder instead
             if export_materials_library:

tools/blenvy/add_ons/auto_export/export/export_gltf.py

@@ -8,9 +8,9 @@ def get_standard_exporter_settings():
     standard_gltf_exporter_settings = load_settings(".blenvy_gltf_settings")
     return standard_gltf_exporter_settings if standard_gltf_exporter_settings is not None else {}
 
-def generate_gltf_export_preferences(settings): 
+def generate_gltf_export_settings(settings): 
     # default values
-    gltf_export_preferences = dict(
+    gltf_export_settings = dict(
         # export_format= 'GLB', #'GLB', 'GLTF_SEPARATE', 'GLTF_EMBEDDED'
         check_existing=False,
 
@@ -49,7 +49,7 @@ def generate_gltf_export_preferences(settings):
     """for key in settings.__annotations__.keys():
         if str(key) not in AutoExportGltfPreferenceNames:
             #print("overriding setting", key, "value", getattr(settings,key))
-            pass#gltf_export_preferences[key] = getattr(settings, key)"""
+            pass#gltf_export_settings[key] = getattr(settings, key)"""
 
 
     standard_gltf_exporter_settings = get_standard_exporter_settings()
@@ -68,8 +68,10 @@ def generate_gltf_export_preferences(settings):
     # a certain number of essential params should NEVER be overwritten , no matter the settings of the standard exporter
     for key in standard_gltf_exporter_settings.keys():
         if str(key) not in constant_keys:
-            gltf_export_preferences[key] =  standard_gltf_exporter_settings.get(key)
-    return gltf_export_preferences
+            gltf_export_settings[key] =  standard_gltf_exporter_settings.get(key)
+
+    print("GLTF EXPORT SETTINGS", gltf_export_settings)
+    return gltf_export_settings
 
 
 #https://docs.blender.org/api/current/bpy.ops.export_scene.html#bpy.ops.export_scene.gltf

tools/blenvy/add_ons/auto_export/export/export_main_scenes.py

@@ -5,19 +5,19 @@ from blenvy.blueprints.blueprint_helpers import inject_blueprints_list_into_main
 
 from ..constants import TEMPSCENE_PREFIX
 from ..helpers.generate_and_export import generate_and_export
-from .export_gltf import (generate_gltf_export_preferences, export_gltf)
+from .export_gltf import (generate_gltf_export_settings, export_gltf)
 from ..modules.bevy_dynamic import is_object_dynamic, is_object_static
 from ..helpers.helpers_scenes import clear_hollow_scene, copy_hollowed_collection_into
 
 def export_main_scene(scene, blend_file_path, settings, blueprints_data): 
-    gltf_export_preferences = generate_gltf_export_preferences(settings)
+    gltf_export_settings = generate_gltf_export_settings(settings)
     assets_path_full = getattr(settings,"assets_path_full")
     levels_path_full = getattr(settings,"levels_path_full")
 
     export_blueprints = getattr(settings.auto_export,"export_blueprints")
     export_separate_dynamic_and_static_objects = getattr(settings.auto_export, "export_separate_dynamic_and_static_objects")
 
-    gltf_export_settings = { **gltf_export_preferences, 
+    gltf_export_settings = { **gltf_export_settings, 
                        'use_active_scene': True, 
                        'use_active_collection':True, 
                        'use_active_collection_with_nested':True,  

tools/blenvy/add_ons/auto_export/export/export_materials.py

@@ -6,7 +6,7 @@ from blenvy.core.helpers_collections import (traverse_tree)
 from blenvy.core.object_makers import make_cube
 from blenvy.materials.materials_helpers import get_all_materials
 from ..helpers.generate_and_export import generate_and_export
-from .export_gltf import (generate_gltf_export_preferences)
+from .export_gltf import (generate_gltf_export_settings)
 
 def clear_material_info(collection_names, library_scenes):
     for scene in library_scenes:
@@ -60,13 +60,13 @@ def clear_materials_scene(temp_scene):
 # exports the materials used inside the current project:
 # the name of the output path is <materials_folder>/<name_of_your_blend_file>_materials_library.gltf/glb
 def export_materials(collections, library_scenes, settings):
-    gltf_export_preferences = generate_gltf_export_preferences(settings)
+    gltf_export_settings = generate_gltf_export_settings(settings)
     materials_path_full = getattr(settings,"materials_path_full")
 
     used_material_names = get_all_materials(collections, library_scenes)
     current_project_name = Path(bpy.context.blend_data.filepath).stem
 
-    gltf_export_settings = { **gltf_export_preferences, 
+    gltf_export_settings = { **gltf_export_settings, 
                     'use_active_scene': True, 
                     'use_active_collection':True, 
                     'use_active_collection_with_nested':True,  

tools/blenvy/add_ons/auto_export/export/prepare_and_export.py

@@ -15,7 +15,7 @@ def prepare_and_export():
         # determine changed objects
         per_scene_changes = get_changes_per_scene()
         # determine changed parameters 
-        setting_changes = get_setting_changes(auto_export_settings)
+        setting_changes = get_setting_changes()
         # do the actual export
         auto_export(per_scene_changes, setting_changes, blenvy)
 

tools/blenvy/add_ons/auto_export/export/project_diff.py

@@ -1,6 +1,7 @@
 import json
 import bpy
 from ..helpers.serialize_scene import serialize_scene
+from blenvy.settings import load_settings, upsert_settings
 
 def bubble_up_changes(object, changes_per_scene):
     if object.parent:
@@ -44,26 +45,25 @@ def serialize_current():
     return current
 
 def get_changes_per_scene():
+    previous = load_settings(".blenvy.project_serialized_previous")
     current = serialize_current()
 
-    previous_stored = bpy.data.texts[".blenvy.project.serialized"] if ".blenvy.project.serialized" in bpy.data.texts else None
-    if previous_stored == None:
-        previous_stored = bpy.data.texts.new(".blenvy.project.serialized")
-        previous_stored.write(json.dumps(current))
-        return {}
-    
-    previous = json.loads(previous_stored.as_string())
-    
-    # determin changes
+    # determine changes
     changes_per_scene = project_diff(previous, current)
 
     # save the current project as previous
-    previous_stored.clear()
-    previous_stored.write(json.dumps(current))
+    upsert_settings(".blenvy.project_serialized_previous", current, overwrite=True)
+
+    print("changes per scene", changes_per_scene)
     return changes_per_scene
 
 
 def project_diff(previous, current):
+    """print("previous", previous)
+    print("current", current)"""
+    if previous is None or current is None:
+        return {}
+    print("HERE")
     
     changes_per_scene = {}
 
@@ -100,5 +100,4 @@ def project_diff(previous, current):
                     bubble_up_changes(bpy.data.objects[object_name], changes_per_scene[scene])
                     # now bubble up for instances & parents
         
-    print("changes per scene", changes_per_scene)
     return changes_per_scene

tools/blenvy/add_ons/auto_export/export/settings_diff.py

@@ -3,16 +3,18 @@ import bpy
 from blenvy.settings import are_settings_identical, load_settings, upsert_settings
 
 # which settings are specific to auto_export # TODO: can we infer this ?
-auto_export_parameter_names = [
+parameter_names_whitelist_common = [
     # blenvy core
     'project_root_path',
     'assets_path',
     'blueprints_path',
     'levels_path',
     'materials_path',
-    #'main_scene_names',
-    #'library_scene_names',
+    'main_scene_names',
+    'library_scene_names',
+]
 
+parameter_names_whitelist_auto_export = [
     # auto export
     'export_scene_settings',
     'export_blueprints',
@@ -22,76 +24,34 @@ auto_export_parameter_names = [
     'export_marked_assets'
 ]
 
-def get_setting_changes(auto_export_settings):
-    print("get setting changes", dict(auto_export_settings))
+def get_setting_changes():
+    print("get setting changes")
+
+    previous_common_settings = load_settings(".blenvy_common_settings_previous")
+    current_common_settings = load_settings(".blenvy_common_settings")
+    common_settings_changed = not are_settings_identical(previous_common_settings, current_common_settings, white_list=parameter_names_whitelist_common)
+
+    previous_export_settings = load_settings(".blenvy_export_settings_previous")
+    current_export_settings = load_settings(".blenvy_export_settings")
+    export_settings_changed = not are_settings_identical(previous_export_settings, current_export_settings, white_list=parameter_names_whitelist_auto_export)
+
     previous_gltf_settings = load_settings(".blenvy_gltf_settings_previous")
     current_gltf_settings = load_settings(".blenvy_gltf_settings")
     gltf_settings_changed = not are_settings_identical(previous_gltf_settings, current_gltf_settings)
 
-    previous_export_settings = load_settings(".blenvy_export_settings_previous")
-    current_export_settings = dict(auto_export_settings) #load_settings(".blenvy_export_settings")
-    export_settings_changed = not are_settings_identical(previous_export_settings, current_export_settings)
-
-    # if there were no setting before, it is new, we need export
-    if previous_gltf_settings is None:
-         pass
-    if previous_export_settings is None:
-         pass
-    
     # write the new settings to the old settings
-    upsert_settings(".blenvy_gltf_settings_previous", current_gltf_settings)
-    upsert_settings(".blenvy_export_settings_previous", current_export_settings)
+    upsert_settings(".blenvy_common_settings_previous", current_common_settings, overwrite=True)
+    upsert_settings(".blenvy_export_settings_previous", current_export_settings, overwrite=True)
+    upsert_settings(".blenvy_gltf_settings_previous", current_gltf_settings, overwrite=True)
 
-    return gltf_settings_changed or export_settings_changed
+    print("common_settings_changed", common_settings_changed,"export_settings_changed", export_settings_changed, "gltf_settings_changed", gltf_settings_changed, )
 
-def did_export_settings_change(self):
+    # if there were no setting before, it is new, we need export # TODO: do we even need this ? I guess in the case where both the previous & the new one are both none ? very unlikely, but still
+    if previous_common_settings is None:
+         return True
+    if previous_export_settings is None:
+         return True
+    if previous_gltf_settings is None:
          return True
-        # compare both the auto export settings & the gltf settings
-        previous_auto_settings = bpy.data.texts[".gltf_auto_export_settings_previous"] if ".gltf_auto_export_settings_previous" in bpy.data.texts else None
-        previous_gltf_settings = bpy.data.texts[".blenvy_gltf_settings_previous"] if ".blenvy_gltf_settings_previous" in bpy.data.texts else None
 
-        current_auto_settings = bpy.data.texts[".gltf_auto_export_settings"] if ".gltf_auto_export_settings" in bpy.data.texts else None
-        current_gltf_settings = bpy.data.texts[".blenvy_gltf_settings"] if ".blenvy_gltf_settings" in bpy.data.texts else None
-
-        #check if params have changed
-        
-        # if there were no setting before, it is new, we need export
-        changed = False
-        if previous_auto_settings == None:
-            #print("previous settings missing, exporting")
-            changed = True
-        elif previous_gltf_settings == None:
-            #print("previous gltf settings missing, exporting")
-            previous_gltf_settings = bpy.data.texts.new(".blenvy_gltf_settings_previous")
-            previous_gltf_settings.write(json.dumps({}))
-            if current_gltf_settings == None:
-                current_gltf_settings = bpy.data.texts.new(".blenvy_gltf_settings")
-                current_gltf_settings.write(json.dumps({}))
-
-            changed = True
-
-        else:
-            auto_settings_changed = sorted(json.loads(previous_auto_settings.as_string()).items()) != sorted(json.loads(current_auto_settings.as_string()).items()) if current_auto_settings != None else False
-            gltf_settings_changed = sorted(json.loads(previous_gltf_settings.as_string()).items()) != sorted(json.loads(current_gltf_settings.as_string()).items()) if current_gltf_settings != None else False
-            
-            """print("auto settings previous", sorted(json.loads(previous_auto_settings.as_string()).items()))
-            print("auto settings current", sorted(json.loads(current_auto_settings.as_string()).items()))
-            print("auto_settings_changed", auto_settings_changed)
-
-            print("gltf settings previous", sorted(json.loads(previous_gltf_settings.as_string()).items()))
-            print("gltf settings current", sorted(json.loads(current_gltf_settings.as_string()).items()))
-            print("gltf_settings_changed", gltf_settings_changed)"""
-
-            changed = auto_settings_changed or gltf_settings_changed
-        # now write the current settings to the "previous settings"
-        if current_auto_settings != None:
-            previous_auto_settings = bpy.data.texts[".gltf_auto_export_settings_previous"] if ".gltf_auto_export_settings_previous" in bpy.data.texts else bpy.data.texts.new(".gltf_auto_export_settings_previous")
-            previous_auto_settings.clear()
-            previous_auto_settings.write(current_auto_settings.as_string()) # TODO : check if this is always valid
-
-        if current_gltf_settings != None:
-            previous_gltf_settings = bpy.data.texts[".blenvy_gltf_settings_previous"] if ".blenvy_gltf_settings_previous" in bpy.data.texts else bpy.data.texts.new(".blenvy_gltf_settings_previous")
-            previous_gltf_settings.clear()
-            previous_gltf_settings.write(current_gltf_settings.as_string())
-
-        return changed
+    return common_settings_changed or gltf_settings_changed or export_settings_changed

tools/blenvy/add_ons/auto_export/helpers/serialize_scene.py

@@ -208,7 +208,8 @@ def serialize_scene():
             collections = [collection.name for collection in object.users_collection]
             materials = materials_hash(object, cache) if len(object.material_slots) > 0 else None
 
-            data[scene.name][object.name] = {
+
+            object_field_hashes = {
                 "name": object.name,
                 "transforms": transform,
                 "visibility": visibility,
@@ -222,6 +223,9 @@ def serialize_scene():
                 "collections": collections,
                 "materials": materials
             }
+            object_field_hashes_filtered = {key: object_field_hashes[key] for key in object_field_hashes.keys() if object_field_hashes[key] is not None}
+            objectHash = str(hash(str(object_field_hashes_filtered)))
+            data[scene.name][object.name] = objectHash
 
     """print("data", data)
     print("")

tools/blenvy/add_ons/auto_export/settings.py

@@ -10,7 +10,7 @@ def save_settings(settings, context):
     if settings.settings_save_enabled:
         settings_dict =  generate_complete_settings_dict(settings, AutoExportSettings, [])
         print("save settings", settings, context, settings_dict)
-        upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list})
+        upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list}, overwrite=True)
 
 class AutoExportSettings(PropertyGroup):
 
@@ -24,7 +24,7 @@ class AutoExportSettings(PropertyGroup):
         update=save_settings
     ) # type: ignore
 
-    #### general
+    #### change detection
     change_detection: BoolProperty(
         name='Change detection',
         description='Use change detection to determine what/if should be exported',
@@ -32,6 +32,13 @@ class AutoExportSettings(PropertyGroup):
         update=save_settings
     ) # type: ignore
 
+    materials_in_depth_scan : BoolProperty(
+        name='In depth scan of materials (could be slow)',
+        description='serializes more details of materials in order to detect changes',
+        default=False,
+        update=save_settings
+    ) # type: ignore
+
     # scenes 
 
     # scene components

tools/blenvy/add_ons/bevy_components/settings.py

@@ -13,7 +13,7 @@ def save_settings(settings, context):
     if settings.settings_save_enabled:
         settings_dict = generate_complete_settings_dict(settings, ComponentsSettings, [])
         print("save settings", settings, context,settings_dict)
-        upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list})
+        upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list}, overwrite=True)
 
 # helper function to deal with timer
 def toggle_watcher(self, context):

tools/blenvy/blueprints/blueprints_registry.py

@@ -17,14 +17,13 @@ def refresh_blueprints():
     try:
         blueprints_registry = bpy.context.window_manager.blueprints_registry
         blueprints_registry.refresh_blueprints()
-        #print('refresh blueprints')
     except:pass
 
     return 3
 
 # this is where we store the information for all available Blueprints
 class BlueprintsRegistry(PropertyGroup):
-    blueprints_data = {}
+    blueprints_data = None
     blueprints_list = []
 
     asset_name_selector: StringProperty(
@@ -66,10 +65,13 @@ class BlueprintsRegistry(PropertyGroup):
         self.blueprints_list.append(blueprint)
 
     def refresh_blueprints(self):
+        #print("titi", self)
         blenvy = bpy.context.window_manager.blenvy
         settings = blenvy
         [main_scene_names, level_scenes, library_scene_names, library_scenes] = get_main_and_library_scenes(settings)
         blueprints_data = blueprints_scan(level_scenes, library_scenes, settings)
         self.blueprints_data = blueprints_data
+        return blueprints_data
+        #print("bla", self.blueprints_data)
         """for blueprint in blueprints_data.blueprints:
             self.add_blueprint(blueprint)"""

tools/blenvy/core/blenvy_manager.py

@@ -22,8 +22,8 @@ def save_settings(settings, context):
 def update_scene_lists(blenvy, context):                
     blenvy.main_scene_names = [scene.name for scene in blenvy.main_scenes] # FIXME: unsure
     blenvy.library_scene_names = [scene.name for scene in blenvy.library_scenes] # FIXME: unsure
-    upsert_settings(blenvy.settings_save_path, {"common_main_scene_names": [scene.name for scene in blenvy.main_scenes]})
-    upsert_settings(blenvy.settings_save_path, {"common_library_scene_names": [scene.name for scene in blenvy.library_scenes]})
+    upsert_settings(blenvy.settings_save_path, {"main_scene_names": [scene.name for scene in blenvy.main_scenes]})
+    upsert_settings(blenvy.settings_save_path, {"library_scene_names": [scene.name for scene in blenvy.library_scenes]})
 
 def update_asset_folders(blenvy, context):
     asset_path_names = ['project_root_path', 'assets_path', 'blueprints_path', 'levels_path', 'materials_path']
@@ -145,12 +145,12 @@ class BlenvyManager(PropertyGroup):
         if settings is not None:
             if "mode" in settings:
                 self.mode = settings["mode"]
-            if "common_main_scene_names" in settings:
-                for main_scene_name in settings["common_main_scene_names"]:
+            if "main_scene_names" in settings:
+                for main_scene_name in settings["main_scene_names"]:
                     added = self.main_scenes.add()
                     added.name = main_scene_name
-            if "common_library_scene_names" in settings:
-                for main_scene_name in settings["common_library_scene_names"]:
+            if "library_scene_names" in settings:
+                for main_scene_name in settings["library_scene_names"]:
                     added = self.library_scenes.add()
                     added.name = main_scene_name
 

tools/blenvy/settings.py

@@ -1,15 +1,19 @@
 import json
 import bpy
 
-def upsert_settings(name, data):
+def upsert_settings(name, data, overwrite=False):
     stored_settings = bpy.data.texts[name] if name in bpy.data.texts else None
     if stored_settings is None:
         stored_settings = bpy.data.texts.new(name)
         stored_settings.write(json.dumps(data))
     else:
-        current_settings = json.loads(stored_settings.as_string())
-        current_settings = {**current_settings, **data}
+        if overwrite:
             stored_settings.clear()  
+            stored_settings.write(json.dumps(data))
+        else:
+            current_settings = json.loads(stored_settings.as_string())
+            stored_settings.clear()
+            current_settings = {**current_settings, **data}
             stored_settings.write(json.dumps(current_settings))
 
 def load_settings(name):
@@ -66,17 +70,19 @@ def are_settings_identical(old, new, white_list=None):
     if old is not None and new is None:
         return False
     
+    #print("TUTU", old_items, new_items)
     old_items = sorted(old.items())
     new_items = sorted(new.items())
+
     if white_list is not None:
         old_items_override = {}
         new_items_override = {}
         for key in white_list:
-            if key in old_items:
-                old_items_override[key] = old_items[key]
-            if key in new_items:
-                new_items_override[key] = new_items[key]
-        old_items = old_items_override
-        new_items = new_items_override
+            if key in old:
+                old_items_override[key] = old[key]
+            if key in new:
+                new_items_override[key] = new[key]
+        old_items = sorted(old_items_override.items())
+        new_items = sorted(new_items_override.items())
 
-    return old_items != new_items if new is not None else False
+    return old_items == new_items