feat(Blenvy): fixed all broken/ missing auto export elements

* fixed overhauled, cleaned up and improved settings change detection
 * serialize_scene now generate a hash per object instead of keeping the details (useless)
 * changed handling of gltf settings, so that only the actual non default, changed settings are saved
 * a ton of minor changes & cleanups to get export working correctly: it works !
 * started adding materials_in_depth_scan (wip) to be able to control more detailed (but costly) materials diffing for change detection
This commit is contained in:
kaosat.dev 2024-05-28 16:56:28 +02:00
parent 2187fee4c8
commit a0b1959a43
16 changed files with 107 additions and 127 deletions

View File

@ -90,7 +90,7 @@ General issues:
- [x] addon-prefs => settings
- [x] generate_gltf_export_preferences => should not use add-on prefs at all ? since we are not overriding gltf settings that way anymore ?
- [x] generate_gltf_export_settings => should not use add-on prefs at all ? since we are not overriding gltf settings that way anymore ?
- [x] remove hard coded path for standard gltf settings
- [x] load settings on file load
- [x] auto_export
@ -100,4 +100,5 @@ General issues:
- [ ] force overwrite of settings files instead of partial updates ?
- [ ] add tests for disabled components
- [ ] fix auto export workflow
- [x] fix auto export workflow
- [ ] should we write the previous _xxx data only AFTER a sucessfull export only ?

View File

@ -39,7 +39,7 @@ def gltf_post_export_callback(data):
gltf_export_settings.clear()
settings = dict(settings)
current_gltf_settings = generate_complete_settings_dict(settings, presets=ExportGLTF2_Base, ignore_list=["use_active_collection", "use_active_collection_with_nested", "use_active_scene", "use_selection", "will_save_settings", "gltf_export_id"], preset_defaults=True)
current_gltf_settings = generate_complete_settings_dict(settings, presets=ExportGLTF2_Base, ignore_list=["use_active_collection", "use_active_collection_with_nested", "use_active_scene", "use_selection", "will_save_settings", "gltf_export_id"], preset_defaults=False)
gltf_export_settings.write(json.dumps(current_gltf_settings))
# now reset the original gltf_settings
if gltf_settings_backup != "":

View File

@ -41,8 +41,9 @@ def auto_export(changes_per_scene, changed_export_parameters, settings):
[main_scene_names, level_scenes, library_scene_names, library_scenes] = get_main_and_library_scenes(settings)
bpy.context.window_manager.blueprints_registry.refresh_blueprints()
blueprints_data = bpy.context.window_manager.blueprints_registry.blueprints_data
blueprints_data = bpy.context.window_manager.blueprints_registry.refresh_blueprints()
#blueprints_data = bpy.context.window_manager.blueprints_registry.blueprints_data
#print("blueprints_data", blueprints_data)
blueprints_per_scene = blueprints_data.blueprints_per_scenes
internal_blueprints = [blueprint.name for blueprint in blueprints_data.internal_blueprints]
external_blueprints = [blueprint.name for blueprint in blueprints_data.external_blueprints]

View File

@ -1,15 +1,13 @@
import os
import bpy
from ..constants import TEMPSCENE_PREFIX
from ..helpers.generate_and_export import generate_and_export
from .export_gltf import (generate_gltf_export_preferences)
from ..helpers.helpers_scenes import clear_hollow_scene, copy_hollowed_collection_into
from .export_gltf import generate_gltf_export_settings
def export_blueprints(blueprints, settings, blueprints_data):
blueprints_path_full = getattr(settings, "blueprints_path_full")
gltf_export_preferences = generate_gltf_export_preferences(settings)
gltf_export_settings = generate_gltf_export_settings(settings)
try:
# save current active collection
@ -19,7 +17,7 @@ def export_blueprints(blueprints, settings, blueprints_data):
for blueprint in blueprints:
print("exporting collection", blueprint.name)
gltf_output_path = os.path.join(blueprints_path_full, blueprint.name)
gltf_export_settings = { **gltf_export_preferences, 'use_active_scene': True, 'use_active_collection': True, 'use_active_collection_with_nested':True}
gltf_export_settings = { **gltf_export_settings, 'use_active_scene': True, 'use_active_collection': True, 'use_active_collection_with_nested':True}
# if we are using the material library option, do not export materials, use placeholder instead
if export_materials_library:

View File

@ -8,9 +8,9 @@ def get_standard_exporter_settings():
standard_gltf_exporter_settings = load_settings(".blenvy_gltf_settings")
return standard_gltf_exporter_settings if standard_gltf_exporter_settings is not None else {}
def generate_gltf_export_preferences(settings):
def generate_gltf_export_settings(settings):
# default values
gltf_export_preferences = dict(
gltf_export_settings = dict(
# export_format= 'GLB', #'GLB', 'GLTF_SEPARATE', 'GLTF_EMBEDDED'
check_existing=False,
@ -49,7 +49,7 @@ def generate_gltf_export_preferences(settings):
"""for key in settings.__annotations__.keys():
if str(key) not in AutoExportGltfPreferenceNames:
#print("overriding setting", key, "value", getattr(settings,key))
pass#gltf_export_preferences[key] = getattr(settings, key)"""
pass#gltf_export_settings[key] = getattr(settings, key)"""
standard_gltf_exporter_settings = get_standard_exporter_settings()
@ -68,8 +68,10 @@ def generate_gltf_export_preferences(settings):
# a certain number of essential params should NEVER be overwritten , no matter the settings of the standard exporter
for key in standard_gltf_exporter_settings.keys():
if str(key) not in constant_keys:
gltf_export_preferences[key] = standard_gltf_exporter_settings.get(key)
return gltf_export_preferences
gltf_export_settings[key] = standard_gltf_exporter_settings.get(key)
print("GLTF EXPORT SETTINGS", gltf_export_settings)
return gltf_export_settings
#https://docs.blender.org/api/current/bpy.ops.export_scene.html#bpy.ops.export_scene.gltf

View File

@ -5,19 +5,19 @@ from blenvy.blueprints.blueprint_helpers import inject_blueprints_list_into_main
from ..constants import TEMPSCENE_PREFIX
from ..helpers.generate_and_export import generate_and_export
from .export_gltf import (generate_gltf_export_preferences, export_gltf)
from .export_gltf import (generate_gltf_export_settings, export_gltf)
from ..modules.bevy_dynamic import is_object_dynamic, is_object_static
from ..helpers.helpers_scenes import clear_hollow_scene, copy_hollowed_collection_into
def export_main_scene(scene, blend_file_path, settings, blueprints_data):
gltf_export_preferences = generate_gltf_export_preferences(settings)
gltf_export_settings = generate_gltf_export_settings(settings)
assets_path_full = getattr(settings,"assets_path_full")
levels_path_full = getattr(settings,"levels_path_full")
export_blueprints = getattr(settings.auto_export,"export_blueprints")
export_separate_dynamic_and_static_objects = getattr(settings.auto_export, "export_separate_dynamic_and_static_objects")
gltf_export_settings = { **gltf_export_preferences,
gltf_export_settings = { **gltf_export_settings,
'use_active_scene': True,
'use_active_collection':True,
'use_active_collection_with_nested':True,

View File

@ -6,7 +6,7 @@ from blenvy.core.helpers_collections import (traverse_tree)
from blenvy.core.object_makers import make_cube
from blenvy.materials.materials_helpers import get_all_materials
from ..helpers.generate_and_export import generate_and_export
from .export_gltf import (generate_gltf_export_preferences)
from .export_gltf import (generate_gltf_export_settings)
def clear_material_info(collection_names, library_scenes):
for scene in library_scenes:
@ -60,13 +60,13 @@ def clear_materials_scene(temp_scene):
# exports the materials used inside the current project:
# the name of the output path is <materials_folder>/<name_of_your_blend_file>_materials_library.gltf/glb
def export_materials(collections, library_scenes, settings):
gltf_export_preferences = generate_gltf_export_preferences(settings)
gltf_export_settings = generate_gltf_export_settings(settings)
materials_path_full = getattr(settings,"materials_path_full")
used_material_names = get_all_materials(collections, library_scenes)
current_project_name = Path(bpy.context.blend_data.filepath).stem
gltf_export_settings = { **gltf_export_preferences,
gltf_export_settings = { **gltf_export_settings,
'use_active_scene': True,
'use_active_collection':True,
'use_active_collection_with_nested':True,

View File

@ -15,7 +15,7 @@ def prepare_and_export():
# determine changed objects
per_scene_changes = get_changes_per_scene()
# determine changed parameters
setting_changes = get_setting_changes(auto_export_settings)
setting_changes = get_setting_changes()
# do the actual export
auto_export(per_scene_changes, setting_changes, blenvy)

View File

@ -1,6 +1,7 @@
import json
import bpy
from ..helpers.serialize_scene import serialize_scene
from blenvy.settings import load_settings, upsert_settings
def bubble_up_changes(object, changes_per_scene):
if object.parent:
@ -44,26 +45,25 @@ def serialize_current():
return current
def get_changes_per_scene():
previous = load_settings(".blenvy.project_serialized_previous")
current = serialize_current()
previous_stored = bpy.data.texts[".blenvy.project.serialized"] if ".blenvy.project.serialized" in bpy.data.texts else None
if previous_stored == None:
previous_stored = bpy.data.texts.new(".blenvy.project.serialized")
previous_stored.write(json.dumps(current))
return {}
previous = json.loads(previous_stored.as_string())
# determin changes
# determine changes
changes_per_scene = project_diff(previous, current)
# save the current project as previous
previous_stored.clear()
previous_stored.write(json.dumps(current))
upsert_settings(".blenvy.project_serialized_previous", current, overwrite=True)
print("changes per scene", changes_per_scene)
return changes_per_scene
def project_diff(previous, current):
"""print("previous", previous)
print("current", current)"""
if previous is None or current is None:
return {}
print("HERE")
changes_per_scene = {}
@ -100,5 +100,4 @@ def project_diff(previous, current):
bubble_up_changes(bpy.data.objects[object_name], changes_per_scene[scene])
# now bubble up for instances & parents
print("changes per scene", changes_per_scene)
return changes_per_scene

View File

@ -3,16 +3,18 @@ import bpy
from blenvy.settings import are_settings_identical, load_settings, upsert_settings
# which settings are specific to auto_export # TODO: can we infer this ?
auto_export_parameter_names = [
parameter_names_whitelist_common = [
# blenvy core
'project_root_path',
'assets_path',
'blueprints_path',
'levels_path',
'materials_path',
#'main_scene_names',
#'library_scene_names',
'main_scene_names',
'library_scene_names',
]
parameter_names_whitelist_auto_export = [
# auto export
'export_scene_settings',
'export_blueprints',
@ -22,76 +24,34 @@ auto_export_parameter_names = [
'export_marked_assets'
]
def get_setting_changes(auto_export_settings):
print("get setting changes", dict(auto_export_settings))
def get_setting_changes():
print("get setting changes")
previous_common_settings = load_settings(".blenvy_common_settings_previous")
current_common_settings = load_settings(".blenvy_common_settings")
common_settings_changed = not are_settings_identical(previous_common_settings, current_common_settings, white_list=parameter_names_whitelist_common)
previous_export_settings = load_settings(".blenvy_export_settings_previous")
current_export_settings = load_settings(".blenvy_export_settings")
export_settings_changed = not are_settings_identical(previous_export_settings, current_export_settings, white_list=parameter_names_whitelist_auto_export)
previous_gltf_settings = load_settings(".blenvy_gltf_settings_previous")
current_gltf_settings = load_settings(".blenvy_gltf_settings")
gltf_settings_changed = not are_settings_identical(previous_gltf_settings, current_gltf_settings)
previous_export_settings = load_settings(".blenvy_export_settings_previous")
current_export_settings = dict(auto_export_settings) #load_settings(".blenvy_export_settings")
export_settings_changed = not are_settings_identical(previous_export_settings, current_export_settings)
# if there were no setting before, it is new, we need export
if previous_gltf_settings is None:
pass
if previous_export_settings is None:
pass
# write the new settings to the old settings
upsert_settings(".blenvy_gltf_settings_previous", current_gltf_settings)
upsert_settings(".blenvy_export_settings_previous", current_export_settings)
upsert_settings(".blenvy_common_settings_previous", current_common_settings, overwrite=True)
upsert_settings(".blenvy_export_settings_previous", current_export_settings, overwrite=True)
upsert_settings(".blenvy_gltf_settings_previous", current_gltf_settings, overwrite=True)
return gltf_settings_changed or export_settings_changed
print("common_settings_changed", common_settings_changed,"export_settings_changed", export_settings_changed, "gltf_settings_changed", gltf_settings_changed, )
def did_export_settings_change(self):
return True
# compare both the auto export settings & the gltf settings
previous_auto_settings = bpy.data.texts[".gltf_auto_export_settings_previous"] if ".gltf_auto_export_settings_previous" in bpy.data.texts else None
previous_gltf_settings = bpy.data.texts[".blenvy_gltf_settings_previous"] if ".blenvy_gltf_settings_previous" in bpy.data.texts else None
# if there were no setting before, it is new, we need export # TODO: do we even need this ? I guess in the case where both the previous & the new one are both none ? very unlikely, but still
if previous_common_settings is None:
return True
if previous_export_settings is None:
return True
if previous_gltf_settings is None:
return True
current_auto_settings = bpy.data.texts[".gltf_auto_export_settings"] if ".gltf_auto_export_settings" in bpy.data.texts else None
current_gltf_settings = bpy.data.texts[".blenvy_gltf_settings"] if ".blenvy_gltf_settings" in bpy.data.texts else None
#check if params have changed
# if there were no setting before, it is new, we need export
changed = False
if previous_auto_settings == None:
#print("previous settings missing, exporting")
changed = True
elif previous_gltf_settings == None:
#print("previous gltf settings missing, exporting")
previous_gltf_settings = bpy.data.texts.new(".blenvy_gltf_settings_previous")
previous_gltf_settings.write(json.dumps({}))
if current_gltf_settings == None:
current_gltf_settings = bpy.data.texts.new(".blenvy_gltf_settings")
current_gltf_settings.write(json.dumps({}))
changed = True
else:
auto_settings_changed = sorted(json.loads(previous_auto_settings.as_string()).items()) != sorted(json.loads(current_auto_settings.as_string()).items()) if current_auto_settings != None else False
gltf_settings_changed = sorted(json.loads(previous_gltf_settings.as_string()).items()) != sorted(json.loads(current_gltf_settings.as_string()).items()) if current_gltf_settings != None else False
"""print("auto settings previous", sorted(json.loads(previous_auto_settings.as_string()).items()))
print("auto settings current", sorted(json.loads(current_auto_settings.as_string()).items()))
print("auto_settings_changed", auto_settings_changed)
print("gltf settings previous", sorted(json.loads(previous_gltf_settings.as_string()).items()))
print("gltf settings current", sorted(json.loads(current_gltf_settings.as_string()).items()))
print("gltf_settings_changed", gltf_settings_changed)"""
changed = auto_settings_changed or gltf_settings_changed
# now write the current settings to the "previous settings"
if current_auto_settings != None:
previous_auto_settings = bpy.data.texts[".gltf_auto_export_settings_previous"] if ".gltf_auto_export_settings_previous" in bpy.data.texts else bpy.data.texts.new(".gltf_auto_export_settings_previous")
previous_auto_settings.clear()
previous_auto_settings.write(current_auto_settings.as_string()) # TODO : check if this is always valid
if current_gltf_settings != None:
previous_gltf_settings = bpy.data.texts[".blenvy_gltf_settings_previous"] if ".blenvy_gltf_settings_previous" in bpy.data.texts else bpy.data.texts.new(".blenvy_gltf_settings_previous")
previous_gltf_settings.clear()
previous_gltf_settings.write(current_gltf_settings.as_string())
return changed
return common_settings_changed or gltf_settings_changed or export_settings_changed

View File

@ -208,7 +208,8 @@ def serialize_scene():
collections = [collection.name for collection in object.users_collection]
materials = materials_hash(object, cache) if len(object.material_slots) > 0 else None
data[scene.name][object.name] = {
object_field_hashes = {
"name": object.name,
"transforms": transform,
"visibility": visibility,
@ -222,6 +223,9 @@ def serialize_scene():
"collections": collections,
"materials": materials
}
object_field_hashes_filtered = {key: object_field_hashes[key] for key in object_field_hashes.keys() if object_field_hashes[key] is not None}
objectHash = str(hash(str(object_field_hashes_filtered)))
data[scene.name][object.name] = objectHash
"""print("data", data)
print("")

View File

@ -10,7 +10,7 @@ def save_settings(settings, context):
if settings.settings_save_enabled:
settings_dict = generate_complete_settings_dict(settings, AutoExportSettings, [])
print("save settings", settings, context, settings_dict)
upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list})
upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list}, overwrite=True)
class AutoExportSettings(PropertyGroup):
@ -24,7 +24,7 @@ class AutoExportSettings(PropertyGroup):
update=save_settings
) # type: ignore
#### general
#### change detection
change_detection: BoolProperty(
name='Change detection',
description='Use change detection to determine what/if should be exported',
@ -32,6 +32,13 @@ class AutoExportSettings(PropertyGroup):
update=save_settings
) # type: ignore
materials_in_depth_scan : BoolProperty(
name='In depth scan of materials (could be slow)',
description='serializes more details of materials in order to detect changes',
default=False,
update=save_settings
) # type: ignore
# scenes
# scene components

View File

@ -13,7 +13,7 @@ def save_settings(settings, context):
if settings.settings_save_enabled:
settings_dict = generate_complete_settings_dict(settings, ComponentsSettings, [])
print("save settings", settings, context,settings_dict)
upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list})
upsert_settings(settings.settings_save_path, {key: settings_dict[key] for key in settings_dict.keys() if key not in settings_black_list}, overwrite=True)
# helper function to deal with timer
def toggle_watcher(self, context):

View File

@ -17,14 +17,13 @@ def refresh_blueprints():
try:
blueprints_registry = bpy.context.window_manager.blueprints_registry
blueprints_registry.refresh_blueprints()
#print('refresh blueprints')
except:pass
return 3
# this is where we store the information for all available Blueprints
class BlueprintsRegistry(PropertyGroup):
blueprints_data = {}
blueprints_data = None
blueprints_list = []
asset_name_selector: StringProperty(
@ -66,10 +65,13 @@ class BlueprintsRegistry(PropertyGroup):
self.blueprints_list.append(blueprint)
def refresh_blueprints(self):
#print("titi", self)
blenvy = bpy.context.window_manager.blenvy
settings = blenvy
[main_scene_names, level_scenes, library_scene_names, library_scenes] = get_main_and_library_scenes(settings)
blueprints_data = blueprints_scan(level_scenes, library_scenes, settings)
self.blueprints_data = blueprints_data
return blueprints_data
#print("bla", self.blueprints_data)
"""for blueprint in blueprints_data.blueprints:
self.add_blueprint(blueprint)"""

View File

@ -22,8 +22,8 @@ def save_settings(settings, context):
def update_scene_lists(blenvy, context):
blenvy.main_scene_names = [scene.name for scene in blenvy.main_scenes] # FIXME: unsure
blenvy.library_scene_names = [scene.name for scene in blenvy.library_scenes] # FIXME: unsure
upsert_settings(blenvy.settings_save_path, {"common_main_scene_names": [scene.name for scene in blenvy.main_scenes]})
upsert_settings(blenvy.settings_save_path, {"common_library_scene_names": [scene.name for scene in blenvy.library_scenes]})
upsert_settings(blenvy.settings_save_path, {"main_scene_names": [scene.name for scene in blenvy.main_scenes]})
upsert_settings(blenvy.settings_save_path, {"library_scene_names": [scene.name for scene in blenvy.library_scenes]})
def update_asset_folders(blenvy, context):
asset_path_names = ['project_root_path', 'assets_path', 'blueprints_path', 'levels_path', 'materials_path']
@ -145,12 +145,12 @@ class BlenvyManager(PropertyGroup):
if settings is not None:
if "mode" in settings:
self.mode = settings["mode"]
if "common_main_scene_names" in settings:
for main_scene_name in settings["common_main_scene_names"]:
if "main_scene_names" in settings:
for main_scene_name in settings["main_scene_names"]:
added = self.main_scenes.add()
added.name = main_scene_name
if "common_library_scene_names" in settings:
for main_scene_name in settings["common_library_scene_names"]:
if "library_scene_names" in settings:
for main_scene_name in settings["library_scene_names"]:
added = self.library_scenes.add()
added.name = main_scene_name

View File

@ -1,16 +1,20 @@
import json
import bpy
def upsert_settings(name, data):
def upsert_settings(name, data, overwrite=False):
stored_settings = bpy.data.texts[name] if name in bpy.data.texts else None
if stored_settings is None:
stored_settings = bpy.data.texts.new(name)
stored_settings.write(json.dumps(data))
else:
current_settings = json.loads(stored_settings.as_string())
current_settings = {**current_settings, **data}
stored_settings.clear()
stored_settings.write(json.dumps(current_settings))
if overwrite:
stored_settings.clear()
stored_settings.write(json.dumps(data))
else:
current_settings = json.loads(stored_settings.as_string())
stored_settings.clear()
current_settings = {**current_settings, **data}
stored_settings.write(json.dumps(current_settings))
def load_settings(name):
stored_settings = bpy.data.texts[name] if name in bpy.data.texts else None
@ -66,17 +70,19 @@ def are_settings_identical(old, new, white_list=None):
if old is not None and new is None:
return False
#print("TUTU", old_items, new_items)
old_items = sorted(old.items())
new_items = sorted(new.items())
if white_list is not None:
old_items_override = {}
new_items_override = {}
for key in white_list:
if key in old_items:
old_items_override[key] = old_items[key]
if key in new_items:
new_items_override[key] = new_items[key]
old_items = old_items_override
new_items = new_items_override
if key in old:
old_items_override[key] = old[key]
if key in new:
new_items_override[key] = new[key]
old_items = sorted(old_items_override.items())
new_items = sorted(new_items_override.items())
return old_items != new_items if new is not None else False
return old_items == new_items