From d8a7d837bd9e5bc1c8b7ee86e9196b45385fd6c6 Mon Sep 17 00:00:00 2001 From: foreverallama Date: Wed, 10 Dec 2025 23:49:11 +0530 Subject: [PATCH 1/2] New type FunctionHandle with RW support in v7 and HDF formats --- src/MAT.jl | 2 +- src/MAT_HDF5.jl | 4 +++- src/MAT_types.jl | 12 +++++++++++- src/MAT_v5.jl | 9 +++++++-- test/read.jl | 12 ++++++------ 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/MAT.jl b/src/MAT.jl index bac8825..17e6150 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -37,7 +37,7 @@ include("MAT_v4.jl") using .MAT_HDF5, .MAT_v5, .MAT_v4, .MAT_subsys export matopen, matread, matwrite, @read, @write -export MatlabStructArray, MatlabClassObject, MatlabOpaque, MatlabTable +export MatlabStructArray, MatlabClassObject, MatlabOpaque, MatlabTable, FunctionHandle # Open a MATLAB file const HDF5_HEADER = UInt8[0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a] diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 894f401..ef6f956 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -324,7 +324,9 @@ function m_read(g::HDF5.Group, subsys::Subsystem) if haskey(attr, sparse_attr_matlab) return read_sparse_matrix(g, mattype) elseif mattype == "function_handle" - # TODO: fall through for now, will become a Dict + if haskey(attr, object_decode_attr_matlab) && read_attribute(g, object_decode_attr_matlab)==1 + is_object = true + end else if haskey(attr, object_decode_attr_matlab) && read_attribute(g, object_decode_attr_matlab)==2 # I think this means it's an old object class similar to mXOBJECT_CLASS in MAT_v5 diff --git a/src/MAT_types.jl b/src/MAT_types.jl index 9167de0..2e856b0 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -41,6 +41,7 @@ export MatlabClassObject export MatlabOpaque, convert_opaque export MatlabTable export ScalarOrArray +export FunctionHandle const ScalarOrArray{T} = Union{T, AbstractArray{T}} @@ -176,7 +177,7 @@ function Base.:(==)(m1::MatlabStructArray{N}, m2::MatlabStructArray{N}) where {N end function Base.isapprox(m1::MatlabStructArray, m2::MatlabStructArray; kwargs...) - return isequal(m1.class, m2.class) && issetequal(m1.names, m2.names) && + return isequal(m1.class, m2.class) && issetequal(m1.names, m2.names) && key_based_isapprox(m1.names, m1, m2; kwargs...) end @@ -300,6 +301,13 @@ struct EmptyStruct end class(m::EmptyStruct) = "" +""" +Function Handles which are stored as structs +""" +struct FunctionHandle + d::Dict{String,Any} +end + """ MatlabClassObject( d::Dict{String, Any}, @@ -357,6 +365,8 @@ function convert_struct_array(d::AbstractDict{String,Any}, class::String="") else if isempty(class) return d + elseif class == "function_handle" + return FunctionHandle(d) else return MatlabClassObject(d, class) end diff --git a/src/MAT_v5.jl b/src/MAT_v5.jl index b268559..3d3d553 100644 --- a/src/MAT_v5.jl +++ b/src/MAT_v5.jl @@ -28,7 +28,7 @@ module MAT_v5 using CodecZlib, HDF5, SparseArrays import Base: read, write, close -import ..MAT_types: MatlabStructArray, MatlabClassObject, MatlabTable +import ..MAT_types: MatlabStructArray, MatlabClassObject, MatlabTable, FunctionHandle using ..MAT_subsys @@ -325,6 +325,11 @@ function read_string(f::IO, swap_bytes::Bool, dimensions::Vector{Int32}) data end +function read_function_handle(f::IO, swap_bytes::Bool, subsys::Subsystem) + data = read_matrix(f, swap_bytes, subsys)[2] # read_matrix returns (var_name, data) + return FunctionHandle(data) +end + function read_opaque(f::IO, swap_bytes::Bool, subsys::Subsystem) type_name = String(read_element(f, swap_bytes, UInt8)) classname = String(read_element(f, swap_bytes, UInt8)) @@ -375,7 +380,7 @@ function read_matrix(f::IO, swap_bytes::Bool, subsys::Subsystem) elseif class == mxCHAR_CLASS && length(dimensions) <= 2 data = read_string(f, swap_bytes, dimensions) elseif class == mxFUNCTION_CLASS - data = read_matrix(f, swap_bytes, subsys) + data = read_function_handle(f, swap_bytes, subsys) elseif class == mxOPAQUE_CLASS data = read_opaque(f, swap_bytes, subsys) else diff --git a/test/read.jl b/test/read.jl index d7d00dd..9121bdb 100644 --- a/test/read.jl +++ b/test/read.jl @@ -222,12 +222,12 @@ end # test reading file containing Matlab function handle, table, and datetime objects let objtestfile = "function_handles.mat" vars = matread(joinpath(dirname(@__FILE__), "v7.3", objtestfile)) - @test "sin" in keys(vars) - @test typeof(vars["sin"]) == Dict{String, Any} - @test Set(keys(vars["sin"])) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) - @test "anonymous" in keys(vars) - @test typeof(vars["anonymous"]) == Dict{String, Any} - @test Set(keys(vars["anonymous"])) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) + @test haskey(vars, "sin") + @test haskey(vars, "anonymous") + @test vars["sin"] isa FunctionHandle + @test vars["anonymous"] isa FunctionHandle + @test Set(keys(vars["sin"].d)) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) + @test Set(keys(vars["anonymous"].d)) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) end for format in ["v7", "v7.3"] From 049b696ef8157e2db8c639101a3c4e7aa1f05389 Mon Sep 17 00:00:00 2001 From: foreverallama Date: Thu, 11 Dec 2025 16:27:32 +0530 Subject: [PATCH 2/2] Write support for function handles; bug fixes in subsystem * MAT_HDF: New method m_write() to write function handles * MAT_HDF: New constants for MATLAB_object_decode 1 to 3 * write.jl: Add RW test * read.jl: Remove standalone read test * MAT_subsys: Fix for writing MatlabOpaque objects with object ID = 0 * MAT_subsys: Added class "function_handle_workspace" to list of saveobj classes * MAT_subsys: Fix for serializing nested objects --- src/MAT_HDF5.jl | 29 +++++++++++++++++++++++++---- src/MAT_subsys.jl | 22 ++++++++++++++-------- test/read.jl | 11 ----------- test/write.jl | 29 +++++++++++++++++++++++++++-- 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index ef6f956..061f4be 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -45,7 +45,8 @@ import ..MAT_types: MatlabStructArray, MatlabTable, ScalarOrArray, - StructArrayField + StructArrayField, + FunctionHandle const HDF5Parent = Union{HDF5.File, HDF5.Group} const HDF5BitsOrBool = Union{HDF5.BitsType,Bool} @@ -160,6 +161,10 @@ const object_type_attr_matlab = "MATLAB_object_decode" const object_decode_attr_matlab = "MATLAB_object_decode" const struct_field_attr_matlab = "MATLAB_fields" +const matlab_function_decode = UInt32(1) +const matlab_object_decode = UInt32(2) +const matlab_opaque_decode = UInt32(3) + ### Reading function read_complex(dtype::HDF5.Datatype, dset::HDF5.Dataset, ::Type{T}) where T if !check_datatype_complex(dtype) @@ -729,6 +734,22 @@ function check_struct_keys(k::Vector) asckeys end +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::FunctionHandle) + g = create_group(parent, name) + try + write_attribute(g, name_type_attr_matlab, "function_handle") + write_attribute(g, object_decode_attr_matlab, matlab_function_decode) + obj_struct = obj.d + all_keys = collect(keys(obj_struct)) + _write_struct_fields(mfile, g, all_keys) + for (ki, vi) in zip(all_keys, values(obj_struct)) + m_write(mfile, g, ki, vi) + end + finally + close(g) + end +end + function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::AbstractArray{MatlabClassObject}) m_write(mfile, parent, name, MatlabStructArray(arr)) end @@ -737,7 +758,7 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M g = create_group(parent, name) try write_attribute(g, name_type_attr_matlab, obj.class) - write_attribute(g, object_decode_attr_matlab, UInt32(2)) + write_attribute(g, object_decode_attr_matlab, matlab_object_decode) all_keys = collect(keys(obj)) _write_struct_fields(mfile, g, all_keys) for (ki, vi) in zip(all_keys, values(obj)) @@ -827,7 +848,7 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M try write_dataset(dset, dtype, metadata) write_attribute(dset, name_type_attr_matlab, obj.class) - write_attribute(dset, object_type_attr_matlab, UInt32(3)) + write_attribute(dset, object_type_attr_matlab, matlab_opaque_decode) finally close(dset) close(dtype) @@ -841,7 +862,7 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::A # TODO: Handle empty array case write_dataset(dset, dtype, metadata) write_attribute(dset, name_type_attr_matlab, first(obj).class) - write_attribute(dset, object_type_attr_matlab, UInt32(3)) + write_attribute(dset, object_type_attr_matlab, matlab_opaque_decode) finally close(dset) close(dtype) diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index 7a21dbb..585f90c 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -36,7 +36,8 @@ const MCOS_IDENTIFIER = 0xdd000000 const matlab_saveobj_ret_types = String[ "string", - "timetable" + "timetable", + "function_handle_workspace" ] # Warning message for unknown regions @@ -556,22 +557,27 @@ end save_nested_props(prop_value, subsys::Subsystem) = prop_value +# Prop values are copied to avoid mutating original with object metadata +# Should be cheap as underlying data will be shared by reference unless modified + function save_nested_props( prop_value::Union{AbstractDict,MatlabStructArray}, subsys::Subsystem ) # Save nested objects in structs + prop_value_copy = copy(prop_value) for (key, value) in prop_value - prop_value[key] = save_nested_props(value, subsys) + prop_value_copy[key] = save_nested_props(value, subsys) end - return prop_value + return prop_value_copy end function save_nested_props(prop_value::Array{Any}, subsys::Subsystem) # Save nested objects in a Cell + prop_value_copy = copy(prop_value) for i in eachindex(prop_value) - prop_value[i] = save_nested_props(prop_value[i], subsys) + prop_value_copy[i] = save_nested_props(prop_value[i], subsys) end - return prop_value + return prop_value_copy end function save_nested_props(prop_value::Union{MatlabOpaque, Array{MatlabOpaque}}, subsys::Subsystem) @@ -581,8 +587,8 @@ function save_nested_props(prop_value::Union{MatlabOpaque, Array{MatlabOpaque}}, # FIXME: Does this overwrite prop_value from the user dict? # Might have to create a copy instead - Test needed - prop_value = set_mcos_object_metadata(subsys, prop_value) - return prop_value + prop_metadata = set_mcos_object_metadata(subsys, prop_value) + return prop_metadata end function serialize_object_props!(subsys::Subsystem, obj::MatlabOpaque, obj_prop_metadata::Vector{UInt32}) @@ -629,7 +635,7 @@ function set_object_id(subsys::Subsystem, obj::MatlabOpaque, saveobj_ret_type=fa # This is a deleted object # MATLAB keeps weak references to deleted objects for some reason class_id = set_class_id!(subsys, obj.class) - obj_id = 0 + obj_id = UInt32(0) return obj_id, class_id end diff --git a/test/read.jl b/test/read.jl index 9121bdb..0472a0c 100644 --- a/test/read.jl +++ b/test/read.jl @@ -219,17 +219,6 @@ let objtestfile = "figure.fig" @test vars["hgS_070000"]["type"] == "figure" end -# test reading file containing Matlab function handle, table, and datetime objects -let objtestfile = "function_handles.mat" - vars = matread(joinpath(dirname(@__FILE__), "v7.3", objtestfile)) - @test haskey(vars, "sin") - @test haskey(vars, "anonymous") - @test vars["sin"] isa FunctionHandle - @test vars["anonymous"] isa FunctionHandle - @test Set(keys(vars["sin"].d)) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) - @test Set(keys(vars["anonymous"].d)) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) -end - for format in ["v7", "v7.3"] @testset "struct_table_datetime $format" begin let objtestfile = "struct_table_datetime.mat" diff --git a/test/write.jl b/test/write.jl index f061572..6eba1a9 100644 --- a/test/write.jl +++ b/test/write.jl @@ -15,8 +15,8 @@ function test_write_data(data; approx = false, kwargs...) end if approx - @test MAT.MAT_types.dict_isapprox(result, data) - else + @test MAT.MAT_types.dict_isapprox(result, data) + else @test isequal(result, data) end end @@ -355,3 +355,28 @@ end test_write(Dict{String,Any}("ms" => ms)) test_write(Dict{String,Any}("ms" => [ms, Millisecond(50000)])) end + +@testset "function handles" begin + let objtestfile = "function_handles.mat" + filepath = joinpath(dirname(@__FILE__), "v7.3", objtestfile) + vars = matread(filepath) + + mktempdir() do tmpdir + tmpfile = joinpath(tmpdir, "test.mat") + matwrite(tmpfile, vars) + vars_write = matread(tmpfile) + + @test haskey(vars_write, "sin") + @test haskey(vars_write, "anonymous") + + @test isa(vars_write["sin"], FunctionHandle) + @test isa(vars_write["anonymous"], FunctionHandle) + + @test Set(keys(vars_write["sin"].d)) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) + @test Set(keys(vars_write["anonymous"].d)) == Set(["function_handle", "sentinel", "separator", "matlabroot"]) + + @test isequal(vars_write["sin"].d, vars["sin"].d) + @test isequal(vars_write["anonymous"].d, vars["anonymous"].d) + end + end +end