HDF5 support ?

[bjarthur]

first let me say i LOVE this package. have wanted sth like this for years but never had time to write it myself.

so to the point. would be nice to use this on HDF5 files. HDF5.jl now shows a static tree (see JuliaIO/HDF5.jl#725). how easy would it be to modify eyeball to dynamically browse the innards of an HDF5 file?

[tshort]

Good question. A start may be to define Eyeball.getoptions() for all HDF5 types that contain objects. Looks like that may at least include File, Group, and Dataset. Then define Eyeball.tostring if you need custom display of elements.

[mkitti]

I'm thinking about a package extension.

[mkitti]

Related:
https://discourse.julialang.org/t/how-does-nested-tab-completion-work-in-the-repl/102953/4

begin
	struct H5Indexer{O <: Union{HDF5.File,HDF5.Group}, C <: NamedTuple}
        parent::O
        children::C
	    function H5Indexer(p)
			pairs = Pair{Symbol,Any}[
				Symbol(k) => index(getindex(p,k))
				for k in keys(p)
			]
			#a = NamedTuple(Symbol(k) => v for (k,v) in attrs(p))
            #push!(pairs, :attrs => a)
            c = NamedTuple(pairs)
	    	new{typeof(p),typeof(c)}(p,c)
		end
	end
	index(h5o::Union{HDF5.File,HDF5.Group}) =
        H5Indexer(h5o)
    index(x) = x
	Base.parent(h5idx::H5Indexer) = getfield(h5idx, :parent)
	Base.getproperty(h5idx::H5Indexer, s::Symbol) =
		(@inline; getfield(getfield(h5idx, :children), s))
	Base.propertynames(h5idx::H5Indexer) =
        propertynames(getfield(h5idx, :children))
	Base.fieldnames(H5IDX::Type{H5Indexer{<: Any, C}}) where C =
		(@inline; fieldnames(C))
	Base.show(io::IO, m::MIME"text/plain", h5idx::H5Indexer) =
		show(io, m, getfield(h5idx, :parent))
end

This currently crashes at the moment for because it seeks information a bit too eagerly and crashes when MPI is not present.

With a small patch I will push soon, Eyeball now works with the above code:

julia> eye(h5idx)
[f] fields [d] docs [e] expand [m/M] methodswith [o] open [r] tree [s] show [S] Sort [t] typeof [z] summarize [.] size [q] quit
 >   : H5Indexer{HDF5.File, NamedTuple{(:METADATA, :ancillary_data, :ds_geosegments, :ds_metrics, :ds_surf_type, :gt1l, :gt1r, :gt2l, :gt2r, :gt3l, :gt3r, :orbit_info, :quality_assessment), Tuple{H5Indexer{HDF
      METADATA: H5Indexer{HDF5.Group, NamedTuple{(:AcquisitionInformation, :DataQuality, :DatasetIdentification, :Extent, :Lineage, :ProcessStep, :ProductSpecificationDocument, :QADatasetIdentification, :Serie
       AcquisitionInformation: H5Indexer{HDF5.Group, NamedTuple{(:lidar, :lidarDocument, :platform, :platformDocument), NTuple{4, H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}}}}   H5Indexer{Group, NamedTuple
        lidar: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/AcquisitionInformation/lidar (file: ATL08_20201121151145_08920913_006_01.h5), Nam
        lidarDocument: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/AcquisitionInformation/lidarDocument (file: ATL08_20201121151145_08920913
        platform: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/AcquisitionInformation/platform (file: ATL08_20201121151145_08920913_006_01.h5
        platformDocument: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/AcquisitionInformation/platformDocument (file: ATL08_20201121151145_08
       DataQuality: H5Indexer{HDF5.Group, NamedTuple{(:CompletenessOmission, :DomainConsistency), Tuple{H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}, H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}}}}   H5Ind
        CompletenessOmission: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/DataQuality/CompletenessOmission (file: ATL08_20201121151145_08920
        DomainConsistency: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/DataQuality/DomainConsistency (file: ATL08_20201121151145_08920913_00
       DatasetIdentification: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/DatasetIdentification (file: ATL08_20201121151145_08920913_006_01.
       Extent: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Extent (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
       Lineage: H5Indexer{HDF5.Group, NamedTuple{(Symbol("ANC06-01"), Symbol("ANC06-02"), Symbol("ANC06-03"), :ANC14, :ANC18, :ANC19, Symbol("ANC25-08"), Symbol("ANC26-08"), :ANC28, :ANC33, :ANC34, Symbol("ANC
        !   Showing items 1-10 of 17. Items remaining=7. Hit [e] to expand.
        ANC06-01: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC06-01 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple()
        ANC06-02: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC06-02 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple()
        ANC06-03: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC06-03 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple()
        ANC14: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC14 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
        ANC18: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC18 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
        ANC19: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC19 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
        ANC25-08: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC25-08 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple()
        ANC26-08: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC26-08 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple()
        ANC28: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC28 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
        ANC33: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/Lineage/ANC33 (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
       ProcessStep: H5Indexer{HDF5.Group, NamedTuple{(:Browse, :Metadata, :PGE, :QA), NTuple{4, H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}}}}   H5Indexer{Group, NamedTuple{(:Browse, :Metadata, :PGE, :QA),
        Browse: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/ProcessStep/Browse (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple()
        Metadata: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/ProcessStep/Metadata (file: ATL08_20201121151145_08920913_006_01.h5), NamedTup
        PGE: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/ProcessStep/PGE (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
        QA: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/ProcessStep/QA (file: ATL08_20201121151145_08920913_006_01.h5), NamedTuple())
       ProductSpecificationDocument: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/ProductSpecificationDocument (file: ATL08_20201121151145_08
       QADatasetIdentification: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/QADatasetIdentification (file: ATL08_20201121151145_08920913_006
       SeriesIdentification: H5Indexer{HDF5.Group, NamedTuple{(), Tuple{}}}   H5Indexer{Group, NamedTuple{(), Tuple{}}}(HDF5.Group: /METADATA/SeriesIdentification (file: ATL08_20201121151145_08920913_006_01.h5
      ancillary_data: H5Indexer{HDF5.Group, NamedTuple{(:atlas_sdp_gps_epoch, :control, :data_end_utc, :data_start_utc, :end_cycle, :end_delta_time, :end_geoseg, :end_gpssow, :end_gpsweek, :end_orbit, :end_reg
       !   Showing items 1-10 of 26. Items remaining=16. Hit [e] to expand.
       atlas_sdp_gps_epoch: HDF5.Dataset   HDF5.Dataset: /ancillary_data/atlas_sdp_gps_epoch (file: ATL08_20201121151145_08920913_006_01.h5 xfer_mode: 792633534417209041)
        id: Int64   360287970189639776
        file: HDF5.File   HDF5.File: (read-only) ATL08_20201121151145_08920913_006_01.h5
         id: Int64   72057594037927937
         filename: String   "ATL08_20201121151145_08920913_006_01.h5"
        xfer: HDF5.DatasetTransferProperties   DatasetTransferProperties(  dxpl_mpio       = :independent,)
         dxpl_mpio: Symbol   :independent
         id: Int64   792633534417209041
       control: HDF5.Dataset   HDF5.Dataset: /ancillary_data/control (file: ATL08_20201121151145_08920913_006_01.h5 xfer_mode: 792633534417209042)
        id: Int64   360287970189639777
        file: HDF5.File   HDF5.File: (read-only) ATL08_20201121151145_08920913_006_01.h5
         id: Int64   72057594037927937
         filename: String   "ATL08_20201121151145_08920913_006_01.h5"
        xfer: HDF5.DatasetTransferProperties   DatasetTransferProperties(  dxpl_mpio       = :independent,)
         dxpl_mpio: Symbol   :independent
v        id: Int64   792633534417209042

[mkitti]

JuliaIO/HDF5.jl#1121