From 3e11d302bdc8dd49aea276668ec5bc17a699a499 Mon Sep 17 00:00:00 2001 From: Kerim Khemraev Date: Tue, 6 Jun 2023 16:04:50 +0300 Subject: [PATCH 1/3] Added support for little endian SEGY files SEGY may have BIG or LITTLE endian byte order. Machine also may have BIG or LITTLE endian order. If machine and SEGY have different byte order then we have to swap bytes while reading data. To check whether machine and SEGY byte order are same we can use 3225-3226 bytes (Data Sample Format). According to SEGY revision document this value may have only value in range 1-16. Than concerns any type greater > 1 byte (even INT16, FLOAT32 or IBMFLOAT32 or any other). For `bswap_needed` correct work it is important that stream (`s::IO)` is opened from the begining of the file. Thus we can't use the stream when reading traces or trace header (there we pass stream that is opened from some offset). That is why in some functions we don't have default value for `swap_bytes::Bool` argument. --- src/SegyIO.jl | 1 + src/read/endianness.jl | 68 ++++++++++++++++++++++++++++++++++ src/read/read_block.jl | 3 +- src/read/read_block_headers.jl | 6 ++- src/read/read_file.jl | 7 +++- src/read/read_fileheader.jl | 12 +++--- src/read/read_trace.jl | 27 +++++++------- src/read/read_traceheader.jl | 22 +++++------ src/scan/scan_block.jl | 8 +++- src/scan/scan_shots.jl | 6 ++- src/types/IBMFloat32.jl | 6 ++- 11 files changed, 126 insertions(+), 40 deletions(-) create mode 100644 src/read/endianness.jl diff --git a/src/SegyIO.jl b/src/SegyIO.jl index 3417923..7897c77 100644 --- a/src/SegyIO.jl +++ b/src/SegyIO.jl @@ -19,6 +19,7 @@ module SegyIO include("types/SeisCon.jl") #Reader + include("read/endianness.jl") include("read/read_fileheader.jl") include("read/read_traceheader.jl") include("read/read_trace.jl") diff --git a/src/read/endianness.jl b/src/read/endianness.jl new file mode 100644 index 0000000..71efde8 --- /dev/null +++ b/src/read/endianness.jl @@ -0,0 +1,68 @@ +is_machine_littleendian() = Base.ENDIAN_BOM == 0x04030201 +is_machine_bigendian() = Base.ENDIAN_BOM == 0x01020304 + +function is_segy_littleendian(s::IO) + bfh = BinaryFileHeader() + fh_b2s = fh_byte2sample() + + old_pos = position(s) + seek(s, fh_b2s["DataSampleFormat"]) + dsf = read(s, typeof(bfh.DataSampleFormat)) + dsf_bs = bswap(dsf) + seek(s, old_pos) + + if (dsf >= 1 && dsf <= 16 && is_machine_littleendian()) || + (dsf_bs >= 1 && dsf_bs <= 16 && is_machine_bigendian()) + return true + else + return false + end +end + +function is_segy_bigendian(s::IO) + bfh = BinaryFileHeader() + fh_b2s = fh_byte2sample() + + old_pos = position(s) + seek(s, fh_b2s["DataSampleFormat"]) + dsf = read(s, typeof(bfh.DataSampleFormat)) + dsf = bswap(read(s, typeof(bfh.DataSampleFormat))) + seek(s, old_pos) + + if (dsf >= 1 && dsf <= 16 && is_machine_bigendian()) || + (dsf_bs >= 1 && dsf_bs <= 16 && is_machine_littleendian()) + return true + else + return false + end +end + + +""" +bswap_needed(s::IO) + +Checks whether SEGY and host machine's byte order are same or not. + +NOTE: comparison is done based on binary file header value (Data Sample Format) +thus stream (s::IO) must be opened from the begining of the file. + +# Examples + +s = open("/my/segy.segy", "r") +swap_bytes = bswap_needed(s) +""" +function bswap_needed(s::IO) + bfh = BinaryFileHeader() + fh_b2s = fh_byte2sample() + + old_pos = position(s) + seek(s, fh_b2s["DataSampleFormat"]) + dsf = read(s, typeof(bfh.DataSampleFormat)) + seek(s, old_pos) + + if dsf >= 1 && dsf <= 16 + return false + else + return true + end +end \ No newline at end of file diff --git a/src/read/read_block.jl b/src/read/read_block.jl index 0acd149..d59f0d8 100644 --- a/src/read/read_block.jl +++ b/src/read/read_block.jl @@ -7,6 +7,7 @@ end function read_block!(b::BlockScan, keys::Array{String, 1}, ns::Int, dsf::Int, tmp_data, tmp_headers) f = open(b.file) + swap_bytes = bswap_needed(f) seek(f, b.startbyte) brange = b.endbyte - b.startbyte s = IOBuffer(read(f, brange)) @@ -29,6 +30,6 @@ function read_block!(b::BlockScan, keys::Array{String, 1}, ns::Int, dsf::Int, tm tracee = min(trace + TRACE_CHUNKSIZE - 1, ntraces) chunk = length(trace:tracee)*trace_size sloc = IOBuffer(read(s, chunk)) - read_traces!(sloc, view(tmp_headers, trace:tracee), view(tmp_data, :, trace:tracee), keys, th_b2s) + read_traces!(sloc, view(tmp_headers, trace:tracee), view(tmp_data, :, trace:tracee), keys, th_b2s; swap_bytes=swap_bytes) end end diff --git a/src/read/read_block_headers.jl b/src/read/read_block_headers.jl index 6294fdf..08bee61 100644 --- a/src/read/read_block_headers.jl +++ b/src/read/read_block_headers.jl @@ -3,6 +3,7 @@ export read_block_headers! function read_block_headers!(b::BlockScan, keys::Array{String, 1}, ns::Int, dsf::Int, headers) f = open(b.file) + swap_bytes = bswap_needed(f) seek(f, b.startbyte) brange = b.endbyte - b.startbyte s = IOBuffer(read(f, brange)) @@ -23,7 +24,7 @@ function read_block_headers!(b::BlockScan, keys::Array{String, 1}, ns::Int, dsf: tmph = zeros(UInt8, 240) # Read each traceheader for trace in 1:ntraces - read_traceheader!(s, keys, th_b2s, headers[trace]; th=tmph) + read_traceheader!(s, keys, th_b2s, headers[trace]; swap_bytes=swap_bytes, th=tmph) skip(s, ns*4) end @@ -32,6 +33,7 @@ end function read_block_headers!(b::BlockScan, ns::Int, dsf::Int, headers) f = open(b.file) + swap_bytes = bswap_needed(f) seek(f, b.startbyte) brange = b.endbyte - b.startbyte s = IOBuffer(read(f, brange)) @@ -50,7 +52,7 @@ function read_block_headers!(b::BlockScan, ns::Int, dsf::Int, headers) th_b2s = th_byte2sample() # Read each traceheader for trace in 1:ntraces - read_traceheader!(s, th_b2s, headers[trace]) + read_traceheader!(s, th_b2s, headers[trace]; swap_bytes=swap_bytes) skip(s, ns*4) end diff --git a/src/read/read_file.jl b/src/read/read_file.jl index 2eff124..70a3951 100644 --- a/src/read/read_file.jl +++ b/src/read/read_file.jl @@ -18,8 +18,11 @@ Read entire SEGY file from stream 's', only reading the header values in 'keys'. """ function read_file(s::IO, keys::Array{String, 1}, warn_user::Bool; start_byte::Int = 3600, end_byte::Int = position(seekend(s))) + + swap_bytes = bswap_needed(s) + # Read File Header - fh = read_fileheader(s) + fh = read_fileheader(s; swap_bytes=swap_bytes) # Move to start of block seek(s, start_byte) @@ -53,7 +56,7 @@ function read_file(s::IO, keys::Array{String, 1}, warn_user::Bool; tracee = min(trace + TRACE_CHUNKSIZE - 1, ntraces) chunk = length(trace:tracee)*trace_size sloc = IOBuffer(read(s, chunk)) - read_traces!(sloc, view(headers, trace:tracee), view(data, :, trace:tracee), keys, th_b2s) + read_traces!(sloc, view(headers, trace:tracee), view(data, :, trace:tracee), keys, th_b2s; swap_bytes=swap_bytes) end return SeisBlock(fh, headers, data) diff --git a/src/read/read_fileheader.jl b/src/read/read_fileheader.jl index 24efaa2..9ca056b 100644 --- a/src/read/read_fileheader.jl +++ b/src/read/read_fileheader.jl @@ -4,17 +4,17 @@ export read_fileheader """ # Info -Use: fileheader = read_fileheader(s::IO; bigendian::Bool = true) +Use: fileheader = read_fileheader(s::IO; swap_bytes::Bool = bswap_needed(s)) Returns a binary file header formed using bytes 3200-3600 from the stream 's'. """ -function read_fileheader(s::IO; bigendian::Bool = true) - return read_fileheader(s, fh_keys(); bigendian=bigendian) +function read_fileheader(s::IO; swap_bytes::Bool = bswap_needed(s)) + return read_fileheader(s, fh_keys(); swap_bytes=swap_bytes) end """ -Use: fileheader = read_fileheader(s::IO, keys::Array{String,1}; bigendian::Bool = true) +Use: fileheader = read_fileheader(s::IO, keys::Array{String,1}; swap_bytes::Bool = bswap_needed(s)) Return a fileheader from stream 's' with the fields defined in 'keys'. @@ -41,7 +41,7 @@ Read only the sample interval and number of traces from the file header. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Dict("expf"=>3226,"sfe"=>3234,"rgc"=>3250,"jobid"=>3200, "dt"=>3216,"nsfr"=>3222,"slen"=>3236,"vpol"=>3258,"renum"=>3208,"dsf"=>3224…)) """ -function read_fileheader(s::IO, keys::Array{String,1}; bigendian::Bool = true) +function read_fileheader(s::IO, keys::Array{String,1}; swap_bytes::Bool = bswap_needed(s)) # Return to start of stream seekstart(s) @@ -52,7 +52,7 @@ function read_fileheader(s::IO, keys::Array{String,1}; bigendian::Bool = true) # Initialize binary file header bfh = BinaryFileHeader() fh_b2s = fh_byte2sample() - swp(x) = bigendian ? bswap(x) : x + swp(x) = swap_bytes ? bswap(x) : x for k in keys diff --git a/src/read/read_trace.jl b/src/read/read_trace.jl index bad4a6e..036c79c 100644 --- a/src/read/read_trace.jl +++ b/src/read/read_trace.jl @@ -1,54 +1,53 @@ -export read_trace +export read_traces """ -Use: read_trace!(s::IO, +Use: read_traces!(s::IO, fh::BinaryFileHeader, datatype::Type, headers::AbstractArray{BinaryTraceHeader,1}, data::AbstractArray{<:Union{IBMFloat32, Float32}, 2}, trace::Int, - th_byte2sample::Dict{String,Int32}) + th_byte2sample::Dict{String,Int32}; + swap_bytes::Bool) Reads 'trace' from the current position of stream 's' into 'headers' and 'data'. """ function read_traces!(s::IO, headers::AbstractVector{BinaryTraceHeader}, data::AbstractMatrix{<:Union{IBMFloat32, Float32}}, - th_byte2sample::Dict{String,Int32}) + th_byte2sample::Dict{String,Int32}; + swap_bytes::Bool) - return read_traces!(s, headers, data, collect(keys(th_byte2sample)), th_byte2sample) + return read_traces!(s, headers, data, collect(keys(th_byte2sample)), th_byte2sample; swap_bytes=swap_bytes) end """ -Use: read_trace!(s::IO, +Use: read_traces!(s::IO, fh::BinaryFileHeader, datatype::Type, headers::AbstractArray{BinaryTraceHeader,1}, data::AbstractArray{<:Union{IBMFloat32, Float32}, 2}, trace::Int, keys::Array{String,1}, - th_byte2sample::Dict{String,Int32}) + th_byte2sample::Dict{String,Int32}; + swap_bytes::Bool) Reads 'trace' from the current position of stream 's' into 'headers' and 'data'. Only the header values in 'keys' and read. """ function read_traces!(s::IO, headers::AbstractVector{BinaryTraceHeader}, data::AbstractMatrix{DT}, keys::Array{String,1}, - th_byte2sample::Dict{String,Int32}) where {DT<:Union{IBMFloat32, Float32}} + th_byte2sample::Dict{String,Int32}; swap_bytes::Bool) where {DT<:Union{IBMFloat32, Float32}} ntrace = size(data, 2) ntrace == 0 && return - swp = swp_func(DT) tmph = zeros(UInt8, 240) for trace_loc=0:ntrace-1 # Read trace header - read_traceheader!(s, keys, th_byte2sample, headers[trace_loc+1]; th=tmph) + read_traceheader!(s, keys, th_byte2sample, headers[trace_loc+1]; swap_bytes=swap_bytes, th=tmph) # Read trace read!(s, view(data, :, trace_loc+1)) end - map!(swp, data, data) + (swap_bytes) && (data = bswap.(data)) nothing end - -swp_func(::Type{Float32}) = bswap -swp_func(::Any) = x -> x diff --git a/src/read/read_traceheader.jl b/src/read/read_traceheader.jl index 6036620..10d8fb7 100644 --- a/src/read/read_traceheader.jl +++ b/src/read/read_traceheader.jl @@ -3,34 +3,34 @@ export read_traceheader """ # Info -Use: fileheader = read_fileheader(s::IO; bigendian::Bool = true) +Use: fileheader = read_fileheader(s::IO; swap_bytes::Bool) Returns a binary trace header formed from the current position in the stream 's'. """ -function read_traceheader(s::IO, th_byte2sample::Dict{String,Int32}; bigendian::Bool=true, th=zeros(UInt8, 240)) - return read_traceheader(s, collect(keys(th_byte2sample)), th_byte2sample; bigendian=bigendian, th=th) +function read_traceheader(s::IO, th_byte2sample::Dict{String,Int32}; swap_bytes::Bool, th=zeros(UInt8, 240)) + return read_traceheader(s, collect(keys(th_byte2sample)), th_byte2sample; swap_bytes=swap_bytes, th=th) end """ -Use: fileheader = read_traceheader(s::IO, keys = Array{String,1}; bigendian::Bool = true) +Use: fileheader = read_traceheader(s::IO, keys = Array{String,1}; swap_bytes::Bool) Returns a binary trace header formed from the current position in the stream 's', only reading header values denoted in 'keys'. """ function read_traceheader(s::IO, keys::Array{String,1}, th_byte2sample::Dict{String, Int32}; - bigendian::Bool = true, th=zeros(UInt8, 240)) + swap_bytes::Bool, th=zeros(UInt8, 240)) # Initialize binary file header traceheader = BinaryTraceHeader() - read_traceheader!(s, keys, th_byte2sample, traceheader; bigendian=bigendian, th=th) + read_traceheader!(s, keys, th_byte2sample, traceheader; swap_bytes=swap_bytes, th=th) return traceheader end function read_traceheader!(s::IO, keys::Array{String,1}, th_byte2sample::Dict{String, Int32}, hdr::BinaryTraceHeader; - bigendian::Bool = true, th=zeros(UInt8, 240)) - + swap_bytes::Bool, th=zeros(UInt8, 240)) + # read full trace header then split read!(s, th) @@ -40,11 +40,11 @@ function read_traceheader!(s::IO, keys::Array{String,1}, th_byte2sample::Dict{St nb = sizeof(getfield(hdr, sym)) - 1 bst = th_byte2sample[k]+1 val = reinterpret(typeof(getfield(hdr, sym)), th[bst:bst+nb])[1] - bigendian && (val = bswap(val)) + swap_bytes && (val = bswap(val)) setfield!(hdr, sym, val) end nothing end -read_traceheader!(s::IO, thb::Dict{String, Int32}, hdr::BinaryTraceHeader; be::Bool = true, th=zeros(UInt8, 240)) = - read_traceheader!(s, collect(keys(thb)), thb, hdr; be=be, th=th) \ No newline at end of file +read_traceheader!(s::IO, thb::Dict{String, Int32}, hdr::BinaryTraceHeader; swap_bytes::Bool, th=zeros(UInt8, 240)) = + read_traceheader!(s, collect(keys(thb)), thb, hdr; swap_bytes=swap_bytes, th=th) \ No newline at end of file diff --git a/src/scan/scan_block.jl b/src/scan/scan_block.jl index 30197c7..ed8ade6 100644 --- a/src/scan/scan_block.jl +++ b/src/scan/scan_block.jl @@ -2,7 +2,11 @@ export scan_block function scan_block(buf::IO, mem_block::Int, mem_trace::Int, keys::Array{String,1}, chunk_start::Int, file::String, th_byte2sample::Dict{String, Int32}) - + + f = open(file) + swap_bytes = bswap_needed(f) + close(f) + # Calc info about this block startbyte = position(buf) + chunk_start ntraces_block = Int(mem_block/mem_trace) @@ -12,7 +16,7 @@ function scan_block(buf::IO, mem_block::Int, mem_trace::Int, keys::Array{String, # Read all headers and record end byte while !eof(buf) && count>> 9) | convert(UInt32,exp << 23) | sgn) :: Float32 -import Base.convert, Base.Float32 +import Base.convert, Base.Float32, Base.bswap function convert(::Type{Float32}, ibm::IBMFloat32) local fr::UInt32 = ntoh(reinterpret(UInt32, ibm)) @@ -29,3 +29,7 @@ function convert(::Type{Float32}, ibm::IBMFloat32) end Float32(ibm::IBMFloat32) = convert(Float32,ibm) ## From JuliaSeis + +function bswap(ibm::IBMFloat32) + return reinterpret(IBMFloat32, bswap(reinterpret(UInt32, ibm))) +end \ No newline at end of file From d7b3c773240c68b4cfd3725dab1bd0f34d047782 Mon Sep 17 00:00:00 2001 From: Kerim Khemraev Date: Tue, 6 Jun 2023 16:10:09 +0300 Subject: [PATCH 2/3] Edited tests --- test/test_read.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_read.jl b/test/test_read.jl index 422dfc0..d431c84 100644 --- a/test/test_read.jl +++ b/test/test_read.jl @@ -7,25 +7,25 @@ global s = IOBuffer(read(joinpath(SegyIO.myRoot,"data/overthrust_2D_shot_1_20.se ##0000 @testset "read_fileheader" begin - fh = read_fileheader(s) + fh = read_fileheader(s, swap_bytes = true) @test typeof(fh) == SegyIO.FileHeader @test sizeof(fh.th) == 3200 @test fh.bfh.ns == 751 @test fh.bfh.Job == 1 - fh = read_fileheader(s, ["ns"; "Job"]) + fh = read_fileheader(s, ["ns"; "Job"], swap_bytes = true) @test typeof(fh) == SegyIO.FileHeader @test sizeof(fh.th) == 3200 @test fh.bfh.ns == 751 @test fh.bfh.Job == 1 - fh = read_fileheader(s, bigendian = false ) + fh = read_fileheader(s, swap_bytes = false) @test typeof(fh) == SegyIO.FileHeader @test sizeof(fh.th) == 3200 @test fh.bfh.ns == -4350 @test fh.bfh.Job == 16777216 - fh = read_fileheader(s, ["ns"; "Job"], bigendian = false ) + fh = read_fileheader(s, ["ns"; "Job"], swap_bytes = false) @test typeof(fh) == SegyIO.FileHeader @test sizeof(fh.th) == 3200 @test fh.bfh.ns == -4350 @@ -40,27 +40,27 @@ global s = IOBuffer(read(joinpath(SegyIO.myRoot,"data/overthrust_2D_shot_1_20.se seek(s, 3600) th_b2s = th_byte2sample() - th = read_traceheader(s, th_b2s) + th = read_traceheader(s, th_b2s, swap_bytes = true) @test typeof(th) == BinaryTraceHeader @test th.ns == 751 @test th.SourceX == 400 @test th.GroupX == 100 seek(s, 3600) - th = read_traceheader(s, ["ns", "SourceX", "GroupX"], th_b2s) + th = read_traceheader(s, ["ns", "SourceX", "GroupX"], th_b2s, swap_bytes = true) @test typeof(th) == BinaryTraceHeader @test th.ns == 751 @test th.SourceX == 400 @test th.GroupX == 100 seek(s, 3600) - th = read_traceheader(s, th_b2s, bigendian = false) + th = read_traceheader(s, th_b2s, swap_bytes = false) @test typeof(th) == BinaryTraceHeader @test th.ns == -4350 @test th.SourceX == -1878982656 seek(s, 3600) - th = read_traceheader(s, ["ns", "SourceX", "GroupX"], th_b2s, bigendian = false) + th = read_traceheader(s, ["ns", "SourceX", "GroupX"], th_b2s, swap_bytes = false) @test typeof(th) == BinaryTraceHeader @test th.ns == -4350 @test th.SourceX == -1878982656 From 98c0c6774a6fb38e583e839706f7f750115eb0d4 Mon Sep 17 00:00:00 2001 From: kerim371 Date: Wed, 7 Jun 2023 22:34:22 +0300 Subject: [PATCH 3/3] Removed unused func and don't swap bytes when writing SEGY If we swap bytes of traces and headers then SEGY will always be of another endianness than machine. No need to swap bytes when writing SEGY. --- src/read/endianness.jl | 42 +---------------------------------- src/write/write_fileheader.jl | 4 ++-- src/write/write_trace.jl | 4 ++-- 3 files changed, 5 insertions(+), 45 deletions(-) diff --git a/src/read/endianness.jl b/src/read/endianness.jl index 71efde8..e42cb05 100644 --- a/src/read/endianness.jl +++ b/src/read/endianness.jl @@ -1,43 +1,3 @@ -is_machine_littleendian() = Base.ENDIAN_BOM == 0x04030201 -is_machine_bigendian() = Base.ENDIAN_BOM == 0x01020304 - -function is_segy_littleendian(s::IO) - bfh = BinaryFileHeader() - fh_b2s = fh_byte2sample() - - old_pos = position(s) - seek(s, fh_b2s["DataSampleFormat"]) - dsf = read(s, typeof(bfh.DataSampleFormat)) - dsf_bs = bswap(dsf) - seek(s, old_pos) - - if (dsf >= 1 && dsf <= 16 && is_machine_littleendian()) || - (dsf_bs >= 1 && dsf_bs <= 16 && is_machine_bigendian()) - return true - else - return false - end -end - -function is_segy_bigendian(s::IO) - bfh = BinaryFileHeader() - fh_b2s = fh_byte2sample() - - old_pos = position(s) - seek(s, fh_b2s["DataSampleFormat"]) - dsf = read(s, typeof(bfh.DataSampleFormat)) - dsf = bswap(read(s, typeof(bfh.DataSampleFormat))) - seek(s, old_pos) - - if (dsf >= 1 && dsf <= 16 && is_machine_bigendian()) || - (dsf_bs >= 1 && dsf_bs <= 16 && is_machine_littleendian()) - return true - else - return false - end -end - - """ bswap_needed(s::IO) @@ -65,4 +25,4 @@ function bswap_needed(s::IO) else return true end -end \ No newline at end of file +end diff --git a/src/write/write_fileheader.jl b/src/write/write_fileheader.jl index d724d43..8748dbd 100644 --- a/src/write/write_fileheader.jl +++ b/src/write/write_fileheader.jl @@ -24,7 +24,7 @@ function write_fileheader(s::IO, fh::FileHeader) ##3200 # Write first section of assigned values for field in fieldnames(typeof(fh.bfh))[1:27] - write(s, bswap(getfield(fh.bfh, field))) + write(s, getfield(fh.bfh, field)) end ##3260 @@ -34,7 +34,7 @@ function write_fileheader(s::IO, fh::FileHeader) ##3500 # Write second section of assigned values for field in fieldnames(typeof(fh.bfh))[28:end] - write(s, bswap(getfield(fh.bfh, field))) + write(s, getfield(fh.bfh, field)) end ##3506 diff --git a/src/write/write_trace.jl b/src/write/write_trace.jl index 47c5f0f..e7c14aa 100644 --- a/src/write/write_trace.jl +++ b/src/write/write_trace.jl @@ -5,10 +5,10 @@ function write_trace(s::IO, block::SeisBlock, t::Int) ##000 # Write Header for field in fieldnames(typeof(block.traceheaders[t])) - write(s, bswap(getfield(block.traceheaders[t], field))) + write(s, getfield(block.traceheaders[t], field)) end ##240 # Write trace - write(s, bswap.(Float32.(block.data[:,t]))) + write(s, Float32.(block.data[:,t])) end