; Reference:
; Himawari-8/9
; Himawari Standard Data
; User's Guide
; Version 1.3
; 3 July, 2017
; Japan Meteorological Agency
; 1-3-4 Otemachi, Chiyoda-ku, Tokyo, 100-8122 Japan
;
;Found here:
; https://www.data.jma.go.jp/mscweb/en/himawari89/space_segment/hsd_sample/HS_D_users_guide_en_v13.pdf
;
; We use fixed-length block definitions to read the data from
; file, but then convert them to hash storage after input. Among other things,
; fixed-length byte strings are converted to IDL strings.
Pro HimawariStandardBlock1Data__Define
; #1 Basic information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock1Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
TotalHeaderBlocks : 0U, $
ByteOrder : 0B, $
SatelliteName_B : BytArr(16), $
ProcessingCenter_B : BytArr(16), $
ObservationArea_B : BytArr(4), $
OtherObservationInfo_B : BytArr(2), $
ObservationTimeline : 0U, $
ObservationStartTime : 0D, $
ObservationEndTime : 0D, $
FileCreationTime : 0D, $
TotalHeaderLength : 0UL, $
TotalDataLength : 0UL, $
QualityFlag1 : 0B, $
QualityFlag2 : 0B, $
QualityFlag3 : 0B, $
QualityFlag4 : 0B, $
FileFormatVersion_B : BytArr(32), $
FileName_B : BytArr(128), $
Spare : BytArr(40)}
End
Pro HimawariStandardBlock2Data__Define
; #2 Data information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock2Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
NumberOfBitsPerPixel : 0U, $
NumberOfColumns : 0U, $
NumberOfLines : 0U, $
CompressFlagForDataBlock12 : 0B, $
Spare : BytArr(40) $
}
End
Pro HimawariStandardBlock3Data__Define
; #3 Projection information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock3Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
Sub_Lon : 0D, $
CFAC : 0UL, $
LFAC : 0UL, $
COFF : 0., $
LOFF : 0., $
DistanceFromEarthCenterToVirtualSatellite : 0D, $
EarthsEquatorialRadius : 0D, $
EarthsPolarRadius : 0D, $
F1 : 0D, $
F2 : 0D, $
F3 : 0D, $
Coefficient : 0D, $
ResamplingTypes : 0U, $
ResamplingSize : 0U, $
Spare : BytArr(40) $
}
End
Pro HimawariStandardBlock4Data__Define
; #4 Navigation information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock4Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
NavigationInformationTime : 0D, $
SSPLongitude : 0D, $
SSPLatitude : 0D, $
DistanceFromEarthsCenterToSatellite : 0D, $
NadirLongitude : 0D, $
NadirLatitude : 0D, $
SunsPosition : DblArr(3), $
MoonsPosition : DblArr(3), $
Spare : BytArr(40) $
}
End
Pro HimawariStandardBlock5Data__Define
; #5 Calibration information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock5Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
BandNumber : 0U, $
CentralWavelength : 0D, $
ValidNumberOfBitsPerPixel : 0U, $
CountValueofErrorPixels : 0U, $
CountValueOfPixelsOutsideScanArea : 0U, $
SlopeForCountRadianceConversion : 0D, $
InterceptForCountRadianceConversion : 0D, $
CorrectionCoefficientRadianceToBrightnessTemperaturec0 : 0D, $
CorrectionCoefficientRadianceToBrightnessTemperaturec1 : 0D, $
CorrectionCoefficientRadianceToBrightnessTemperaturec2 : 0D, $
CorrectionCoefficientForSensorBrightnessTemperatureToRadiancec0 : 0D, $
CorrectionCoefficientForSensorBrightnessTemperatureToRadiancec1 : 0D, $
CorrectionCoefficientForSensorBrightnessTemperatureToRadiancec2 : 0D, $
SpeedOfLight : 0D, $
PlanckConstant : 0D, $
BoltzmannConstant : 0D, $
Spare : BytArr(40) $
}
End
Pro HimawariStandardBlock6Data__Define
; #6 Inter-calibration information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock6Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
GSICSCalibrationCoefficientIntercept : 0D, $
GSICSCalibrationCoefficientSlope : 0D, $
GSICSCalibrationCoefficienQuadtraticTerm : 0D, $
RadianceBiasForStandardScene : 0D, $
UncertaintyOfRadianceBiasForStandardScene : 0D, $
RadianceForStandardScene : 0D, $
StartTimeOfGCSICSCorrectionValidityPeriod : 0D, $
EndTimeOfGSCISCorrectionValidityPeriod: 0D, $
RaidanceValidityRangeOfGSICSCalibrationCoefficientsUpperLimit : 0., $
RaidanceValidityRangeOfGSICSCalibrationCoefficientsLowerLimit : 0., $
FileNameOfGSICSCorrection_B : Bytarr(128), $
Spare : BytArr(56) $
}
End
Pro HimawariStandardBlock7Data__Define
; #7 Segment information block
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock7Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
TotalNumberOfSegments : 0B, $
SegmentSequenceNumber : 0B, $
FirstLineNumberOfImageSegment : 0U, $
Spare : BytArr(40) $
}
End
Pro HimawariStandardBlock8Data__Define
; #8 Navigation correction information block
; Block 8 has an indeterminate number of extra records for
; shift data, along with a Spare vector that are added
; later.
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock8Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
CenterColumnOfRotation : 0., $
CenterLineOfRotation : 0., $
AmountOfRotationalCorrection : 0D, $
NumberofCorrectionInformationDataForColumnAndLineDirection : 0U $
}
End
Pro HimawariStandardBlock8ShiftData__Define
; Block 8 may have 0 to N of these shift data
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock8ShiftData, $
LineNumberAfterRotation : 0U, $
ShiftAmountForColumnDirection : 0., $
ShiftAmountForLineDirection : 0. $
}
End
Pro HimawariStandardBlock9Data__Define
; #9 Observation time information block
; Block 9 has an indeterminate number of extra records for
; observation time data, along with a Spare vector that are added
; later.
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock9Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
NumberOfObservationTimes : 0U $
}
End
Pro HimawariStandardBlock9ObservationTimeData__Define
; Block 9 may have 0 to N of these observation time data
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock9ObservationTimeData, $
LineNumber : 0U, $
ObservationTime : 0D $
}
End
Pro HimawariStandardBlock10Data__Define
; #10 Error information block
; Block 10 has an indeterminate number of extra records for
; error information data, along with a Spare vector that are added
; later.
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock10Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0L, $
NumberOfErrorInformationData : 0U $
}
End
Pro HimawariStandardBlock10ErrorInformationData__Define
; Block 10 may have 0 to N of these error information data
Compile_Opt StrictArr, Hidden
!null = {HimawariStandardBlock10ErrorInformationData, $
LineNumber : 0U, $
NumberOfErrorPixelsPerLine : 0U $
}
End
Pro HimawariStandardBlock11Data__Define
; #11 Spare block
Compile_Opt Strictarr, Hidden
!null = {HimawariStandardBlock11Data, $
HeaderBlockNumber : 0B, $
BlockLength : 0U, $
Spare : BytArr(256) $
}
End
Pro HimawariStandardBlock__Define
Compile_Opt StrictArr
On_Error, 2
!null = {HimawariStandardBlock, $
Inherits OrderedHash, $
BlockNumber : 0U $
}
End
Function HimawariStandardBlock::Init, BlockNumber
Compile_Opt StrictArr
On_Error, 2
If (~self.OrderedHash::Init()) then Return, 0
self.BlockNumber = BlockNumber
Return, 1
End
Function HimawariStandardBlock::Read, LUN, Error = Error
Compile_Opt StrictArr
On_Error, 2
Error = !null
Catch, ErrorNumber
If (ErrorNumber ne 0) then Begin
Error = !error_state.msg + ', Block ' + self.BlockNumber.ToString()
Return, !false
EndIf
Case self.BlockNumber of
1 : Data = {HimawariStandardBlock1Data}
2 : Data = {HimawariStandardBlock2Data}
3 : Data = {HimawariStandardBlock3Data}
4 : Data = {HimawariStandardBlock4Data}
5 : Data = {HimawariStandardBlock5Data}
6 : Data = {HimawariStandardBlock6Data}
7 : Data = {HimawariStandardBlock7Data}
8 : Data = {HimawariStandardBlock8Data}
9 : Data = {HimawariStandardBlock9Data}
10 : Data = {HimawariStandardBlock10Data}
11 : Data = {HimawariStandardBlock11Data}
Else : Message, 'Invalid block number ' + self.BlockNumber.ToString(), /Traceback
EndCase
; We're using structures to read the data, then converting the info
; into a hash with key/value pairs, instead. The lowercase version
; of the tag name in the structure becomes the corresponding key,
; with the slight exception of strings.
ReadU, LUN, Data
Tags = Tag_Names(Data)
ForEach Tag, Tags, Index Do Begin
If (Tag.EndsWith('_B')) then Begin
; Structure tags that are defined as fixed-length byte arrays
; are converted to IDL strings, and we strip the "_B" from
; the end of the tag name for use as the key.
self[(Tag.Split('_'))[0].ToLower()] = String(Data.(Index))
EndIf Else Begin
self[Tag.ToLower()] = Data.(Index)
EndElse
EndForEach
; Blocks 8 through 10 have an indeterminate number of additional records,
; followed by an allocation of spare data.
Case self.BlockNumber of
8 : Begin
ShiftData = List()
Data = {HimawariStandardBlock8ShiftData}
Tags = Tag_Names(Data)
For I = 0L, Long(self['numberofcorrectioninformationdataforcolumnandlinedirection']) - 1 Do Begin
ReadU, LUN, Data
Items = OrderedHash()
ForEach Tag, Tags, Index Do Begin
Items[Tag.ToLower()] = Data.(Index)
EndForEach
ShiftData.Add, Items
EndFor
self['shiftdata'] = ShiftData
Spare = BytArr(40)
ReadU, LUN, Spare
self['spare'] = Spare
End
9 : Begin
ObservationTimes = List()
Data = {HimawariStandardBlock9ObservationTimeData}
Tags = Tag_Names(Data)
For I = 0L, Long(self['numberofobservationtimes']) - 1 Do Begin
ReadU, LUN, Data
Items = OrderedHash()
ForEach Tag, Tags, Index Do Begin
Items[Tag.ToLower()] = Data.(Index)
EndForEach
ObservationTimes.Add, Items
EndFor
self['observationtimes'] = ObservationTimes
Spare = BytArr(40)
ReadU, LUN, Spare
self['spare'] = Spare
End
10 : Begin
ErrorInformation = List()
Data = {HimawariStandardBlock10ErrorInformationData}
Tags = Tag_Names(Data)
For I = 0L, Long(self['numberoferrorinformationdata']) - 1 Do Begin
ReadU, LUN, Data
Items = OrderedHash()
ForEach Tag, Tags, Index Do Begin
Items[Tag.ToLower()] = Data.(Index)
EndForEach
ErrorInformation.Add, Data
EndFor
self['errorinformation'] = ErrorInformation
Spare = BytArr(40)
ReadU, LUN, Spare
self['spare'] = Spare
End
Else :
EndCase
Return, !true
End
Pro HimawariStandard::Cleanup
Compile_Opt StrictArr
On_Error, 2
If (self.LUN ne 0) then Begin
; Close the data file
Free_LUN, self.LUN, /Force
self.LUN = 0
EndIf
End
Pro HimawariStandard::GetProperty, $
Image_Data = Image_Data, $
Metadata = Metadata
Compile_Opt StrictArr
On_Error, 2
If (Arg_Present(Image_Data)) then Begin
; Return a copy of the image data.
Image_Data = *self.pImage
EndIf
If (Arg_Present(Metadata)) then Begin
; Return the complete hash of blocks 1 through 11
Metadata = self.Blocks
EndIf
End
Function HimawariStandard::Open, File, Error = Error
Compile_Opt StrictArr
On_Error, 2
Error = !null
If (File ne !null) then Begin
self.File = File
Status = self.OpenFile(Error = Error)
Return, Status
EndIf
Return, !False
End
Function HimawariStandard::Init, $
File = File, Error = Error
Compile_Opt StrictArr
On_Error, 2
Error = !null
If (File ne !null) then Begin
Status = self.Open(File, Error = Error)
Return, Status
EndIf
Return, 1
End
Function HimawariStandard::OpenFile, Error = Error
Compile_Opt StrictArr
On_Error, 2
Error = !null
Catch, ErrorNumber
If (ErrorNumber ne 0) then Begin
Catch, /Cancel
Error = !error_state.msg
If (self.LUN ne 0) then Begin
Free_LUN, self.LUN, /Force
EndIf
self.LUN = 0
Return, !false
EndIf
If (~File_Test(self.File)) then Begin
Error = 'File does not exist'
Return, !false
EndIf
; Check if byte order of the data matches the OS and use SWAP_ENDIAN on OpenR, if necessary.
OpenR, LUN, self.File, /Get_LUN
DummyBuffer = BytArr(5) ; Skip the first 5 bytes
ReadU, LUN, DummyBuffer
ByteOrder = 0B
ReadU, LUN, ByteOrder
If (ByteOrder eq 1 && (!d.name).ToUpper() eq 'WIN') || $
(ByteOrder eq 0 && (!d.name).ToUpper() eq 'UNIX') then Begin
Free_LUN, LUN, /Force
OpenR, LUN, self.File, /Get_LUN, Swap_Endian = 1
EndIf Else Begin
; No byte-swapping, rewind the file pointer.
Point_LUN, LUN, 0
EndElse
self.LUN = LUN
Return, !true
End
Function HimawariStandard::Read, Error = Error
Compile_Opt StrictArr
On_Error, 2
self.Blocks = OrderedHash()
For I = 1, 11 Do Begin
Block = Obj_New('HimawariStandardBlock', I)
Status = Block.Read(self.LUN, Error = Error)
If (~Status) then Return, !false
; Notice that we're using a hash rather than a list to store the blocks
; so we "index" using 1-based integer key values 1 through 11 corresponding to
; the defined block numbers instead of list index values 0 through 10.
self.Blocks[I] = Block
EndFor
; "Block 12" has no metadata, only image data, so we simply
; put the image into a pointer, instead of constructing more hash info.
Image = UIntArr(self.Blocks[2, 'numberofcolumns'], self.Blocks[2, 'numberoflines'])
; Technically, we should check the compression flag from Block 2 rather than
; assuming the image in uncompressed. What is the compression scheme if it
; is compressed?
ReadU, self.LUN, Image
self.pImage = Ptr_New(Image)
Free_LUN, self.LUN, /Force
self.LUN = 0
Return, !true
End
Function HimawariStandard::RadianceToBrightnessTemperature, RadianceData, Metadata, $
Replace_Bad = ReplaceBad
; Raidance in W/(m^2 steradian micron) to brightness temperature
; Note: This is a static method
Compile_Opt StrictArr, Static
On_Error, 2
Exceptsave = !except
; Radiance data expected to come from ::CountToRadiance where "bad" pixels
; have been replaced set to 0.
Bad = Where(RadianceData eq 0, NBad, Complement = Good)
M = Metadata[5]
!except = 0
Num = (M['planckconstant'] * M['speedoflight'])/(M['boltzmannconstant']*M['centralwavelength'])
Denom = ALog2((2.*M['planckconstant']*M['speedoflight']^2)/((M['centralwavelength']^5)*RadianceData) + 1.d)
Te = Num/Denom
Tb = M['correctioncoefficientradiancetobrightnesstemperaturec0'] + $
M['correctioncoefficientradiancetobrightnesstemperaturec1']*Te + $
M['correctioncoefficientradiancetobrightnesstemperaturec2']*Te^2
If (NBad ne 0) then Tb[Bad] = ReplaceBad ne !null ? ReplaceBad : Min(Tb[Good], /NAN)
!null = Check_Math()
!except = ExceptSave
Return, Tb
End
Function HimawariStandard::CountToRadiance, ImageData, Metadata
; counts to W/(m^2 steradian micron)
; Note: this is a static method
Compile_Opt StrictArr, Static
On_Error, 2
Bad = Where(ImageData ge 65534, NBad)
M = Metadata[5]
RadianceData = (ImageData * M['slopeforcountradianceconversion'] + $
M['interceptforcountradianceconversion']) > 0
If (NBad ne 0) then RadianceData[Bad] = 0
Return, RadianceData
End
Pro HimawariStandard__Define
Compile_Opt StrictArr
On_Error, 2
!null = {HimawariStandard, $
Inherits IDL_Object, $
File : '', $
LUN : 0L, $
Blocks : Obj_New(), $
pImage : Ptr_New() $
}
End