From c61cf126b62d8bb23565dbdcc580e48e193c68c1 Mon Sep 17 00:00:00 2001 From: Daniel Caballero <132241327+dlcaballero16@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:19:01 -0500 Subject: [PATCH] Update ultralite data (#504) * Initial commit, trying things to get ultralite data working * Cleaned up application.yml, added script and required files to run ultralite data * Undo changes and remove prints * Make unit tests pass * Added use of RemoveLogs to make run even smaller --- src/snapred/backend/data/LocalDataService.py | 13 +- src/snapred/resources/application.yml | 10 + .../ultralite/CRACKLEFocGroup_Column.xml | 21 ++ .../ultralite/CRACKLELiteDataMap.xml | 59 +++++ .../ultralite/CRACKLELite_Definition.xml | 237 ++++++++++++++++++ .../ultralite/CRACKLE_Definition.xml | 221 ++++++++++++++++ .../ultralite/create_ultralite_data.py | 140 +++++++++++ tests/resources/application.yml | 3 + 8 files changed, 698 insertions(+), 6 deletions(-) create mode 100644 src/snapred/resources/ultralite/CRACKLEFocGroup_Column.xml create mode 100644 src/snapred/resources/ultralite/CRACKLELiteDataMap.xml create mode 100644 src/snapred/resources/ultralite/CRACKLELite_Definition.xml create mode 100644 src/snapred/resources/ultralite/CRACKLE_Definition.xml create mode 100644 src/snapred/resources/ultralite/create_ultralite_data.py diff --git a/src/snapred/backend/data/LocalDataService.py b/src/snapred/backend/data/LocalDataService.py index 0fe667c07..8d0840555 100644 --- a/src/snapred/backend/data/LocalDataService.py +++ b/src/snapred/backend/data/LocalDataService.py @@ -801,8 +801,9 @@ def readDetectorState(self, runId: str) -> DetectorState: detectorState = None pvFile = self._readPVFile(runId) wav_value = None - wav_key_1 = "entry/DASlogs/BL3:Chop:Gbl:WavelengthReq/value" - wav_key_2 = "entry/DASlogs/BL3:Chop:Skf1:WavelengthUserReq/value" + logsLocation = Config["constants.logsLocation"] + wav_key_1 = f"{logsLocation}/BL3:Chop:Gbl:WavelengthReq/value" + wav_key_2 = f"{logsLocation}/BL3:Chop:Skf1:WavelengthUserReq/value" if wav_key_1 in pvFile: wav_value = pvFile.get(wav_key_1)[0] @@ -813,11 +814,11 @@ def readDetectorState(self, runId: str) -> DetectorState: try: detectorState = DetectorState( - arc=[pvFile.get("entry/DASlogs/det_arc1/value")[0], pvFile.get("entry/DASlogs/det_arc2/value")[0]], + arc=[pvFile.get(f"{logsLocation}/det_arc1/value")[0], pvFile.get(f"{logsLocation}/det_arc2/value")[0]], wav=wav_value, - freq=pvFile.get("entry/DASlogs/BL3:Det:TH:BL:Frequency/value")[0], - guideStat=pvFile.get("entry/DASlogs/BL3:Mot:OpticsPos:Pos/value")[0], - lin=[pvFile.get("entry/DASlogs/det_lin1/value")[0], pvFile.get("entry/DASlogs/det_lin2/value")[0]], + freq=pvFile.get(f"{logsLocation}/BL3:Det:TH:BL:Frequency/value")[0], + guideStat=pvFile.get(f"{logsLocation}/BL3:Mot:OpticsPos:Pos/value")[0], + lin=[pvFile.get(f"{logsLocation}/det_lin1/value")[0], pvFile.get(f"{logsLocation}/det_lin2/value")[0]], ) except (TypeError, KeyError) as e: raise ValueError(f"Could not find all required logs in file '{self._constructPVFilePath(runId)}': {e}") diff --git a/src/snapred/resources/application.yml b/src/snapred/resources/application.yml index 8d6c5ba56..b4658b4b9 100644 --- a/src/snapred/resources/application.yml +++ b/src/snapred/resources/application.yml @@ -29,17 +29,23 @@ instrument: # Here "{IPTS}" will be substituted with the IPTS-directory name for a specified home: ${instrument.home}/{IPTS}/shared/SNAPRed + # Swap the commented out fields when using ultralite data config: ${instrument.calibration.home}/SNAPInstPrm.json native: pixelResolution: 1179648 + # pixelResolution: 72 definition: file: ${module.root}/resources/SNAP_Definition.xml + # file: ${module.root}/resources/ultralite/CRACKLE_Definition.xml lite: pixelResolution: 18432 + # pixelResolution: 18 definition: file: ${module.root}/resources/SNAPLite_Definition.xml + # file: ${module.root}/resources/ultralite/CRACKLELite_Definition.xml map: file: ${instrument.calibration.home}/Powder/LiteGroupMap.hdf + # file: ${module.root}/resources/ultralite/CRACKLELiteDataMap.xml startingRunNumber: 10000 minimumRunNumber: 46342 maxNumberOfRuns: 10 @@ -190,6 +196,10 @@ constants: PeakIntensityFractionThreshold: 0.05 m2cm: 10000.0 # conversion factor for m^2 to cm^2 maskedPixelThreshold: 0.15 + # Swap these when running with ultralite data + logsLocation: "entry/DASlogs" + # logsLocation: "/mantid_workspace_1/logs" + CrystallographicInfo: crystalDMin: 0.4 diff --git a/src/snapred/resources/ultralite/CRACKLEFocGroup_Column.xml b/src/snapred/resources/ultralite/CRACKLEFocGroup_Column.xml new file mode 100644 index 000000000..3a6fe2fb8 --- /dev/null +++ b/src/snapred/resources/ultralite/CRACKLEFocGroup_Column.xml @@ -0,0 +1,21 @@ + + + + 0-11 + + + 12-23 + + + 24-35 + + + 36-47 + + + 48-59 + + + 60-71 + + diff --git a/src/snapred/resources/ultralite/CRACKLELiteDataMap.xml b/src/snapred/resources/ultralite/CRACKLELiteDataMap.xml new file mode 100644 index 000000000..81f4bcd02 --- /dev/null +++ b/src/snapred/resources/ultralite/CRACKLELiteDataMap.xml @@ -0,0 +1,59 @@ + + + + + 0,1,2,3 + + + 4,5,6,7 + + + 8,9,10,11 + + + 12,13,14,15 + + + 16,17,18,19 + + + 20,21,22,23 + + + 24,25,26,27 + + + 28,29,30,31 + + + 32,33,34,35 + + + + 36,37,38,39 + + + 40,41,42,43 + + + 44,45,46,47 + + + 48,49,50,51 + + + 52,53,54,55 + + + 56,57,58,59 + + + 60,61,62,63 + + + 64,65,66,67 + + + 68,69,70,71 + + diff --git a/src/snapred/resources/ultralite/CRACKLELite_Definition.xml b/src/snapred/resources/ultralite/CRACKLELite_Definition.xml new file mode 100644 index 000000000..f849c97b6 --- /dev/null +++ b/src/snapred/resources/ultralite/CRACKLELite_Definition.xml @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/snapred/resources/ultralite/CRACKLE_Definition.xml b/src/snapred/resources/ultralite/CRACKLE_Definition.xml new file mode 100644 index 000000000..05e80e474 --- /dev/null +++ b/src/snapred/resources/ultralite/CRACKLE_Definition.xml @@ -0,0 +1,221 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/snapred/resources/ultralite/create_ultralite_data.py b/src/snapred/resources/ultralite/create_ultralite_data.py new file mode 100644 index 000000000..21f427938 --- /dev/null +++ b/src/snapred/resources/ultralite/create_ultralite_data.py @@ -0,0 +1,140 @@ +# import mantid algorithms, numpy and matplotlib + +from mantid.simpleapi import * +from snapred.backend.dao.ingredients.GroceryListItem import GroceryListItem +from snapred.backend.data.DataFactoryService import DataFactoryService +from snapred.backend.data.GroceryService import GroceryService +from snapred.meta.Config import Resource + +Resource._resourcesPath = os.path.expanduser("~/SNS/SNAP/shared/Calibration_next/Powder/") +liteInstrumentFile = Resource.getPath("CRACKLE_Definition.xml") +dfs = DataFactoryService() + + +def superID(nativeID, xdim, ydim): + # accepts a numpy array of native ID from standard SNAP nexus file and returns a numpy array with + # super pixel ID according to provided dimensions xdim and ydim of the super pixel. + # xdim and ydim shall be multiples of 2 + + Nx = 256 # native number of horizontal pixels + Ny = 256 # native number of vertical pixels + NNat = Nx * Ny # native number of pixels per panel + + firstPix = (nativeID // NNat) * NNat + redID = nativeID % NNat # reduced ID beginning at zero in each panel + + (i, j) = divmod(redID, Ny) # native (reduced) coordinates on pixel face + superi = divmod(i, xdim)[0] + superj = divmod(j, ydim)[0] + + # some basics of the super panel + superNx = Nx / xdim # 32 running from 0 to 31 + superNy = Ny / ydim + superN = superNx * superNy + + superFirstPix = (firstPix / NNat) * superN + + superVal = superi * superNy + superj + superFirstPix + + return superVal + + +# create the mapping +LoadEmptyInstrument( + Filename="/SNS/SNAP/shared/Malcolm/dataFiles/SNAP_Definition.xml", + OutputWorkspace="SNAP", +) + +mapToCrackle = "map_from_SNAP_to_CRACKLE" +if mapToCrackle not in mtd: + # create the lite grouping ws using input run as template + CreateGroupingWorkspace( + InputWorkspace="SNAP", + GroupDetectorsBy="All", + OutputWorkspace=mapToCrackle, + ) + ws = mtd[mapToCrackle] + nHst = ws.getNumberHistograms() + for spec in range(nHst): + ws.setY(spec, [superID(spec, 128, 128) + 1]) + +# select run to convert to ultralite data, can convert multiple runs at once +runs_to_reduce = ["58882"] # ["46680", "58810", "58813", "57514"] + +clerk = GroceryListItem.builder() +for x in runs_to_reduce: + clerk.neutron(x).native().add() +groceries = GroceryService().fetchGroceryList(clerk.buildList()) + + +# The FileName should point to a "diffract_consts__v#.h5 file, this gets saved at the end of a diffcal run +LoadDiffCal( + InputWorkspace=groceries[0], + FileName="/SNS/users/8l2/SNS/SNAP/shared/Calibration_next/Powder/04bd2c53f6bf6754/native/diffraction/v_0003/diffract_consts_057514_v0003.h5", + WorkspaceName="57514", +) +# If set to False, will output data as histograms +eventMode = True + +for grocery in groceries: + ws = mtd[grocery] + ultralite = f"{grocery}_ULTRALITE" + CloneWorkspace( + InputWorkspace=grocery, + OutputWorkspace=ultralite, + ) + ConvertUnits( + InputWorkspace=ultralite, + OutputWorkspace=ultralite, + Target="dSpacing", + ) + if not eventMode: + uws = mtd[ultralite] + Rebin(InputWorkspace=ultralite, OutputWorkspace=ultralite, Params=(uws.getTofMin(), -0.001, uws.getTofMax())) + DiffractionFocussing( + InputWorkspace=ultralite, + OutputWorkspace=ultralite, + GroupingWorkspace=mapToCrackle, + PreserveEvents=eventMode, + ) + LoadInstrument( + Workspace=ultralite, + Filename=liteInstrumentFile, + RewriteSpectraMap=True, + ) + ConvertUnits( + InputWorkspace=ultralite, + OutputWorkspace=ultralite, + Target="TOF", + ) + if eventMode: + CompressEvents( + InputWorkspace=ultralite, + OutputWorkspace=ultralite, + BinningMode="Logarithmic", + Tolerance=-0.0001, + ) + uws = mtd[ultralite] + Rebin(InputWorkspace=ultralite, OutputWorkspace=ultralite, Params=(uws.getTofMax() - uws.getTofMin())) + logs = ( + "BL3:Det:TH:BL:Frequency", + "BL3:Mot:OpticsPos:Pos", + "BL3:Chop:Gbl:WavelengthReq", + "BL3:Chop:Skf1:WavelengthUserReq", + "BL3:Chop:Gbl:WavelengthReq", + "BL3:Chop:Skf1:WavelengthUserReq", + "det_arc1", + "det_arc2", + "BL3:Det:TH:BL:Frequency", + "BL3:Mot:OpticsPos:Pos", + "det_lin1", + "det_lin2", + "proton_charge", + "gd_prtn_chrg", + ) + RemoveLogs(Workspace=ultralite, KeepLogs=logs) + SaveNexusProcessed( + InputWorkspace=ultralite, + Filename=f"~/Documents/ultralite/{ultralite}.nxs.h5", + CompressNexus=True, + ) diff --git a/tests/resources/application.yml b/tests/resources/application.yml index 9517ed290..567120867 100644 --- a/tests/resources/application.yml +++ b/tests/resources/application.yml @@ -209,6 +209,9 @@ constants: PeakIntensityFractionThreshold: 0.05 m2cm: 10000.0 # conversion factor for m^2 to cm^2 maskedPixelThreshold: 0.15 + # Swap these when running with ultralite data + logsLocation: "entry/DASlogs" + # logsLocation: "/mantid_workspace_1/logs" CrystallographicInfo: crystalDMin: 0.4